PapiRunner.kt 8.56 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
3
4
package se.lth.cs

import papi.EventSet
import papi.Papi
5
import papi.PapiException
6
import java.io.FileWriter
7
8
import java.time.Duration
import java.time.Instant
9
10
import se.lth.cs.bcgen.*
import org.openjdk.jmh.infra.Blackhole
Noric Couderc's avatar
Noric Couderc committed
11

12
open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
13

14
15
16
    init {
        Papi.init()
    }
17
18

    val counterSpec = counters
19
    val numRuns = numRuns
20

Noric Couderc's avatar
Noric Couderc committed
21
22
23
24
25
26
27
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
    fun emptyBenchmark(): MutableMap<String, List<Long>> {
        // For each counter,
        // we store the values for each run (10 runs)
Noric Couderc's avatar
Noric Couderc committed
28
        var data: MutableMap<String, List<Long>> = mutableMapOf()
Noric Couderc's avatar
Noric Couderc committed
29

30
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
31
32
            val evset = EventSet.create(kvp.value)

Noric Couderc's avatar
Noric Couderc committed
33
            val current: MutableList<Long> = mutableListOf()
Noric Couderc's avatar
Noric Couderc committed
34

Noric Couderc's avatar
Noric Couderc committed
35
            for (warmup in 0..100) {
Noric Couderc's avatar
Noric Couderc committed
36
37
38
39
40
41
42
43
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
Noric Couderc's avatar
Noric Couderc committed
44
                    if (acc % 2 == 1) {
Noric Couderc's avatar
Noric Couderc committed
45
46
47
48
49
50
51
52
53
54
55
56
57
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
            data.set(kvp.key, current)
        }
        return data
    }

58
59
60
61
    /**
     * Runs a function several times
     * @return A map from PAPI counter names to list of values
     */
62
    inline fun runFunction(counter: String, function: () -> Unit): MutableList<Long> {
63
        val counterId = counterSpec.getCounter(counter)!!
Noric Couderc's avatar
Noric Couderc committed
64
65
66
67
        // We record only one counter
        var evset = EventSet.create()
        try {
            evset = EventSet.create(counterId)
Noric Couderc's avatar
Noric Couderc committed
68
        } catch (e: PapiException) {
Noric Couderc's avatar
Noric Couderc committed
69
70
            error("Failed to sample counter: ${counter}")
        }
71

Noric Couderc's avatar
Noric Couderc committed
72
73
74
75
76
77
78
79
80
81
82
        // We run the function n times
        var values = mutableListOf<Long>()
        for (run in 0..numRuns) {
            // We do the measurements
            evset.start()
            val result = function()
            evset.stop()

            // We record the data
            val data = evset.counters
            values.addAll(data.toList())
83
        }
Noric Couderc's avatar
Noric Couderc committed
84
        return values
85
86
    }

87
88
89
    data class RunSpec(val numRuns: Int,
                       val counter : String,
                       val eventSet: EventSet,
90
                       val syntheticBenchmark: BCBenchmarkPackage<*>)
91
92
93
94

    /**
     * Creates a list of specifications of what counter to get from what benchmarks
     */
95
    fun createRunSpecs(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<RunSpec> {
96
97
98
99
100
101
102
103
        val counters = counterSpec.currentSpec

        // Let's say we want to minimize the counter change
        // We put it in the outer loop
        val specification = mutableListOf<RunSpec>()
        for (c in counters) {
            val eventSet = EventSet.create(c.value)
            for (b in syntheticBenchmarks) {
104
                specification.add(RunSpec(numRuns, c.key, eventSet, b))
105
106
107
108
109
110
            }
        }
        return specification.toList()
    }

    fun runSpec(spec : RunSpec): List<Long> {
111
        println("Running benchmark: " + spec.syntheticBenchmark.benchmarkIdentifier)
112
113
        val writer = FileWriter("/dev/null")
        val samples = mutableListOf<Long>()
114
        for (i in 0 until spec.numRuns) {
115
116
117
            val app = spec.syntheticBenchmark
            val evset = spec.eventSet
            // We do the measurements
118
            app.reset(app.getDatastructureSize())
119
120
            var accumulator = 0
            evset.start()
Noric Couderc's avatar
Noric Couderc committed
121
            app.runBenchmark()
122
            evset.stop()
123
            val resultDataStructure = app.getDatastructure()
124
125
126
127
128
129
130
131
132
133

            // We reset the benchmark
            app.reset(0)

            // Write the result somewhere to prevent dead code elimination
            writer.write(accumulator)
            writer.write(resultDataStructure.toString())
            // We record the data
            samples.addAll(evset.counters.toList())
        }
Noric Couderc's avatar
Noric Couderc committed
134
        writer.close()
135
136
        return samples
    }
137

Noric Couderc's avatar
Noric Couderc committed
138
139
140
    /**
     * A function for running one benchmark, gathering all the hardware counters
     * Present in the counter specification we have.
141
     */
142
    fun runApplication(app: BCBenchmarkPackage<*>) : Map<String, List<Long>> {
Noric Couderc's avatar
Noric Couderc committed
143
144
145
146
147
148
149
        // We will write the aggregators to nowhere
        val writer = FileWriter("/dev/null")
        // We get integers for all the counters.
        val counters = counterSpec.currentSpec

        var result = mutableMapOf<String, List<Long>>()
        for (counter in counters.keys) {
150
151
            val counterId = counterSpec.getCounter(counter)
            val evset = EventSet.create(counterId!!)
Noric Couderc's avatar
Noric Couderc committed
152
153
154
            val samples = mutableListOf<Long>()
            for (run in 0 until numRuns) {
                // We do the measurements
155
                app.reset(app.getDatastructureSize())
Noric Couderc's avatar
Noric Couderc committed
156
157
                var accumulator = 0
                evset.start()
158
		app.runBenchmark()
Noric Couderc's avatar
Noric Couderc committed
159
                evset.stop()
160
		System.out.println("missed trace methods count = " + app.getMissedMethods())
Christoph Reichenbach's avatar
Christoph Reichenbach committed
161
		if (app.getMissedMethods() > 0) {
162
		    System.out.println("[XWARN:SIMMISS]: missed " + Integer.toString(app.getMissedMethods()) + " out of " + Integer.toString(app.getTrace().size) + " trace methods")
Christoph Reichenbach's avatar
Christoph Reichenbach committed
163
		}
164
                val resultDataStructure = app.getDatastructure()
165

Noric Couderc's avatar
Noric Couderc committed
166
167
                // We reset the benchmark
                app.reset(0)
168

Noric Couderc's avatar
Noric Couderc committed
169
170
171
172
173
                // Write the result somewhere to prevent dead code elimination
                writer.write(accumulator)
                writer.write(resultDataStructure.toString())
                // We record the data
                samples.addAll(evset.counters.toList())
174
            }
Noric Couderc's avatar
Noric Couderc committed
175
176
            result[counter] = ArrayList(samples)
            samples.clear()
Noric Couderc's avatar
Noric Couderc committed
177
        }
178
	app.clearAll();
179
180
181

        writer.close()

Noric Couderc's avatar
Noric Couderc committed
182
        return result.toMap()
Noric Couderc's avatar
Noric Couderc committed
183
184
    }

Noric Couderc's avatar
Noric Couderc committed
185
186
187
188
189
190
191
    /**
     * Runs a function several times, gather the performance counters
     * and returns their median
     * @param numRuns Number of times the function should be ran
     * @param function The function to benchmark
     * @return A map from PAPI counter names to the median of their values over numRuns
     */
192
193
    fun runFunctionMedian(counter: String, function: () -> Unit): Double {
        val data = runFunction(counter, function)
Noric Couderc's avatar
Noric Couderc committed
194
        return medianLong(data)
Noric Couderc's avatar
Noric Couderc committed
195
196
    }

197
198


199
    fun runApplications(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<Triple<BCBenchmarkPackage<*>, String, List<Long>>> {
200
        // return syntheticBenchmarks.map { b -> runApplication(iterations, b) }
201
        val specs = createRunSpecs(syntheticBenchmarks)
202
        val spectToCounters = specs.map {
203
204
205
            val samples = runSpec(it)
            Pair(it, samples)
        }
206
207
208
209
        return spectToCounters.map {
            Triple(it.first.syntheticBenchmark,
                    it.first.counter,
                    it.second)
210
        }
211

212
213
    }

214
    fun runApplicationsMedian(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<Triple<BCBenchmarkPackage<*>, String, Double>> {
215
216
217
        val samples = runApplications(syntheticBenchmarks)
        return samples.map {
            Triple(it.first, it.second, medianLong(it.third))
218
219
220
        }
    }

221
    fun runApplicationsNormalized(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<Triple<BCBenchmarkPackage<*>, String, Double>> {
222
223
224
225
226
227
        val countersMedians = runApplicationsMedian(syntheticBenchmarks)
        val instructionsForBenchmark = countersMedians.filter { it.second == "PAPI_TOT_INS" }
                .map { Pair(it.first, it.third) }
                .toMap()
        return countersMedians.map {
            Triple(it.first, it.second, it.third / instructionsForBenchmark[it.first]!!)
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
        }
    }

    /**
     * Compute the median of samples for each counter
     * @param counters: A map from PAPI counter names to lists of samples
     */
    fun medianFeatures(counters : Map<String, List<Long>>) : Map<String, Double> {
        return counters.mapValues {
            medianLong(it.value)
        }
    }

    /**
     * Normalize a map of counters by the PAPI_TOT_INS (Total number of instructions)
     * @param counters: A map from PAPI counter names to their median value
     */
    fun normalizeFeatures(counters : Map<String, Double>) : Map<String, Double> {
        val total = counters["PAPI_TOT_INS"] ?: error("'PAPI_TOT_INS' not present in counters")

        return counters.mapValues {
            it.value / total
        }
251
    }
Noric Couderc's avatar
Noric Couderc committed
252
}