PapiRunner.kt 8.17 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
package se.lth.cs

3
import org.openjdk.jmh.infra.Blackhole
Noric Couderc's avatar
Noric Couderc committed
4
5
import papi.EventSet
import papi.Papi
6
import papi.PapiException
7
import se.lth.cs.bcgen.BCBenchmarkPackage
8
import java.io.FileWriter
Noric Couderc's avatar
Noric Couderc committed
9

10
open class PapiRunner(private val numRuns: Int, counters: CounterSpecification) {
11

12
13
14
    init {
        Papi.init()
    }
15
16
17

    val counterSpec = counters

18
19
20
21
    data class BenchmarkRunData(val benchmark : BCBenchmarkPackage<*>,
                                val counter : String,
                                val value : Double)

22
23
    protected var blackhole = Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.")

Noric Couderc's avatar
Noric Couderc committed
24
25
26
27
28
29
30
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
    fun emptyBenchmark(): MutableMap<String, List<Long>> {
        // For each counter,
        // we store the values for each run (10 runs)
Noric Couderc's avatar
Noric Couderc committed
31
        var data: MutableMap<String, List<Long>> = mutableMapOf()
Noric Couderc's avatar
Noric Couderc committed
32

33
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
34
35
            val evset = EventSet.create(kvp.value)

Noric Couderc's avatar
Noric Couderc committed
36
            val current: MutableList<Long> = mutableListOf()
Noric Couderc's avatar
Noric Couderc committed
37

Noric Couderc's avatar
Noric Couderc committed
38
            for (warmup in 0..100) {
Noric Couderc's avatar
Noric Couderc committed
39
40
41
42
43
44
45
46
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
Noric Couderc's avatar
Noric Couderc committed
47
                    if (acc % 2 == 1) {
Noric Couderc's avatar
Noric Couderc committed
48
49
50
51
52
53
54
55
56
57
58
59
60
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
            data.set(kvp.key, current)
        }
        return data
    }

61
62
63
    data class RunSpec(val numRuns: Int,
                       val counter : String,
                       val eventSet: EventSet,
64
                       val syntheticBenchmark: BCBenchmarkPackage<*>)
65
66
67
68

    /**
     * Creates a list of specifications of what counter to get from what benchmarks
     */
69
    fun createRunSpecs(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<RunSpec> {
70
71
72
73
74
75
76
77
        val counters = counterSpec.currentSpec

        // Let's say we want to minimize the counter change
        // We put it in the outer loop
        val specification = mutableListOf<RunSpec>()
        for (c in counters) {
            val eventSet = EventSet.create(c.value)
            for (b in syntheticBenchmarks) {
78
                specification.add(RunSpec(numRuns, c.key, eventSet, b))
79
80
81
82
83
            }
        }
        return specification.toList()
    }

84
    open fun runSpec(spec : RunSpec): List<Long> {
85
        // println("Running benchmark: " + spec.syntheticBenchmark.benchmarkIdentifier)
86
87
        val writer = FileWriter("/dev/null")
        val samples = mutableListOf<Long>()
88
        for (i in 0 until spec.numRuns) {
89
90
91
            val app = spec.syntheticBenchmark
            val evset = spec.eventSet
            // We do the measurements
92
            app.reset(app.getDatastructureSize())
93
94
            var accumulator = 0
            evset.start()
95
            app.runBenchmark(blackhole)
96
            evset.stop()
97
            val resultDataStructure = app.getDatastructure()
98
99
100
101
102
103
104
105
106
107

            // We reset the benchmark
            app.reset(0)

            // Write the result somewhere to prevent dead code elimination
            writer.write(accumulator)
            writer.write(resultDataStructure.toString())
            // We record the data
            samples.addAll(evset.counters.toList())
        }
Noric Couderc's avatar
Noric Couderc committed
108
        writer.close()
109
110
        return samples
    }
111

Noric Couderc's avatar
Noric Couderc committed
112
113
114
    /**
     * A function for running one benchmark, gathering all the hardware counters
     * Present in the counter specification we have.
115
     */
Noric Couderc's avatar
Noric Couderc committed
116
    @Deprecated("Used only in tests??")
117
    fun runApplication(app: BCBenchmarkPackage<*>) : Map<String, List<Long>> {
Noric Couderc's avatar
Noric Couderc committed
118
119
120
121
122
123
124
        // We will write the aggregators to nowhere
        val writer = FileWriter("/dev/null")
        // We get integers for all the counters.
        val counters = counterSpec.currentSpec

        var result = mutableMapOf<String, List<Long>>()
        for (counter in counters.keys) {
125
126
            val counterId = counterSpec.getCounter(counter)
            val evset = EventSet.create(counterId!!)
Noric Couderc's avatar
Noric Couderc committed
127
128
129
            val samples = mutableListOf<Long>()
            for (run in 0 until numRuns) {
                // We do the measurements
130
                app.reset(app.getDatastructureSize())
Noric Couderc's avatar
Noric Couderc committed
131
132
                var accumulator = 0
                evset.start()
133
                app.runBenchmark(blackhole)
Noric Couderc's avatar
Noric Couderc committed
134
                evset.stop()
135
136
137
138
139
140
141
                System.out.println("missed trace methods count = " + app.getMissedMethods())
                if (app.getMissedMethods() > 0) {
                    System.out.println("[XWARN:SIMMISS]: missed "
                            + Integer.toString(app.getMissedMethods())
                            + " out of "
                            + Integer.toString(app.getTrace().size) + " trace methods")
                }
142
                val resultDataStructure = app.getDatastructure()
143

Noric Couderc's avatar
Noric Couderc committed
144
145
                // We reset the benchmark
                app.reset(0)
146

Noric Couderc's avatar
Noric Couderc committed
147
148
149
150
151
                // Write the result somewhere to prevent dead code elimination
                writer.write(accumulator)
                writer.write(resultDataStructure.toString())
                // We record the data
                samples.addAll(evset.counters.toList())
152
            }
Noric Couderc's avatar
Noric Couderc committed
153
154
            result[counter] = ArrayList(samples)
            samples.clear()
Noric Couderc's avatar
Noric Couderc committed
155
        }
Noric Couderc's avatar
Noric Couderc committed
156
        app.clearAll();
157
158
159

        writer.close()

Noric Couderc's avatar
Noric Couderc committed
160
        return result.toMap()
Noric Couderc's avatar
Noric Couderc committed
161
162
    }

163

164
    fun runApplications(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<Triple<BCBenchmarkPackage<*>, String, List<Long>>> {
165
        // return syntheticBenchmarks.map { b -> runApplication(iterations, b) }
166
        val specs = createRunSpecs(syntheticBenchmarks)
167
168
        val numberBenchmarks = specs.size
        var i = 0
169
        val spectToCounters = specs.map {
170
171
            print("Running spec: $i / $numberBenchmarks\r")
            i++
172
173
174
            val samples = runSpec(it)
            Pair(it, samples)
        }
175
        println("Finished running specs.")
176
177
178
179
        return spectToCounters.map {
            Triple(it.first.syntheticBenchmark,
                    it.first.counter,
                    it.second)
180
        }
181

182
183
    }

184
    fun runApplicationsMedian(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<BenchmarkRunData> {
185
186
        val samples = runApplications(syntheticBenchmarks)
        return samples.map {
187
            BenchmarkRunData(it.first, it.second, medianLong(it.third))
188
189
190
        }
    }

191
    fun runApplicationsNormalized(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<BenchmarkRunData> {
192
        val countersMedians = runApplicationsMedian(syntheticBenchmarks)
193
194
195
        val instructionsForBenchmark = countersMedians.filter { it.counter == "PAPI_TOT_INS" }
                .associate { Pair(it.benchmark, it.value) }

196
        return countersMedians.map {
197
            BenchmarkRunData(it.benchmark, it.counter, it.value / instructionsForBenchmark[it.benchmark]!!)
198
199
200
201
202
203
204
205
206
207
208
209
210
        }
    }

    /**
     * Normalize a map of counters by the PAPI_TOT_INS (Total number of instructions)
     * @param counters: A map from PAPI counter names to their median value
     */
    fun normalizeFeatures(counters : Map<String, Double>) : Map<String, Double> {
        val total = counters["PAPI_TOT_INS"] ?: error("'PAPI_TOT_INS' not present in counters")

        return counters.mapValues {
            it.value / total
        }
211
    }
Noric Couderc's avatar
Noric Couderc committed
212
}
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231

/**
 * A mockup class which returns deterministic results when you "run" a spec.
 */
class MockupPapiRunner(numRuns : Int, counters : CounterSpecification) : PapiRunner(numRuns, counters) {
    override fun runSpec(spec : RunSpec) : List<Long> {
        val seed = spec.syntheticBenchmark.seed
        // We map the counters to integers
        // val counterToInt = mapOf<String, Long>(
        //     Pair("PAPI_CA_ITV", 0),
        //     Pair("PAPI_L1_DCM", 1),
        //     Pair("PAPI_TOT_CYC", 2),
        //     Pair("PAPI_TOT_INS", 3)
        // )
        // val n = seed + counterToInt[spec.counter]!!
        val n = seed
        return listOf(n, n, n)
    }
}