PapiRunner.kt 8.11 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
package se.lth.cs

3
import org.openjdk.jmh.infra.Blackhole
Noric Couderc's avatar
Noric Couderc committed
4
5
import papi.EventSet
import papi.Papi
6
import se.lth.cs.bcgen.BCBenchmarkPackage
7
import se.lth.cs.papicounters.PAPICounter
8
import se.lth.cs.papicounters.PapiBenchmarkAnalyzer
9
import se.lth.cs.timing.OperationType
10
import java.io.FileWriter
Noric Couderc's avatar
Noric Couderc committed
11

12
open class PapiRunner(val numRuns: Int, val counters: List<PAPICounter>) : PapiBenchmarkAnalyzer {
13

14
15
16
    init {
        Papi.init()
    }
17

18
19
    protected var blackhole = Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.")

Noric Couderc's avatar
Noric Couderc committed
20
21
22
23
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
24
    fun emptyBenchmark(): Map<PAPICounter, List<Long>> {
Noric Couderc's avatar
Noric Couderc committed
25
26
        // For each counter,
        // we store the values for each run (10 runs)
27
        var data: MutableMap<PAPICounter, List<Long>> = mutableMapOf()
Noric Couderc's avatar
Noric Couderc committed
28

29
30
        for (counter in counters) {
            val evset = EventSet.create(counter.toPAPIConstant()!!)
Noric Couderc's avatar
Noric Couderc committed
31

Noric Couderc's avatar
Noric Couderc committed
32
            val current: MutableList<Long> = mutableListOf()
Noric Couderc's avatar
Noric Couderc committed
33

Noric Couderc's avatar
Noric Couderc committed
34
            for (warmup in 0..100) {
Noric Couderc's avatar
Noric Couderc committed
35
36
37
38
39
40
41
42
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
Noric Couderc's avatar
Noric Couderc committed
43
                    if (acc % 2 == 1) {
Noric Couderc's avatar
Noric Couderc committed
44
45
46
47
48
49
50
51
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
52
            data.set(counter, current)
Noric Couderc's avatar
Noric Couderc committed
53
        }
54
        return data.toMap()
Noric Couderc's avatar
Noric Couderc committed
55
56
    }

57
58
59
    /**
     * Creates a list of specifications of what counter to get from what benchmarks
     */
60
    fun createRunSpecs(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<PapiBenchmarkAnalyzer.RunSpec> {
61
62
        // Let's say we want to minimize the counter change
        // We put it in the outer loop
63
        val specification = mutableListOf<PapiBenchmarkAnalyzer.RunSpec>()
64
        val counterValues = counters.map { it.toPAPIConstant()!! }
65
66
        val eventSet = EventSet.create(*counterValues.toIntArray())
        for (b in syntheticBenchmarks) {
67
            specification.add(PapiBenchmarkAnalyzer.RunSpec(counters, eventSet, b))
68
69
70
71
        }
        return specification.toList()
    }

72
    override fun runSpec(spec : PapiBenchmarkAnalyzer.RunSpec): List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
73
        // println("Running benchmark: " + spec.syntheticBenchmark.benchmarkIdentifier)
74
        val writer = FileWriter("/dev/null")
75
76
        val samples = mutableListOf<PapiBenchmarkAnalyzer.BenchmarkRunData>()

77
        for (i in 0 until numRuns) {
78
79
80
            val app = spec.syntheticBenchmark
            val evset = spec.eventSet
            // We do the measurements
81
            app.reset(app.getDatastructureSize())
82
83
            var accumulator = 0
            evset.start()
84
            app.runBenchmark(blackhole)
85
            evset.stop()
86
            val resultDataStructure = app.getDatastructure()
87
88
89
90
91

            // Write the result somewhere to prevent dead code elimination
            writer.write(accumulator)
            writer.write(resultDataStructure.toString())
            // We record the data
92
            val runSamples = spec.counters().zip(evset.counters.toList()).map {
93
94
                PapiBenchmarkAnalyzer.BenchmarkRunSample(it.first, it.second.toDouble())
            }
95
96
97
98
99
100
            val runData = PapiBenchmarkAnalyzer.BenchmarkRunData(
                spec.syntheticBenchmark,
                i,
                runSamples,
                mapOf()
            )
101
            samples.add(runData)
102
        }
Noric Couderc's avatar
Noric Couderc committed
103
        writer.close()
104
105
        return samples
    }
106

Noric Couderc's avatar
Noric Couderc committed
107
108
109
    /**
     * A function for running one benchmark, gathering all the hardware counters
     * Present in the counter specification we have.
110
     */
Noric Couderc's avatar
Noric Couderc committed
111
    @Deprecated("Used only in tests??")
112
    fun runApplication(app: BCBenchmarkPackage<*>) : Map<PAPICounter, List<Long>> {
Noric Couderc's avatar
Noric Couderc committed
113
114
115
116
        // We will write the aggregators to nowhere
        val writer = FileWriter("/dev/null")
        // We get integers for all the counters.

117
118
119
120
        var result = mutableMapOf<PAPICounter, List<Long>>()
        for (counter in counters) {
            val counterId = counter.toPAPIConstant()!!
            val evset = EventSet.create(counterId)
Noric Couderc's avatar
Noric Couderc committed
121
122
123
            val samples = mutableListOf<Long>()
            for (run in 0 until numRuns) {
                // We do the measurements
124
                app.reset(app.getDatastructureSize())
Noric Couderc's avatar
Noric Couderc committed
125
126
                var accumulator = 0
                evset.start()
127
                app.runBenchmark(blackhole)
Noric Couderc's avatar
Noric Couderc committed
128
                evset.stop()
129
130
131
132
133
134
135
                System.out.println("missed trace methods count = " + app.getMissedMethods())
                if (app.getMissedMethods() > 0) {
                    System.out.println("[XWARN:SIMMISS]: missed "
                            + Integer.toString(app.getMissedMethods())
                            + " out of "
                            + Integer.toString(app.getTrace().size) + " trace methods")
                }
136
                val resultDataStructure = app.getDatastructure()
137

Noric Couderc's avatar
Noric Couderc committed
138
139
                // We reset the benchmark
                app.reset(0)
140

Noric Couderc's avatar
Noric Couderc committed
141
142
143
144
145
                // Write the result somewhere to prevent dead code elimination
                writer.write(accumulator)
                writer.write(resultDataStructure.toString())
                // We record the data
                samples.addAll(evset.counters.toList())
146
            }
Noric Couderc's avatar
Noric Couderc committed
147
148
            result[counter] = ArrayList(samples)
            samples.clear()
Noric Couderc's avatar
Noric Couderc committed
149
        }
Noric Couderc's avatar
Noric Couderc committed
150
        app.clearAll();
151
152
153

        writer.close()

Noric Couderc's avatar
Noric Couderc committed
154
        return result.toMap()
Noric Couderc's avatar
Noric Couderc committed
155
156
    }

Noric Couderc's avatar
Noric Couderc committed
157
    override fun runApplications(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
158
        // return syntheticBenchmarks.map { b -> runApplication(iterations, b) }
159
        val specs = createRunSpecs(syntheticBenchmarks)
160
161
        val numberBenchmarks = specs.size
        var i = 0
162
        val samples = specs.flatMap {
163
164
            print("Running spec: $i / $numberBenchmarks\r")
            i++
Noric Couderc's avatar
Noric Couderc committed
165
            runSpec(it) }
166

167
        println("Finished running specs.")
168

169
        return samples
170
171
    }

Noric Couderc's avatar
Noric Couderc committed
172
    fun runApplicationsNormalized(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
173
        val benchmarkRuns = runApplications(syntheticBenchmarks)
174

175
        return benchmarkRuns.map {
176
            it.normalized()
177
178
179
180
181
182
183
184
185
186
187
188
189
        }
    }

    /**
     * Normalize a map of counters by the PAPI_TOT_INS (Total number of instructions)
     * @param counters: A map from PAPI counter names to their median value
     */
    fun normalizeFeatures(counters : Map<String, Double>) : Map<String, Double> {
        val total = counters["PAPI_TOT_INS"] ?: error("'PAPI_TOT_INS' not present in counters")

        return counters.mapValues {
            it.value / total
        }
190
    }
Noric Couderc's avatar
Noric Couderc committed
191
}
192
193
194
195

/**
 * A mockup class which returns deterministic results when you "run" a spec.
 */
196
class MockupPapiRunner(numRuns : Int, counters : List<PAPICounter>) : PapiRunner(numRuns, counters) {
197
    override fun runSpec(spec : PapiBenchmarkAnalyzer.RunSpec) : List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
198
199
200
201
202
203
204
205
        // We map the counters to integers
        // val counterToInt = mapOf<String, Long>(
        //     Pair("PAPI_CA_ITV", 0),
        //     Pair("PAPI_L1_DCM", 1),
        //     Pair("PAPI_TOT_CYC", 2),
        //     Pair("PAPI_TOT_INS", 3)
        // )
        // val n = seed + counterToInt[spec.counter]!!
206
        val iterations = 0 until numRuns
207
        return iterations.map {
208
            val samples = spec.counters().map {
209
                PapiBenchmarkAnalyzer.BenchmarkRunSample(it, 10000.0)
210
            }
211
212
213
214
215
216
217
218
219

            val cyclesPerOpType = mapOf(
                Pair(OperationType.DELETE, 1000.0),
                Pair(OperationType.INSERT, 1000.0),
                Pair(OperationType.ITERATE, 1000.0),
                Pair(OperationType.SEARCH, 1000.0)
            )

            PapiBenchmarkAnalyzer.BenchmarkRunData(spec.syntheticBenchmark, it, samples, cyclesPerOpType )
220
        }
221
222
    }
}