PapiRunner.kt 6.34 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
package se.lth.cs

3
import org.openjdk.jmh.infra.Blackhole
Noric Couderc's avatar
Noric Couderc committed
4
5
import papi.EventSet
import papi.Papi
6
import se.lth.cs.bcgen.BCBenchmarkPackage
Noric Couderc's avatar
Noric Couderc committed
7
import se.lth.cs.papicounters.EventSetBuilder
8
import se.lth.cs.papicounters.PAPICounter
9
import se.lth.cs.papicounters.PapiBenchmarkAnalyzer
10
import se.lth.cs.timing.OperationType
Noric Couderc's avatar
Noric Couderc committed
11
import se.lth.cs.util.FeatureSet
12
import java.io.FileWriter
Noric Couderc's avatar
Noric Couderc committed
13

14
open class PapiRunner() : PapiBenchmarkAnalyzer {
15

16
17
18
    init {
        Papi.init()
    }
19

20
21
    protected var blackhole = Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.")

Noric Couderc's avatar
Noric Couderc committed
22
23
24
25
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
Noric Couderc's avatar
Noric Couderc committed
26
    fun emptyBenchmark(counters : FeatureSet): Map<PAPICounter, List<Long>> {
Noric Couderc's avatar
Noric Couderc committed
27
28
        // For each counter,
        // we store the values for each run (10 runs)
29
        var data: MutableMap<PAPICounter, List<Long>> = mutableMapOf()
30
        val eb = EventSetBuilder(counters)
Noric Couderc's avatar
Noric Couderc committed
31

Noric Couderc's avatar
Noric Couderc committed
32
33
        for (counter in eb.getCounterSet()) {
            val evset = eb.getEventSet()
Noric Couderc's avatar
Noric Couderc committed
34

Noric Couderc's avatar
Noric Couderc committed
35
            val current: MutableList<Long> = mutableListOf()
Noric Couderc's avatar
Noric Couderc committed
36

Noric Couderc's avatar
Noric Couderc committed
37
            for (warmup in 0..100) {
Noric Couderc's avatar
Noric Couderc committed
38
39
40
41
42
43
44
45
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
Noric Couderc's avatar
Noric Couderc committed
46
                    if (acc % 2 == 1) {
Noric Couderc's avatar
Noric Couderc committed
47
48
49
50
                        b[i] = acc
                    }
                }
                // Get data
Noric Couderc's avatar
Noric Couderc committed
51
                val currentData = evset.stop()
Noric Couderc's avatar
Noric Couderc committed
52
53
                current.addAll(currentData.toList())
            }
54
            data.set(counter, current)
Noric Couderc's avatar
Noric Couderc committed
55
        }
56
        return data.toMap()
Noric Couderc's avatar
Noric Couderc committed
57
58
    }

59
60
61
    /**
     * Creates a list of specifications of what counter to get from what benchmarks
     */
Noric Couderc's avatar
Noric Couderc committed
62
    fun createRunSpecs(numRuns: Int, features : FeatureSet,
63
                       syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<PapiBenchmarkAnalyzer.RunSpec> {
64
65
        // Let's say we want to minimize the counter change
        // We put it in the outer loop
66
        val specification = mutableListOf<PapiBenchmarkAnalyzer.RunSpec>()
67
        for (b in syntheticBenchmarks) {
Noric Couderc's avatar
Noric Couderc committed
68
            specification.add(PapiBenchmarkAnalyzer.RunSpec(numRuns, features, b))
69
70
71
72
        }
        return specification.toList()
    }

73
    override fun runSpec(spec : PapiBenchmarkAnalyzer.RunSpec): List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
74
        // println("Running benchmark: " + spec.syntheticBenchmark.benchmarkIdentifier)
75
        val writer = FileWriter("/dev/null")
76
77
        val samples = mutableListOf<PapiBenchmarkAnalyzer.BenchmarkRunData>()

78
        for (i in 0 until spec.numberRuns) {
79
            val app = spec.syntheticBenchmark
Noric Couderc's avatar
Noric Couderc committed
80
            val evset = spec.eventSet()
81
            // We do the measurements
82
            app.reset(app.getDatastructureSize())
83
84
            var accumulator = 0
            evset.start()
85
            app.runBenchmark(blackhole)
Noric Couderc's avatar
Noric Couderc committed
86
            val readings = evset.stop()
87
            val resultDataStructure = app.getDatastructure()
88
89
90
91
92

            // Write the result somewhere to prevent dead code elimination
            writer.write(accumulator)
            writer.write(resultDataStructure.toString())
            // We record the data
Noric Couderc's avatar
Noric Couderc committed
93
            val runSamples = spec.counters().zip(readings.toList()).map {
94
95
                PapiBenchmarkAnalyzer.BenchmarkRunSample(it.first, it.second.toDouble())
            }
96
97
98
99
100
101
            val runData = PapiBenchmarkAnalyzer.BenchmarkRunData(
                spec.syntheticBenchmark,
                i,
                runSamples,
                mapOf()
            )
102
            samples.add(runData)
103
        }
Noric Couderc's avatar
Noric Couderc committed
104
        writer.close()
105
106
        return samples
    }
107

108
    override fun runApplications(numberRuns: Int,
Noric Couderc's avatar
Noric Couderc committed
109
                                 features : FeatureSet,
110
                                 syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
111
        // return syntheticBenchmarks.map { b -> runApplication(iterations, b) }
Noric Couderc's avatar
Noric Couderc committed
112
        val specs = createRunSpecs(numberRuns, features, syntheticBenchmarks)
113
114
        val numberBenchmarks = specs.size
        var i = 0
115
        val samples = specs.flatMap {
116
117
            print("Running spec: $i / $numberBenchmarks\r")
            i++
Noric Couderc's avatar
Noric Couderc committed
118
            runSpec(it) }
119

120
        println("Finished running specs.")
121

122
        return samples
123
124
    }

Noric Couderc's avatar
Noric Couderc committed
125
    fun runApplicationsNormalized(numberRuns : Int, features : FeatureSet,
126
                                  syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
Noric Couderc's avatar
Noric Couderc committed
127
        val benchmarkRuns = runApplications(numberRuns, features, syntheticBenchmarks)
128

129
        return benchmarkRuns.map {
130
            it.normalized()
131
132
133
134
135
136
137
138
139
140
141
142
143
        }
    }

    /**
     * Normalize a map of counters by the PAPI_TOT_INS (Total number of instructions)
     * @param counters: A map from PAPI counter names to their median value
     */
    fun normalizeFeatures(counters : Map<String, Double>) : Map<String, Double> {
        val total = counters["PAPI_TOT_INS"] ?: error("'PAPI_TOT_INS' not present in counters")

        return counters.mapValues {
            it.value / total
        }
144
    }
Noric Couderc's avatar
Noric Couderc committed
145
}
146
147
148
149

/**
 * A mockup class which returns deterministic results when you "run" a spec.
 */
150
class MockupPapiRunner() : PapiRunner() {
151
    override fun runSpec(spec : PapiBenchmarkAnalyzer.RunSpec) : List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
152
153
154
155
156
157
158
159
        // We map the counters to integers
        // val counterToInt = mapOf<String, Long>(
        //     Pair("PAPI_CA_ITV", 0),
        //     Pair("PAPI_L1_DCM", 1),
        //     Pair("PAPI_TOT_CYC", 2),
        //     Pair("PAPI_TOT_INS", 3)
        // )
        // val n = seed + counterToInt[spec.counter]!!
160
        val iterations = 0 until spec.numberRuns
161
        return iterations.map {
162
            val samples = spec.counters().map {
163
                PapiBenchmarkAnalyzer.BenchmarkRunSample(it, 10000.0)
164
            }
165
166
167
168
169
170
171
172
173

            val cyclesPerOpType = mapOf(
                Pair(OperationType.DELETE, 1000.0),
                Pair(OperationType.INSERT, 1000.0),
                Pair(OperationType.ITERATE, 1000.0),
                Pair(OperationType.SEARCH, 1000.0)
            )

            PapiBenchmarkAnalyzer.BenchmarkRunData(spec.syntheticBenchmark, it, samples, cyclesPerOpType )
174
        }
175
176
    }
}