PapiRunner.kt 7.17 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
3
4
package se.lth.cs

import papi.EventSet
import papi.Papi
5
import papi.PapiException
6
import java.io.FileWriter
7
8
import java.time.Duration
import java.time.Instant
Noric Couderc's avatar
Noric Couderc committed
9

Noric Couderc's avatar
Noric Couderc committed
10
open class PapiRunner(counters: CounterSpecification) {
11
12
13
    init {
        Papi.init()
    }
14
15
16

    val counterSpec = counters

Noric Couderc's avatar
Noric Couderc committed
17
18
19
20
21
22
23
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
    fun emptyBenchmark(): MutableMap<String, List<Long>> {
        // For each counter,
        // we store the values for each run (10 runs)
Noric Couderc's avatar
Noric Couderc committed
24
        var data: MutableMap<String, List<Long>> = mutableMapOf()
Noric Couderc's avatar
Noric Couderc committed
25

26
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
27
28
            val evset = EventSet.create(kvp.value)

Noric Couderc's avatar
Noric Couderc committed
29
            val current: MutableList<Long> = mutableListOf()
Noric Couderc's avatar
Noric Couderc committed
30

Noric Couderc's avatar
Noric Couderc committed
31
            for (warmup in 0..100) {
Noric Couderc's avatar
Noric Couderc committed
32
33
34
35
36
37
38
39
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
Noric Couderc's avatar
Noric Couderc committed
40
                    if (acc % 2 == 1) {
Noric Couderc's avatar
Noric Couderc committed
41
42
43
44
45
46
47
48
49
50
51
52
53
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
            data.set(kvp.key, current)
        }
        return data
    }

54
55
56
57
    /**
     * Runs a function several times
     * @return A map from PAPI counter names to list of values
     */
Noric Couderc's avatar
Noric Couderc committed
58
    inline fun runFunction(numRuns: Int, counter: String, function: () -> Unit): MutableList<Long> {
59
        val counterId = counterSpec.getCounter(counter)!!
Noric Couderc's avatar
Noric Couderc committed
60
61
62
63
        // We record only one counter
        var evset = EventSet.create()
        try {
            evset = EventSet.create(counterId)
Noric Couderc's avatar
Noric Couderc committed
64
        } catch (e: PapiException) {
Noric Couderc's avatar
Noric Couderc committed
65
66
            error("Failed to sample counter: ${counter}")
        }
67

Noric Couderc's avatar
Noric Couderc committed
68
69
70
71
72
73
74
75
76
77
78
        // We run the function n times
        var values = mutableListOf<Long>()
        for (run in 0..numRuns) {
            // We do the measurements
            evset.start()
            val result = function()
            evset.stop()

            // We record the data
            val data = evset.counters
            values.addAll(data.toList())
79
        }
Noric Couderc's avatar
Noric Couderc committed
80
        return values
81
82
    }

83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
    data class RunSpec(val numRuns: Int,
                       val counter : String,
                       val eventSet: EventSet,
                       val syntheticBenchmark: SyntheticBenchmark<*>)

    /**
     * Creates a list of specifications of what counter to get from what benchmarks
     */
    fun createRunSpecs(numRuns : Int, syntheticBenchmarks: MutableList<SyntheticBenchmark<*>>): List<RunSpec> {
        val counters = counterSpec.currentSpec

        // Let's say we want to minimize the counter change
        // We put it in the outer loop
        val specification = mutableListOf<RunSpec>()
        for (c in counters) {
            val eventSet = EventSet.create(c.value)
            for (b in syntheticBenchmarks) {
                for (i in 0 until numRuns) {
                    specification.add(RunSpec(i, c.key, eventSet, b))
                }
            }
        }
        return specification.toList()
    }

    fun runSpec(spec : RunSpec): List<Long> {
        val writer = FileWriter("/dev/null")

        val samples = mutableListOf<Long>()
        for (i in 0 until spec.numRuns) {
            val app = spec.syntheticBenchmark
            val evset = spec.eventSet
            // We do the measurements
            app.reset(app.baseDataStructureSize)
            var accumulator = 0
            evset.start()
            while (app.hasNext()) {
                val result: Int? = app.invokeCurrentMethod() as? Int
                accumulator += result ?: 1
                app.tick()
            }
            val resultDataStructure = app.getDataStructure()
            evset.stop()

            // We reset the benchmark
            app.reset(0)

            // Write the result somewhere to prevent dead code elimination
            writer.write(accumulator)
            writer.write(resultDataStructure.toString())
            // We record the data
            samples.addAll(evset.counters.toList())
        }
        return samples
    }
138

Noric Couderc's avatar
Noric Couderc committed
139
140
141
    /**
     * A function for running one benchmark, gathering all the hardware counters
     * Present in the counter specification we have.
142
     */
Noric Couderc's avatar
Noric Couderc committed
143
144
145
146
147
148
149
150
    fun runApplication(numRuns: Int, app: SyntheticBenchmark<*>) : Map<String, List<Long>> {
        // We will write the aggregators to nowhere
        val writer = FileWriter("/dev/null")
        // We get integers for all the counters.
        val counters = counterSpec.currentSpec

        var result = mutableMapOf<String, List<Long>>()
        for (counter in counters.keys) {
151
152
            val counterId = counterSpec.getCounter(counter)
            val evset = EventSet.create(counterId!!)
Noric Couderc's avatar
Noric Couderc committed
153
154
155
156
157
158
159
160
161
162
            val samples = mutableListOf<Long>()
            for (run in 0 until numRuns) {
                // We do the measurements
                app.reset(app.baseDataStructureSize)
                var accumulator = 0
                evset.start()
                while (app.hasNext()) {
                    val result: Int? = app.invokeCurrentMethod() as? Int
                    accumulator += result ?: 1
                    app.tick()
163
                }
Noric Couderc's avatar
Noric Couderc committed
164
165
                val resultDataStructure = app.getDataStructure()
                evset.stop()
166

Noric Couderc's avatar
Noric Couderc committed
167
168
                // We reset the benchmark
                app.reset(0)
169

Noric Couderc's avatar
Noric Couderc committed
170
171
172
173
174
                // Write the result somewhere to prevent dead code elimination
                writer.write(accumulator)
                writer.write(resultDataStructure.toString())
                // We record the data
                samples.addAll(evset.counters.toList())
175
            }
Noric Couderc's avatar
Noric Couderc committed
176
177
            result[counter] = ArrayList(samples)
            samples.clear()
Noric Couderc's avatar
Noric Couderc committed
178
        }
179
		app.clearDataStructure()
180
181
182

        writer.close()

Noric Couderc's avatar
Noric Couderc committed
183
        return result.toMap()
Noric Couderc's avatar
Noric Couderc committed
184
185
    }

Noric Couderc's avatar
Noric Couderc committed
186
187
188
189
190
191
192
    /**
     * Runs a function several times, gather the performance counters
     * and returns their median
     * @param numRuns Number of times the function should be ran
     * @param function The function to benchmark
     * @return A map from PAPI counter names to the median of their values over numRuns
     */
Noric Couderc's avatar
Noric Couderc committed
193
    fun runFunctionMedian(numRuns: Int, counter: String, function: () -> Unit): Double {
Noric Couderc's avatar
Noric Couderc committed
194
195
        val data = runFunction(numRuns, counter, function)
        return medianLong(data)
Noric Couderc's avatar
Noric Couderc committed
196
197
    }

Noric Couderc's avatar
Noric Couderc committed
198
    fun runApplications(iterations: Int, syntheticBenchmarks: MutableList<SyntheticBenchmark<*>>): List<Map<String, List<Long>>> {
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
        // return syntheticBenchmarks.map { b -> runApplication(iterations, b) }
        val specs = createRunSpecs(iterations, syntheticBenchmarks)
        val map = specs.map {
            val samples = runSpec(it)
            Pair(it, samples)
        }
        // TODO: Return a Map<Pair<CounterString, Benchmark>, List<Long>> instead of this mess.
        val map1 = map.groupBy { s -> s.first.syntheticBenchmark }
        return map1.mapValues {
            it.value.map {
                Pair(it.first.counter, it.second)
            }.toMap()
        }.map {
            it.value
        }
214
    }
Noric Couderc's avatar
Noric Couderc committed
215
}