PapiRunner.kt 9.66 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
3
4
package se.lth.cs

import papi.EventSet
import papi.Papi
5
import papi.PapiException
6
import java.io.FileWriter
7
8
import java.time.Duration
import java.time.Instant
Noric Couderc's avatar
Noric Couderc committed
9

Noric Couderc's avatar
Noric Couderc committed
10
open class PapiRunner(counters: CounterSpecification) {
11
12
13
    init {
        Papi.init()
    }
14
15
16

    val counterSpec = counters

Noric Couderc's avatar
Noric Couderc committed
17
18
19
20
21
22
23
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
    fun emptyBenchmark(): MutableMap<String, List<Long>> {
        // For each counter,
        // we store the values for each run (10 runs)
Noric Couderc's avatar
Noric Couderc committed
24
        var data: MutableMap<String, List<Long>> = mutableMapOf()
Noric Couderc's avatar
Noric Couderc committed
25

26
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
27
28
            val evset = EventSet.create(kvp.value)

Noric Couderc's avatar
Noric Couderc committed
29
            val current: MutableList<Long> = mutableListOf()
Noric Couderc's avatar
Noric Couderc committed
30

Noric Couderc's avatar
Noric Couderc committed
31
            for (warmup in 0..100) {
Noric Couderc's avatar
Noric Couderc committed
32
33
34
35
36
37
38
39
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
Noric Couderc's avatar
Noric Couderc committed
40
                    if (acc % 2 == 1) {
Noric Couderc's avatar
Noric Couderc committed
41
42
43
44
45
46
47
48
49
50
51
52
53
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
            data.set(kvp.key, current)
        }
        return data
    }

54
55
56
57
    /**
     * Runs a function several times
     * @return A map from PAPI counter names to list of values
     */
Noric Couderc's avatar
Noric Couderc committed
58
    inline fun runFunction(numRuns: Int, counter: String, function: () -> Unit): MutableList<Long> {
59
        val counterId = counterSpec.getCounter(counter)!!
Noric Couderc's avatar
Noric Couderc committed
60
61
62
63
        // We record only one counter
        var evset = EventSet.create()
        try {
            evset = EventSet.create(counterId)
Noric Couderc's avatar
Noric Couderc committed
64
        } catch (e: PapiException) {
Noric Couderc's avatar
Noric Couderc committed
65
66
            error("Failed to sample counter: ${counter}")
        }
67

Noric Couderc's avatar
Noric Couderc committed
68
69
70
71
72
73
74
75
76
77
78
        // We run the function n times
        var values = mutableListOf<Long>()
        for (run in 0..numRuns) {
            // We do the measurements
            evset.start()
            val result = function()
            evset.stop()

            // We record the data
            val data = evset.counters
            values.addAll(data.toList())
79
        }
Noric Couderc's avatar
Noric Couderc committed
80
        return values
81
82
    }

83
84
85
86

    /** Runs a set of programs (functions) without interleaving
     * (Performance should get better if there is JIT compilation)
     * @Returns A map from couples (counter, program-name) -> values over all runs
Noric Couderc's avatar
Noric Couderc committed
87
     * TODO: Implement this using cartesianProduct from Guava instead
88
     */
89
    open fun runApplications(numRuns: Int, applications: Sequence<SyntheticBenchmark<*>>):
90
            List<MutableMap<String, List<Long>>> {
Noric Couderc's avatar
Noric Couderc committed
91
        // We store a map from program names to map with counters and list of values
92
93
        var list = mutableListOf<MutableMap<String, List<Long>>>()

Noric Couderc's avatar
Noric Couderc committed
94
        // We initialize data with empty maps
95
        for (app in applications) {
96
            list.add(mutableMapOf())
Noric Couderc's avatar
Noric Couderc committed
97
        }
98
99

        // For each counter that is available
100
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
101
102
103
            val counterName = kvp.key
            println("Streamlined mode: '$counterName'")

104
105
            val before = Instant.now()

106
107
108
            // We record only one counter
            val evset = EventSet.create(kvp.value)
            // For each program...
109
110
            var appIndex = 0
            for (app in applications) {
111
112
113
114
                if (list.get(appIndex).containsKey(counterName)) {
                    throw Exception(String.format("'%s' key already exists", counterName))
                }
                list.get(appIndex)[counterName] = runApplication(numRuns, evset, app)
115
                appIndex++
116
            }
117
118
119
120

            val after = Instant.now()
            val totalDuration = Duration.between(before, after)
            println("Done: ${totalDuration}")
121
122
        }

123
        return list.toList()
124
125
    }

126
    fun runApplications(numRuns: Int, syntheticBenchmarks : List<SyntheticBenchmark<*>>):
127
            List<MutableMap<String, List<Long>>> {
128
		return runApplications(numRuns, syntheticBenchmarks.asSequence())
129
130
    }

131
    private fun runApplication(numRuns: Int, evset: EventSet, app: SyntheticBenchmark<*>) : MutableList<Long> {
Noric Couderc's avatar
Noric Couderc committed
132
        // We run it n times
133
134
135

        val writer = FileWriter("/dev/null")

Noric Couderc's avatar
Noric Couderc committed
136
137
138
        var values = mutableListOf<Long>()
        for (run in 0 until numRuns) {
            // We do the measurements
139
            app.reset(app.baseDataStructureSize)
140
            var accumulator = 0
Noric Couderc's avatar
Noric Couderc committed
141
            evset.start()
142
            while (app.hasNext()) {
143
144
                val result : Int? = app.invokeCurrentMethod() as? Int
                accumulator += result ?: 1
145
                app.tick()
146
            }
147
            val resultDataStructure = app.getDataStructure()
Noric Couderc's avatar
Noric Couderc committed
148
            evset.stop()
149
            app.reset(0)
Noric Couderc's avatar
Noric Couderc committed
150

151
            writer.write(accumulator)
152
            writer.write(resultDataStructure.toString())
Noric Couderc's avatar
Noric Couderc committed
153
154
155
            // We record the data
            values.addAll(evset.counters.toList())
        }
156
		app.clearDataStructure()
157
158
159

        writer.close()

Noric Couderc's avatar
Noric Couderc committed
160
161
162
        return values
    }

Noric Couderc's avatar
Noric Couderc committed
163
164
165
166
167
168
169
    /**
     * Runs a function several times, gather the performance counters
     * and returns their median
     * @param numRuns Number of times the function should be ran
     * @param function The function to benchmark
     * @return A map from PAPI counter names to the median of their values over numRuns
     */
Noric Couderc's avatar
Noric Couderc committed
170
    fun runFunctionMedian(numRuns: Int, counter: String, function: () -> Unit): Double {
Noric Couderc's avatar
Noric Couderc committed
171
172
        val data = runFunction(numRuns, counter, function)
        return medianLong(data)
Noric Couderc's avatar
Noric Couderc committed
173
174
175
176
177
178
    }

    /**
     * A class for feature vectors with the label of the app (seed), the fastest datastructure for that app,
     * and the performance counters for that app
     */
Noric Couderc's avatar
Noric Couderc committed
179
180
181
182
    data class FeatureVector(val appLabel: String,
                             val dataStructure: String,
                             val bestDataStructure: String,
                             val counters: Map<String, Double>)
Noric Couderc's avatar
Noric Couderc committed
183
184
185
186

    /**
     * Runs a couple of generated applications and returns their feature vectors
     */
187
    fun getFeatures(numRuns: Int, syntheticBenchmarks: List<SyntheticBenchmark<*>>):
Noric Couderc's avatar
Noric Couderc committed
188
            List<FeatureVector> {
Noric Couderc's avatar
Noric Couderc committed
189

190
        val trainingSet =
191
                SyntheticBenchmarkRunner().runBenchmarks(syntheticBenchmarks).toList()
192

Noric Couderc's avatar
Noric Couderc committed
193
        val distribution = trainingSet.groupBy { it.bestDataStructure }.mapValues { it.value.size }
194
195
        println("Benchmark distribution : $distribution")

196
        val apps = trainingSet.map { it.syntheticBenchmark }
197
198
199
        val counters =
                runApplications(numRuns, apps.asSequence()).map {
                    it.mapValues { medianLong(it.value) }
Noric Couderc's avatar
Noric Couderc committed
200
                }
201

202
        return trainingSet.zip(counters) { v, c ->
203
            FeatureVector(
204
                    v.syntheticBenchmark.identifier,
205
206
207
                    v.dataStructure,
                    v.bestDataStructure,
                    c
Noric Couderc's avatar
Noric Couderc committed
208
209
            )
        }
210
211
    }

Noric Couderc's avatar
Noric Couderc committed
212
213
214
215
216
    /**
     * Saves a list of feature vectors to a CSV file
     * @param vectors The list of feature vectors to save
     * @return the text of the file to be saved
     */
Noric Couderc's avatar
Noric Couderc committed
217
    fun featuresToCSV(vectors: List<FeatureVector>): String {
218
219
220
221
        if (vectors.isEmpty()) return ""

        var header = mutableListOf(
                "application",
Noric Couderc's avatar
Noric Couderc committed
222
223
                "data_structure",
                "best_data_structure")
224
225

        val counters = vectors.map { it.counters.keys }
Noric Couderc's avatar
Noric Couderc committed
226
                .fold(setOf()) { s: Set<String>, v -> s.union(v) }
227
228
229
230
231
232
233
234
235

        header.addAll(counters)
        val headerText = header.joinToString(",")

        var values = mutableListOf<List<String>>()
        for (v in vectors) {
            var l = mutableListOf<String>()
            l.add(v.appLabel)
            l.add(v.dataStructure)
Noric Couderc's avatar
Noric Couderc committed
236
            l.add(v.bestDataStructure)
237
238
239
240
241
242
243
244
245
246
247
248
249
            for (c in counters) {
                l.add(v.counters[c]?.toString() ?: "None")
            }
            values.add(l)
        }

        val valuesTexts = values.map {
            it.joinToString(",")
        }

        val valuesText = valuesTexts.joinToString("\n")
        return "$headerText\n$valuesText"
    }
250

251

Noric Couderc's avatar
Noric Couderc committed
252
253
254
255
256
257
258
259

    /**
     * Transforms a sequence of JMH records to a list of feature vectors,
     * by re-generating the application
     * @param numRuns number of times each app is deemed to be ran
     * @param jmhData the sequence of JMH records
     * @return return a list of feature vectors
     */
260
    fun processJMHData(numRuns: Int, jmhData: Sequence<JMHProcessor.JMHRecord>): List<FeatureVector> {
261
        val applications = jmhData.map {
Noric Couderc's avatar
Noric Couderc committed
262
            it.generateSyntheticBenchmark()!!
263
        }
264

265
        val appsBest = jmhData.map {
Noric Couderc's avatar
Noric Couderc committed
266
            val app = it.generateSyntheticBenchmark()!!
267
268
269
            val best = it.best
            Pair(app, best)
        }
270
271

        // Map from application to list of counters
272
        val results = runApplications(numRuns, applications)
273
274
275
276
277
278
279
280
281
                .asSequence()

        return appsBest.zip(results) { appAndBest, counters ->
            val aggregates = counters.mapValues { medianLong(it.value) }
            FeatureVector(appAndBest.first.seedString,
                    appAndBest.first.dataStructureName,
                    appAndBest.second,
                    aggregates)
        }.toList()
282
    }
Noric Couderc's avatar
Noric Couderc committed
283
284
285
286
287
288
289
290
291
292
293

    /**
     * Transforms a list of JMH records to a list of feature vectors,
     * by re-generating the application
     * @param numRuns number of times each app is deemed to be ran
     * @param jmhData the sequence of JMH records
     * @return return a list of feature vectors
     */
	fun processJMHData(numRuns: Int, jmhData: List<JMHProcessor.JMHRecord>): List<FeatureVector> {
		return processJMHData(numRuns, jmhData.asSequence())
    }}