PapiRunner.kt 8.77 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
3
4
package se.lth.cs

import papi.EventSet
import papi.Papi
5
import papi.PapiException
Noric Couderc's avatar
Noric Couderc committed
6
import se.lth.cs.ApplicationGeneration.ListApplicationGenerator
Noric Couderc's avatar
Noric Couderc committed
7
import java.io.File
8
9
import java.time.Duration
import java.time.Instant
Noric Couderc's avatar
Noric Couderc committed
10
import kotlin.system.exitProcess
Noric Couderc's avatar
Noric Couderc committed
11

Noric Couderc's avatar
Noric Couderc committed
12
open class PapiRunner(counters: CounterSpecification) {
13
14
15
    init {
        Papi.init()
    }
16
17
18

    val counterSpec = counters

Noric Couderc's avatar
Noric Couderc committed
19
20
21
22
23
24
25
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
    fun emptyBenchmark(): MutableMap<String, List<Long>> {
        // For each counter,
        // we store the values for each run (10 runs)
Noric Couderc's avatar
Noric Couderc committed
26
        var data: MutableMap<String, List<Long>> = mutableMapOf()
Noric Couderc's avatar
Noric Couderc committed
27

28
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
29
30
            val evset = EventSet.create(kvp.value)

Noric Couderc's avatar
Noric Couderc committed
31
            val current: MutableList<Long> = mutableListOf()
Noric Couderc's avatar
Noric Couderc committed
32

Noric Couderc's avatar
Noric Couderc committed
33
            for (warmup in 0..100) {
Noric Couderc's avatar
Noric Couderc committed
34
35
36
37
38
39
40
41
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
Noric Couderc's avatar
Noric Couderc committed
42
                    if (acc % 2 == 1) {
Noric Couderc's avatar
Noric Couderc committed
43
44
45
46
47
48
49
50
51
52
53
54
55
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
            data.set(kvp.key, current)
        }
        return data
    }

56
57
58
59
    /**
     * Runs a function several times
     * @return A map from PAPI counter names to list of values
     */
Noric Couderc's avatar
Noric Couderc committed
60
    inline fun runFunction(numRuns: Int, counter: String, function: () -> Unit): MutableList<Long> {
61
        val counterId = counterSpec.getCounter(counter)!!
Noric Couderc's avatar
Noric Couderc committed
62
63
64
65
        // We record only one counter
        var evset = EventSet.create()
        try {
            evset = EventSet.create(counterId)
Noric Couderc's avatar
Noric Couderc committed
66
        } catch (e: PapiException) {
Noric Couderc's avatar
Noric Couderc committed
67
68
            error("Failed to sample counter: ${counter}")
        }
69

Noric Couderc's avatar
Noric Couderc committed
70
71
72
73
74
75
76
77
78
79
80
        // We run the function n times
        var values = mutableListOf<Long>()
        for (run in 0..numRuns) {
            // We do the measurements
            evset.start()
            val result = function()
            evset.stop()

            // We record the data
            val data = evset.counters
            values.addAll(data.toList())
81
        }
Noric Couderc's avatar
Noric Couderc committed
82
        return values
83
84
    }

85
86
87
88

    /** Runs a set of programs (functions) without interleaving
     * (Performance should get better if there is JIT compilation)
     * @Returns A map from couples (counter, program-name) -> values over all runs
Noric Couderc's avatar
Noric Couderc committed
89
     * TODO: Implement this using cartesianProduct from Guava instead
90
     */
91
92
    open fun runApplications(numRuns: Int, applications: List<Application<*>>):
            Map<Application<*>, MutableMap<String, List<Long>>> {
93

Noric Couderc's avatar
Noric Couderc committed
94
        // We store a map from program names to map with counters and list of values
95
        var data = mutableMapOf<Application<*>, MutableMap<String, List<Long>>>()
Noric Couderc's avatar
Noric Couderc committed
96
        // We initialize data with empty maps
97
98
        for (app in applications) {
            data.put(app, mutableMapOf())
Noric Couderc's avatar
Noric Couderc committed
99
        }
100
101

        // For each counter that is available
102
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
103
104
105
            val counterName = kvp.key
            println("Streamlined mode: '$counterName'")

106
107
            val before = Instant.now()

108
109
110
            // We record only one counter
            val evset = EventSet.create(kvp.value)
            // For each program...
111
            for (app in applications) {
Noric Couderc's avatar
Noric Couderc committed
112
                data[app]!![counterName] = runApplication(numRuns, evset, app)
113
            }
114
115
116
117

            val after = Instant.now()
            val totalDuration = Duration.between(before, after)
            println("Done: ${totalDuration}")
118
119
120
121
122
        }

        return data
    }

Noric Couderc's avatar
Noric Couderc committed
123
124
125
126
127
128
129
130
    private fun runApplication(numRuns: Int, evset: EventSet, app: Application<*>) : MutableList<Long> {
        // We run it n times
        var values = mutableListOf<Long>()
        for (run in 0 until numRuns) {
            // We do the measurements
            evset.start()
            val result = app.benchmark()
            evset.stop()
131
            app.reset(0)
Noric Couderc's avatar
Noric Couderc committed
132
133
134
135
136
137
138
139

            //println(result)
            // We record the data
            values.addAll(evset.counters.toList())
        }
        return values
    }

Noric Couderc's avatar
Noric Couderc committed
140
141
142
143
144
145
146
    /**
     * Runs a function several times, gather the performance counters
     * and returns their median
     * @param numRuns Number of times the function should be ran
     * @param function The function to benchmark
     * @return A map from PAPI counter names to the median of their values over numRuns
     */
Noric Couderc's avatar
Noric Couderc committed
147
    fun runFunctionMedian(numRuns: Int, counter: String, function: () -> Unit): Double {
Noric Couderc's avatar
Noric Couderc committed
148
149
        val data = runFunction(numRuns, counter, function)
        return medianLong(data)
Noric Couderc's avatar
Noric Couderc committed
150
151
152
153
154
155
    }

    /**
     * A class for feature vectors with the label of the app (seed), the fastest datastructure for that app,
     * and the performance counters for that app
     */
Noric Couderc's avatar
Noric Couderc committed
156
157
158
159
    data class FeatureVector(val appLabel: String,
                             val dataStructure: String,
                             val bestDataStructure: String,
                             val counters: Map<String, Double>)
Noric Couderc's avatar
Noric Couderc committed
160
161
162
163

    /**
     * Runs a couple of generated applications and returns their feature vectors
     */
Noric Couderc's avatar
Noric Couderc committed
164
    fun getFeatures(numRuns: Int, applications: List<Application<*>>):
Noric Couderc's avatar
Noric Couderc committed
165
            List<FeatureVector> {
Noric Couderc's avatar
Noric Couderc committed
166

167
168
        val trainingSet =
                ApplicationRunner().runBenchmarks(applications)
169

Noric Couderc's avatar
Noric Couderc committed
170
        val distribution = trainingSet.groupBy { it.bestDataStructure }.mapValues { it.value.size }
171
172
173
        println("Benchmark distribution : $distribution")

        val apps = trainingSet.map { it.application }
Noric Couderc's avatar
Noric Couderc committed
174
        val appsTocounters =
175
                runApplications(numRuns, apps).mapValues {
Noric Couderc's avatar
Noric Couderc committed
176
177
                    it.value.mapValues { medianLong(it.value) }
                }
178

Noric Couderc's avatar
Noric Couderc committed
179
        val featureVectors = trainingSet.map {
180
            FeatureVector(
Noric Couderc's avatar
Noric Couderc committed
181
182
183
                    it.application.identifier,
                    it.dataStructure,
                    it.bestDataStructure,
184
                    appsTocounters.getValue(it.application)
Noric Couderc's avatar
Noric Couderc committed
185
186
            )
        }
Noric Couderc's avatar
Noric Couderc committed
187
        return featureVectors
188
189
    }

Noric Couderc's avatar
Noric Couderc committed
190
    fun featuresToCSV(vectors: List<FeatureVector>): String {
191
192
193
194
        if (vectors.isEmpty()) return ""

        var header = mutableListOf(
                "application",
Noric Couderc's avatar
Noric Couderc committed
195
196
                "data_structure",
                "best_data_structure")
197
198

        val counters = vectors.map { it.counters.keys }
Noric Couderc's avatar
Noric Couderc committed
199
                .fold(setOf()) { s: Set<String>, v -> s.union(v) }
200
201
202
203
204
205
206
207
208

        header.addAll(counters)
        val headerText = header.joinToString(",")

        var values = mutableListOf<List<String>>()
        for (v in vectors) {
            var l = mutableListOf<String>()
            l.add(v.appLabel)
            l.add(v.dataStructure)
Noric Couderc's avatar
Noric Couderc committed
209
            l.add(v.bestDataStructure)
210
211
212
213
214
215
216
217
218
219
220
221
222
            for (c in counters) {
                l.add(v.counters[c]?.toString() ?: "None")
            }
            values.add(l)
        }

        val valuesTexts = values.map {
            it.joinToString(",")
        }

        val valuesText = valuesTexts.joinToString("\n")
        return "$headerText\n$valuesText"
    }
223

224
    fun processJMHData(numRuns: Int, jmhData: List<JMHProcessor.JMHRecord>): List<FeatureVector> {
225
        val applications = jmhData.map { processJMHRecord(it)!! }
226
227
228
229
230
231
        val bests = jmhData.map { it.best }

        val appsBest = applications.zip(bests)
                .toMap()

        // Map from application to list of counters
232
233
234
235
        val results = runApplications(numRuns, applications)
        return results.map {
            val aggregates = it.value.mapValues { medianLong(it.value) }
            val app = it.key
236
237
            val best = appsBest[app]!!
            FeatureVector(app.seedString, app.dataStructureName, best, aggregates)
238
239
240
        }
    }

Noric Couderc's avatar
Noric Couderc committed
241
    private fun getClassFromSimpleName(name : String) : Any {
242
243
244
245
        val className = "java.util.$name"
        return Class.forName(className).getConstructor().newInstance()
    }

Noric Couderc's avatar
Noric Couderc committed
246
    private fun processJMHRecord(record : JMHProcessor.JMHRecord) : Application<*>? {
247
        var application : Application<*>? = null
248
        val dataStructure = getClassFromSimpleName(record.datastructure)
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
        if (record.collection == "List") {
            application = ListApplication(record.seed, record.size, dataStructure as MutableList<Int>?)
            return application
        }

        if (record.collection == "Map") {
            application = MapApplication(record.seed, record.size, dataStructure as MutableMap<Int, Int>?)
            return application
        }

        if (record.collection == "Set") {
            application = SetApplication(record.seed, record.size, dataStructure as MutableSet<Int>?)
            return application
        }
        return null
264
    }
Noric Couderc's avatar
Noric Couderc committed
265
}