PapiRunner.kt 15 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
3
4
5
package se.lth.cs

import papi.Constants
import papi.EventSet
import papi.Papi
6
import papi.PapiException
Noric Couderc's avatar
Noric Couderc committed
7
import se.lth.cs.ApplicationGeneration.ListApplicationGenerator
Noric Couderc's avatar
Noric Couderc committed
8
9
10
import java.io.File

val counterSpec =
Noric Couderc's avatar
Noric Couderc committed
11
            sortedMapOf(
Noric Couderc's avatar
Noric Couderc committed
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
                    "PAPI_L1_DCM" to Constants.PAPI_L1_DCM,
                    "PAPI_L1_ICM" to Constants.PAPI_L1_ICM,
                    "PAPI_L2_DCM" to Constants.PAPI_L2_DCM,
                    "PAPI_L2_ICM" to Constants.PAPI_L2_ICM,
                    "PAPI_L1_TCM" to Constants.PAPI_L1_TCM,
                    "PAPI_L2_TCM" to Constants.PAPI_L2_TCM,
                    "PAPI_L3_TCM" to Constants.PAPI_L3_TCM,
                    "PAPI_CA_SNP" to Constants.PAPI_CA_SNP,
                    "PAPI_CA_SHR" to Constants.PAPI_CA_SHR,
                    "PAPI_CA_CLN" to Constants.PAPI_CA_CLN,
                    "PAPI_CA_ITV" to Constants.PAPI_CA_ITV,
                    "PAPI_L3_LDM" to Constants.PAPI_L3_LDM,
                    "PAPI_TLB_DM" to Constants.PAPI_TLB_DM,
                    "PAPI_TLB_IM" to Constants.PAPI_TLB_IM,
                    "PAPI_L1_LDM" to Constants.PAPI_L1_LDM,
                    "PAPI_L1_STM" to Constants.PAPI_L1_STM,
                    "PAPI_L2_LDM" to Constants.PAPI_L2_LDM,
                    "PAPI_L2_STM" to Constants.PAPI_L2_STM,
                    "PAPI_PRF_DM" to Constants.PAPI_PRF_DM,
                    "PAPI_MEM_WCY" to Constants.PAPI_MEM_WCY,
                    "PAPI_STL_ICY" to Constants.PAPI_STL_ICY,
                    "PAPI_FUL_ICY" to Constants.PAPI_FUL_ICY,
                    "PAPI_STL_CCY" to Constants.PAPI_STL_CCY,
                    "PAPI_FUL_CCY" to Constants.PAPI_FUL_CCY,
                    "PAPI_BR_UCN" to Constants.PAPI_BR_UCN,
                    "PAPI_BR_CN" to Constants.PAPI_BR_CN,
                    "PAPI_BR_TKN" to Constants.PAPI_BR_TKN,
                    "PAPI_BR_NTK" to Constants.PAPI_BR_NTK,
                    "PAPI_BR_MSP" to Constants.PAPI_BR_MSP,
                    "PAPI_BR_PRC" to Constants.PAPI_BR_PRC,
                    "PAPI_TOT_INS" to Constants.PAPI_TOT_INS,
                    "PAPI_LD_INS" to Constants.PAPI_LD_INS,
                    "PAPI_SR_INS" to Constants.PAPI_SR_INS,
                    "PAPI_BR_INS" to Constants.PAPI_BR_INS,
                    "PAPI_RES_STL" to Constants.PAPI_RES_STL,
                    "PAPI_TOT_CYC" to Constants.PAPI_TOT_CYC,
                    "PAPI_LST_INS" to Constants.PAPI_LST_INS,
                    "PAPI_L2_DCA" to Constants.PAPI_L2_DCA,
                    "PAPI_L3_DCA" to Constants.PAPI_L3_DCA,
                    "PAPI_L2_DCR" to Constants.PAPI_L2_DCR,
                    "PAPI_L3_DCR" to Constants.PAPI_L3_DCR,
                    "PAPI_L2_DCW" to Constants.PAPI_L2_DCW,
                    "PAPI_L3_DCW" to Constants.PAPI_L3_DCW,
                    "PAPI_L2_ICH" to Constants.PAPI_L2_ICH,
                    "PAPI_L2_ICA" to Constants.PAPI_L2_ICA,
                    "PAPI_L3_ICA" to Constants.PAPI_L3_ICA,
                    "PAPI_L2_ICR" to Constants.PAPI_L2_ICR,
                    "PAPI_L3_ICR" to Constants.PAPI_L3_ICR,
                    "PAPI_L2_TCA" to Constants.PAPI_L2_TCA,
                    "PAPI_L3_TCA" to Constants.PAPI_L3_TCA,
                    "PAPI_L2_TCR" to Constants.PAPI_L2_TCR,
                    "PAPI_L3_TCR" to Constants.PAPI_L3_TCR,
                    "PAPI_L2_TCW" to Constants.PAPI_L2_TCW,
                    "PAPI_L3_TCW" to Constants.PAPI_L3_TCW,
                    "PAPI_SP_OPS" to Constants.PAPI_SP_OPS,
                    "PAPI_DP_OPS" to Constants.PAPI_DP_OPS,
                    "PAPI_VEC_SP" to Constants.PAPI_VEC_SP,
                    "PAPI_VEC_DP" to Constants.PAPI_VEC_DP,
                    "PAPI_REF_CYC" to Constants.PAPI_REF_CYC
            )

    val counters = counterSpec.values.toIntArray()

class PapiRunner() {
76
77
78
    init {
        Papi.init()
    }
Noric Couderc's avatar
Noric Couderc committed
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
    fun emptyBenchmark(): MutableMap<String, List<Long>> {
        // For each counter,
        // we store the values for each run (10 runs)
        var data : MutableMap<String, List<Long>> = mutableMapOf()

        for (kvp in counterSpec) {
            val evset = EventSet.create(kvp.value)

            val current : MutableList<Long> = mutableListOf()

            for(warmup in 0 .. 100) {
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
                    if(acc % 2 == 1) {
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
            data.set(kvp.key, current)
        }
        return data
    }

116
117
118
119
120
121
122
    class CounterAndProgram(val counter : String, val programName : String) : Comparable<CounterAndProgram> {
        override fun toString(): String { return "${counter}_$programName"}
        override fun compareTo(other: CounterAndProgram): Int {
            return toString().compareTo(other.toString())
        }
    }

Noric Couderc's avatar
Noric Couderc committed
123
124
    /** Runs a set of programs (functions) without interleaving
     * (Performance should get better if there is JIT compilation)
125
     * Known bug: Crashes if there are too many functions (creates even sets too many times)
Noric Couderc's avatar
Noric Couderc committed
126
     * @Returns A map from couples counter_program-name -> List<Long> over all runs
Noric Couderc's avatar
Noric Couderc committed
127
     */
Noric Couderc's avatar
Noric Couderc committed
128
    fun runWithoutInterleaving(numRuns : Int, functions : List<Pair<String,() -> Unit>>):
129
130
            MutableMap<CounterAndProgram, List<Long>> {
        var data : MutableMap<CounterAndProgram, List<Long>> = mutableMapOf()
131
132
133
        for (labelAndFunction in functions) {
            val function = labelAndFunction.second
            val label = labelAndFunction.first
Noric Couderc's avatar
Noric Couderc committed
134
135
136
137
            val values : MutableMap<String, List<Long>> = mutableMapOf()
            for (counter in counterSpec.keys) {
                values.put(counter, runFunction(numRuns, counter, function))
            }
138
            for (counterAndValues in values) {
139
                data.put(CounterAndProgram(counterAndValues.key, label),
140
141
142
143
144
145
                        counterAndValues.value)
            }
        }
        return data
    }

146
147
148
149
    /**
     * Runs a function several times
     * @return A map from PAPI counter names to list of values
     */
Noric Couderc's avatar
Noric Couderc committed
150
151
152
153
154
155
156
157
158
    inline fun runFunction(numRuns : Int, counter : String, function : () -> Unit): MutableList<Long> {
        val counterId = counterSpec[counter]!!
        // We record only one counter
        var evset = EventSet.create()
        try {
            evset = EventSet.create(counterId)
        } catch (e : PapiException) {
            error("Failed to sample counter: ${counter}")
        }
159

Noric Couderc's avatar
Noric Couderc committed
160
161
162
163
164
165
166
167
168
169
170
        // We run the function n times
        var values = mutableListOf<Long>()
        for (run in 0..numRuns) {
            // We do the measurements
            evset.start()
            val result = function()
            evset.stop()

            // We record the data
            val data = evset.counters
            values.addAll(data.toList())
171
        }
Noric Couderc's avatar
Noric Couderc committed
172
        return values
173
174
    }

175
176
177
178
179
180

    /** Runs a set of programs (functions) without interleaving
     * (Performance should get better if there is JIT compilation)
     * @Returns A map from couples (counter, program-name) -> values over all runs
     */
    inline fun runWithoutInterleaving2(numRuns : Int, functions : List<Pair<String,() -> Any>>):
Noric Couderc's avatar
Noric Couderc committed
181
            MutableMap<String, MutableMap<String, List<Long>>> {
182

Noric Couderc's avatar
Noric Couderc committed
183
184
185
186
187
188
        // We store a map from program names to map with counters and list of values
        var data = mutableMapOf<String, MutableMap<String, List<Long>>>()
        // We initialize data with empty maps
        for (f in functions) {
            data.put(f.first, mutableMapOf())
        }
189
190
191

        // For each counter that is available
        for (kvp in counterSpec) {
Noric Couderc's avatar
Noric Couderc committed
192
193
194
            val counterName = kvp.key
            println("Streamlined mode: '$counterName'")

195
196
197
198
            // We record only one counter
            val evset = EventSet.create(kvp.value)
            // For each program...
            for (function in functions) {
Noric Couderc's avatar
Noric Couderc committed
199
200
201
                val appName = function.first

                val current = Pair(counterName, appName)
202
203
                // We run it n times
                var values = mutableListOf<Long>()
204
                for (run in 0 until numRuns) {
205
206
207
208
209
210
211
                    // We do the measurements
                    evset.start()
                    val result = function.second()
                    evset.stop()

                    //println(result)
                    // We record the data
Noric Couderc's avatar
Noric Couderc committed
212
                    values.addAll(evset.counters.toList())
213
                }
Noric Couderc's avatar
Noric Couderc committed
214
                data[appName]!![counterName] = values
215
216
217
218
219
220
            }
        }

        return data
    }

Noric Couderc's avatar
Noric Couderc committed
221
222
223
224
225
226
227
    /**
     * Runs a function several times, gather the performance counters
     * and returns their median
     * @param numRuns Number of times the function should be ran
     * @param function The function to benchmark
     * @return A map from PAPI counter names to the median of their values over numRuns
     */
Noric Couderc's avatar
Noric Couderc committed
228
229
230
    fun runFunctionMedian(numRuns : Int, counter : String, function : () -> Unit) : Double {
        val data = runFunction(numRuns, counter, function)
        return medianLong(data)
Noric Couderc's avatar
Noric Couderc committed
231
232
    }

233
234
    /**
     * Runs a list of generated applications without interleaving
Noric Couderc's avatar
Noric Couderc committed
235
     * @Returns A map from couple counter_program-name -> values over all runs
236
     */
237
    fun runListApplications(numRuns: Int, applications: List<Application<*>>):
Noric Couderc's avatar
Noric Couderc committed
238
            MutableMap<String, MutableMap<String, List<Long>>> {
239
        val apps = applications.map {
Noric Couderc's avatar
Noric Couderc committed
240
            Pair(it.identifier, { it.benchmark() })
241
242
        }

Noric Couderc's avatar
Noric Couderc committed
243
        return runWithoutInterleaving2(numRuns, apps)
244
245
    }

Noric Couderc's avatar
Noric Couderc committed
246
247
248
249
    /**
     * A class for feature vectors with the label of the app (seed), the fastest datastructure for that app,
     * and the performance counters for that app
     */
Noric Couderc's avatar
Noric Couderc committed
250
251
252
253
    data class FeatureVector(val appLabel : String,
                             val dataStructure : String,
                             val bestDataStructure : String,
                             val counters : Map<String, Double>)
Noric Couderc's avatar
Noric Couderc committed
254
255
256
257

    /**
     * Runs a couple of generated applications and returns their feature vectors
     */
258
    fun getFeatures(numRuns: Int, applications : List<Application<*>>):
Noric Couderc's avatar
Noric Couderc committed
259
            List<FeatureVector> {
Noric Couderc's avatar
Noric Couderc committed
260

261
262
        val trainingSet =
                ApplicationRunner().runBenchmarks(applications)
263

Noric Couderc's avatar
Noric Couderc committed
264
        val distribution = trainingSet.groupBy { it.bestDataStructure }.mapValues { it.value.size }
265
266
267
        println("Benchmark distribution : $distribution")

        val apps = trainingSet.map { it.application }
Noric Couderc's avatar
Noric Couderc committed
268
269
270
271
        val appsTocounters =
                runListApplications(numRuns, apps).mapValues {
                    it.value.mapValues { medianLong(it.value) }
                }
272

Noric Couderc's avatar
Noric Couderc committed
273
        val featureVectors = trainingSet.map {
274
            FeatureVector(
Noric Couderc's avatar
Noric Couderc committed
275
276
277
278
                    it.application.identifier,
                    it.dataStructure,
                    it.bestDataStructure,
                    appsTocounters[it.application.identifier]!!
Noric Couderc's avatar
Noric Couderc committed
279
280
            )
        }
Noric Couderc's avatar
Noric Couderc committed
281
        return featureVectors
282
283
    }

284
285
286
287
288
    fun featuresToCSV(vectors : List<FeatureVector>) : String {
        if (vectors.isEmpty()) return ""

        var header = mutableListOf(
                "application",
Noric Couderc's avatar
Noric Couderc committed
289
290
                "data_structure",
                "best_data_structure")
291
292
293
294
295
296
297
298
299
300
301
302

        val counters = vectors.map { it.counters.keys }
                .fold(setOf()) { s : Set<String>, v -> s.union(v)}

        header.addAll(counters)
        val headerText = header.joinToString(",")

        var values = mutableListOf<List<String>>()
        for (v in vectors) {
            var l = mutableListOf<String>()
            l.add(v.appLabel)
            l.add(v.dataStructure)
Noric Couderc's avatar
Noric Couderc committed
303
            l.add(v.bestDataStructure)
304
305
306
307
308
309
310
311
312
313
314
315
316
317
            for (c in counters) {
                l.add(v.counters[c]?.toString() ?: "None")
            }
            values.add(l)
        }

        val valuesTexts = values.map {
            it.joinToString(",")
        }

        val valuesText = valuesTexts.joinToString("\n")
        return "$headerText\n$valuesText"
    }

318
    data class BenchmarkId(val counter : String, val program : String)
Noric Couderc's avatar
Noric Couderc committed
319

320
    inline fun runWithInterleaving(numRuns : Int, functions : List<Pair<String, () -> Any>>):
321
322
323
324
325
326
            Map<String, List<Long>> {
        var data : MutableMap<BenchmarkId, MutableList<Long>> = mutableMapOf()

        for (kvp in counterSpec) {
            val evset = EventSet.create(kvp.value)

327
            println("Interleaved mode: " + "'" + kvp.key + "'")
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
            // For each run-number
            for (run in 0..numRuns) {
                // We run each program
                for (function in functions) {
                    val current = BenchmarkId(kvp.key, function.first)
                    if (!data.containsKey(current)) {
                        data[current] = mutableListOf()
                    }
                    // We do the measurements
                    evset.start()
                    val result = function.second()
                    evset.stop()

                    //println(result)
                    // We record the data
                    val counterdata = evset.counters
                    data[current]?.addAll(counterdata.toList())
                }
            }
        }

        return data.mapKeys { current ->
            current.key.counter + "_" + current.key.program }
                .mapValues { l -> l.value.toList() }
    }
Noric Couderc's avatar
Noric Couderc committed
353
354
355
356
}

fun main(args : Array<String>) {
    val r = PapiRunner()
Noric Couderc's avatar
Noric Couderc committed
357
358
359
    val apps = // ListApplicationGenerator().createApplications(0, 100, 100)
        ApplicationRunner().createListApplicationsSpread(20, 100, ListApplicationGenerator())
    val data = r.getFeatures(20, apps.map { it.application })
360
    val file = File("benchmarkoutput.csv")
361
    file.writeText(r.featuresToCSV(data))
362

363
    /*
364
    val gson = Gson()
365
366
367
    val data1 = r.runListApplications(100, apps)
    val splitted = data1.toList().groupBy {
        it.first.programName.split(":")[1] // Name of data structure
368
    }.mapValues { it.value.toMap() }
369

370
371
    for (kvp in splitted) {
        val data = kvp.value.mapKeys {
372
            it.key.toString().split(":")[0] // We only save before the separator
373
374
375
376
377
        }
        // val suffix = kvp.value.hashCode()
        val file = File("benchmarkoutput-${kvp.key}.json")
        file.writeText(gson.toJson(data))
    }
378
    */
379

380
    /* GENERATING A BENCHMARK
Noric Couderc's avatar
Noric Couderc committed
381
382
383
384
385
    val functions = listOf(
            Pair("1", { test1() }),
            Pair("2", { test2() }),
            Pair("3", { test3() }))

386
    val data = r.runWithoutInterleaving(1000, functions)
387

388
389
    val suffix = data.hashCode()
    val file = File("benchmarkoutput-warmup-$suffix.json")
Noric Couderc's avatar
Noric Couderc committed
390
    file.writeText(gson.toJson(data))
391
392

    val data1 = r.runWithInterleaving(1000, functions)
393
394
    val suffix1 = data1.hashCode()
    val file1 = File("benchmarkoutput-interleaved-warmup-$suffix1.json")
395
    file1.writeText(gson.toJson(data1))
396
    */
Noric Couderc's avatar
Noric Couderc committed
397
}