PapiRunner.kt 12.7 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
3
4
5
6
7
package se.lth.cs

import com.google.gson.Gson
import papi.Constants
import papi.EventSet
import papi.Papi
import java.io.File
8
import java.util.*
Noric Couderc's avatar
Noric Couderc committed
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

val counterSpec =
            hashMapOf(
                    "PAPI_L1_DCM" to Constants.PAPI_L1_DCM,
                    "PAPI_L1_ICM" to Constants.PAPI_L1_ICM,
                    "PAPI_L2_DCM" to Constants.PAPI_L2_DCM,
                    "PAPI_L2_ICM" to Constants.PAPI_L2_ICM,
                    "PAPI_L1_TCM" to Constants.PAPI_L1_TCM,
                    "PAPI_L2_TCM" to Constants.PAPI_L2_TCM,
                    "PAPI_L3_TCM" to Constants.PAPI_L3_TCM,
                    "PAPI_CA_SNP" to Constants.PAPI_CA_SNP,
                    "PAPI_CA_SHR" to Constants.PAPI_CA_SHR,
                    "PAPI_CA_CLN" to Constants.PAPI_CA_CLN,
                    "PAPI_CA_ITV" to Constants.PAPI_CA_ITV,
                    "PAPI_L3_LDM" to Constants.PAPI_L3_LDM,
                    "PAPI_TLB_DM" to Constants.PAPI_TLB_DM,
                    "PAPI_TLB_IM" to Constants.PAPI_TLB_IM,
                    "PAPI_L1_LDM" to Constants.PAPI_L1_LDM,
                    "PAPI_L1_STM" to Constants.PAPI_L1_STM,
                    "PAPI_L2_LDM" to Constants.PAPI_L2_LDM,
                    "PAPI_L2_STM" to Constants.PAPI_L2_STM,
                    "PAPI_PRF_DM" to Constants.PAPI_PRF_DM,
                    "PAPI_MEM_WCY" to Constants.PAPI_MEM_WCY,
                    "PAPI_STL_ICY" to Constants.PAPI_STL_ICY,
                    "PAPI_FUL_ICY" to Constants.PAPI_FUL_ICY,
                    "PAPI_STL_CCY" to Constants.PAPI_STL_CCY,
                    "PAPI_FUL_CCY" to Constants.PAPI_FUL_CCY,
                    "PAPI_BR_UCN" to Constants.PAPI_BR_UCN,
                    "PAPI_BR_CN" to Constants.PAPI_BR_CN,
                    "PAPI_BR_TKN" to Constants.PAPI_BR_TKN,
                    "PAPI_BR_NTK" to Constants.PAPI_BR_NTK,
                    "PAPI_BR_MSP" to Constants.PAPI_BR_MSP,
                    "PAPI_BR_PRC" to Constants.PAPI_BR_PRC,
                    "PAPI_TOT_INS" to Constants.PAPI_TOT_INS,
                    "PAPI_LD_INS" to Constants.PAPI_LD_INS,
                    "PAPI_SR_INS" to Constants.PAPI_SR_INS,
                    "PAPI_BR_INS" to Constants.PAPI_BR_INS,
                    "PAPI_RES_STL" to Constants.PAPI_RES_STL,
                    "PAPI_TOT_CYC" to Constants.PAPI_TOT_CYC,
                    "PAPI_LST_INS" to Constants.PAPI_LST_INS,
                    "PAPI_L2_DCA" to Constants.PAPI_L2_DCA,
                    "PAPI_L3_DCA" to Constants.PAPI_L3_DCA,
                    "PAPI_L2_DCR" to Constants.PAPI_L2_DCR,
                    "PAPI_L3_DCR" to Constants.PAPI_L3_DCR,
                    "PAPI_L2_DCW" to Constants.PAPI_L2_DCW,
                    "PAPI_L3_DCW" to Constants.PAPI_L3_DCW,
                    "PAPI_L2_ICH" to Constants.PAPI_L2_ICH,
                    "PAPI_L2_ICA" to Constants.PAPI_L2_ICA,
                    "PAPI_L3_ICA" to Constants.PAPI_L3_ICA,
                    "PAPI_L2_ICR" to Constants.PAPI_L2_ICR,
                    "PAPI_L3_ICR" to Constants.PAPI_L3_ICR,
                    "PAPI_L2_TCA" to Constants.PAPI_L2_TCA,
                    "PAPI_L3_TCA" to Constants.PAPI_L3_TCA,
                    "PAPI_L2_TCR" to Constants.PAPI_L2_TCR,
                    "PAPI_L3_TCR" to Constants.PAPI_L3_TCR,
                    "PAPI_L2_TCW" to Constants.PAPI_L2_TCW,
                    "PAPI_L3_TCW" to Constants.PAPI_L3_TCW,
                    "PAPI_SP_OPS" to Constants.PAPI_SP_OPS,
                    "PAPI_DP_OPS" to Constants.PAPI_DP_OPS,
                    "PAPI_VEC_SP" to Constants.PAPI_VEC_SP,
                    "PAPI_VEC_DP" to Constants.PAPI_VEC_DP,
                    "PAPI_REF_CYC" to Constants.PAPI_REF_CYC
            )

    val counters = counterSpec.values.toIntArray()

class PapiRunner() {
76
77
78
    init {
        Papi.init()
    }
Noric Couderc's avatar
Noric Couderc committed
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
    fun emptyBenchmark(): MutableMap<String, List<Long>> {
        // For each counter,
        // we store the values for each run (10 runs)
        var data : MutableMap<String, List<Long>> = mutableMapOf()

        for (kvp in counterSpec) {
            val evset = EventSet.create(kvp.value)

            val current : MutableList<Long> = mutableListOf()

            for(warmup in 0 .. 100) {
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
                    if(acc % 2 == 1) {
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
            data.set(kvp.key, current)
        }
        return data
    }

    fun benchmark() {
        // Throws exception
        val evset = EventSet.create(*counters)

        val results = IntArray(10)

        // 9 warmup runs before measuring
        for (warmup in 10 downTo 0) {
            for (i in 0..9) {
                evset.start()

                // some weird code to measure
                for (k in 0..i * 10) {
                    results[i] += k * k
                }
                // done with the code

                evset.stop()
                val data = evset.counters

                // only print the 10th run
                if (warmup == 0) {
                    println("#" + i + ":\t" + data[0] + "\t" + data[1])
                }
            }
        }
    }

    /** Runs a set of programs (functions) without interleaving
     * (Performance should get better if there is JIT compilation)
146
     * Known bug: Crashes if there are too many functions (creates even sets too many times)
Noric Couderc's avatar
Noric Couderc committed
147
     * @Returns A map from couples counter_program-name -> List<Long> over all runs
Noric Couderc's avatar
Noric Couderc committed
148
     */
149
    fun runWithoutInterleaving(numRuns : Int, functions : List<Pair<String,() -> Any>>):
Noric Couderc's avatar
Noric Couderc committed
150
            MutableMap<String, List<Long>> {
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
        var data : MutableMap<String, List<Long>> = mutableMapOf()
        for (labelAndFunction in functions) {
            val function = labelAndFunction.second
            val label = labelAndFunction.first
            val values = runFunction(numRuns, function)
            for (counterAndValues in values) {
                data.put(counterAndValues.key + "_" + label,
                        counterAndValues.value)
            }
        }
        return data
    }

    /**
     * Runs a function several times
     * @return A map from PAPI counter names to list of values
     */
168
    inline fun runFunction(numRuns : Int, function : () -> Any): SortedMap<String, List<Long>> {
Noric Couderc's avatar
Noric Couderc committed
169
170
171
172
173
        var data : MutableMap<String, List<Long>> = mutableMapOf()

        for (kvp in counterSpec) {
            // We record only one counter
            val evset = EventSet.create(kvp.value)
174
175
176
177
178
179
180
181
182
183
184
            // We run the function n times
            var values = mutableListOf<Long>()
            for (run in 0..numRuns) {
                // We do the measurements
                evset.start()
                val result = function()
                evset.stop()

                // We record the data
                val data = evset.counters
                values.addAll(data.toList())
Noric Couderc's avatar
Noric Couderc committed
185
            }
186
            data[kvp.key] = values
Noric Couderc's avatar
Noric Couderc committed
187
        }
188
        return data.toSortedMap()
Noric Couderc's avatar
Noric Couderc committed
189
190
    }

Noric Couderc's avatar
Noric Couderc committed
191
192
193
194
195
196
197
    /**
     * Runs a function several times, gather the performance counters
     * and returns their median
     * @param numRuns Number of times the function should be ran
     * @param function The function to benchmark
     * @return A map from PAPI counter names to the median of their values over numRuns
     */
198
    fun runFunctionMedian(numRuns : Int, function : () -> Any) : SortedMap<String, Float> {
Noric Couderc's avatar
Noric Couderc committed
199
200
201
        val data = runFunction(numRuns, function)
        return data.mapValues {
            median(it.value.map { it.toFloat() })
202
        }.toSortedMap()
Noric Couderc's avatar
Noric Couderc committed
203
204
    }

205
206
    /**
     * Runs a list of generated applications without interleaving
Noric Couderc's avatar
Noric Couderc committed
207
     * @Returns A map from couple counter_program-name -> values over all runs
208
209
210
211
212
213
214
215
216
     */
    fun runListApplications(numRuns: Int, applications : List<Application<*>>): Map<String, List<Long>> {
        val apps = applications.map {
            Pair("App${it.seed}:${it.dataStructure.javaClass.canonicalName}", { it.benchmark() })
        }

        return runWithoutInterleaving(numRuns, apps).toMap()
    }

Noric Couderc's avatar
Noric Couderc committed
217
218
219
220
    /**
     * A class for feature vectors with the label of the app (seed), the fastest datastructure for that app,
     * and the performance counters for that app
     */
221
    data class FeatureVector(val appLabel : String, val dataStructure : String, val counters : SortedMap<String, Float>)
Noric Couderc's avatar
Noric Couderc committed
222
223
224
225

    /**
     * Runs a couple of generated applications and returns their feature vectors
     */
226
    fun getFeatures(numRuns: Int, applications : List<Application<*>>):
Noric Couderc's avatar
Noric Couderc committed
227
            List<FeatureVector> {
228
229

        val trainingSet = ApplicationRunner().runBenchmarks(applications)
230

Noric Couderc's avatar
Noric Couderc committed
231
232
233
234
235
236
237
        return trainingSet.map {
            FeatureVector(
                    it.application.seed.toString(),
                    it.dataStructure,
                    runFunctionMedian(numRuns) { it.application.benchmark() }
            )
        }
238
239
    }

240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
    fun featuresToCSV(vectors : List<FeatureVector>) : String {
        if (vectors.isEmpty()) return ""

        var header = mutableListOf(
                "application",
                "data_structure")

        val counters = vectors.map { it.counters.keys }
                .fold(setOf()) { s : Set<String>, v -> s.union(v)}

        header.addAll(counters)
        val headerText = header.joinToString(",")

        var values = mutableListOf<List<String>>()
        for (v in vectors) {
            var l = mutableListOf<String>()
            l.add(v.appLabel)
            l.add(v.dataStructure)
            for (c in counters) {
                l.add(v.counters[c]?.toString() ?: "None")
            }
            values.add(l)
        }

        val valuesTexts = values.map {
            it.joinToString(",")
        }

        val valuesText = valuesTexts.joinToString("\n")
        return "$headerText\n$valuesText"
    }

272
    data class BenchmarkId(val counter : String, val program : String)
Noric Couderc's avatar
Noric Couderc committed
273

274
    inline fun runWithInterleaving(numRuns : Int, functions : List<Pair<String, () -> Any>>):
275
276
277
278
279
280
            Map<String, List<Long>> {
        var data : MutableMap<BenchmarkId, MutableList<Long>> = mutableMapOf()

        for (kvp in counterSpec) {
            val evset = EventSet.create(kvp.value)

281
            println("Interleaved mode: " + "'" + kvp.key + "'")
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
            // For each run-number
            for (run in 0..numRuns) {
                // We run each program
                for (function in functions) {
                    val current = BenchmarkId(kvp.key, function.first)
                    if (!data.containsKey(current)) {
                        data[current] = mutableListOf()
                    }
                    // We do the measurements
                    evset.start()
                    val result = function.second()
                    evset.stop()

                    //println(result)
                    // We record the data
                    val counterdata = evset.counters
                    data[current]?.addAll(counterdata.toList())
                }
            }
        }

        return data.mapKeys { current ->
            current.key.counter + "_" + current.key.program }
                .mapValues { l -> l.value.toList() }
    }
Noric Couderc's avatar
Noric Couderc committed
307
308
309
310
}

fun main(args : Array<String>) {
    val r = PapiRunner()
311
    val gson = Gson()
312
    val apps = ApplicationGenerator.createMapApplications(4, 1000)
313
314
315
316
317
318
319
320
321
322
323
324
325
326
    val data = r.runListApplications(1000, apps)
    val splitted = data.toList().groupBy {
        it.first.split(":")[1] // Name of data structure
    }.mapValues { it.value.toMap() }
    for (kvp in splitted) {
        val data = kvp.value.mapKeys {
            it.key.split(":")[0] // We only save before the separator
        }
        // val suffix = kvp.value.hashCode()
        val file = File("benchmarkoutput-${kvp.key}.json")
        file.writeText(gson.toJson(data))
    }

    /*
Noric Couderc's avatar
Noric Couderc committed
327
328
329
330
331
    val functions = listOf(
            Pair("1", { test1() }),
            Pair("2", { test2() }),
            Pair("3", { test3() }))

332
    val data = r.runWithoutInterleaving(1000, functions)
333

334
335
    val suffix = data.hashCode()
    val file = File("benchmarkoutput-warmup-$suffix.json")
Noric Couderc's avatar
Noric Couderc committed
336
    file.writeText(gson.toJson(data))
337
338

    val data1 = r.runWithInterleaving(1000, functions)
339
340
    val suffix1 = data1.hashCode()
    val file1 = File("benchmarkoutput-interleaved-warmup-$suffix1.json")
341
    file1.writeText(gson.toJson(data1))
342
    */
Noric Couderc's avatar
Noric Couderc committed
343
}