PapiRunner.kt 12.6 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
3
4
5
6
7
package se.lth.cs

import com.google.gson.Gson
import papi.Constants
import papi.EventSet
import papi.Papi
import java.io.File
8
import java.util.*
Noric Couderc's avatar
Noric Couderc committed
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

val counterSpec =
            hashMapOf(
                    "PAPI_L1_DCM" to Constants.PAPI_L1_DCM,
                    "PAPI_L1_ICM" to Constants.PAPI_L1_ICM,
                    "PAPI_L2_DCM" to Constants.PAPI_L2_DCM,
                    "PAPI_L2_ICM" to Constants.PAPI_L2_ICM,
                    "PAPI_L1_TCM" to Constants.PAPI_L1_TCM,
                    "PAPI_L2_TCM" to Constants.PAPI_L2_TCM,
                    "PAPI_L3_TCM" to Constants.PAPI_L3_TCM,
                    "PAPI_CA_SNP" to Constants.PAPI_CA_SNP,
                    "PAPI_CA_SHR" to Constants.PAPI_CA_SHR,
                    "PAPI_CA_CLN" to Constants.PAPI_CA_CLN,
                    "PAPI_CA_ITV" to Constants.PAPI_CA_ITV,
                    "PAPI_L3_LDM" to Constants.PAPI_L3_LDM,
                    "PAPI_TLB_DM" to Constants.PAPI_TLB_DM,
                    "PAPI_TLB_IM" to Constants.PAPI_TLB_IM,
                    "PAPI_L1_LDM" to Constants.PAPI_L1_LDM,
                    "PAPI_L1_STM" to Constants.PAPI_L1_STM,
                    "PAPI_L2_LDM" to Constants.PAPI_L2_LDM,
                    "PAPI_L2_STM" to Constants.PAPI_L2_STM,
                    "PAPI_PRF_DM" to Constants.PAPI_PRF_DM,
                    "PAPI_MEM_WCY" to Constants.PAPI_MEM_WCY,
                    "PAPI_STL_ICY" to Constants.PAPI_STL_ICY,
                    "PAPI_FUL_ICY" to Constants.PAPI_FUL_ICY,
                    "PAPI_STL_CCY" to Constants.PAPI_STL_CCY,
                    "PAPI_FUL_CCY" to Constants.PAPI_FUL_CCY,
                    "PAPI_BR_UCN" to Constants.PAPI_BR_UCN,
                    "PAPI_BR_CN" to Constants.PAPI_BR_CN,
                    "PAPI_BR_TKN" to Constants.PAPI_BR_TKN,
                    "PAPI_BR_NTK" to Constants.PAPI_BR_NTK,
                    "PAPI_BR_MSP" to Constants.PAPI_BR_MSP,
                    "PAPI_BR_PRC" to Constants.PAPI_BR_PRC,
                    "PAPI_TOT_INS" to Constants.PAPI_TOT_INS,
                    "PAPI_LD_INS" to Constants.PAPI_LD_INS,
                    "PAPI_SR_INS" to Constants.PAPI_SR_INS,
                    "PAPI_BR_INS" to Constants.PAPI_BR_INS,
                    "PAPI_RES_STL" to Constants.PAPI_RES_STL,
                    "PAPI_TOT_CYC" to Constants.PAPI_TOT_CYC,
                    "PAPI_LST_INS" to Constants.PAPI_LST_INS,
                    "PAPI_L2_DCA" to Constants.PAPI_L2_DCA,
                    "PAPI_L3_DCA" to Constants.PAPI_L3_DCA,
                    "PAPI_L2_DCR" to Constants.PAPI_L2_DCR,
                    "PAPI_L3_DCR" to Constants.PAPI_L3_DCR,
                    "PAPI_L2_DCW" to Constants.PAPI_L2_DCW,
                    "PAPI_L3_DCW" to Constants.PAPI_L3_DCW,
                    "PAPI_L2_ICH" to Constants.PAPI_L2_ICH,
                    "PAPI_L2_ICA" to Constants.PAPI_L2_ICA,
                    "PAPI_L3_ICA" to Constants.PAPI_L3_ICA,
                    "PAPI_L2_ICR" to Constants.PAPI_L2_ICR,
                    "PAPI_L3_ICR" to Constants.PAPI_L3_ICR,
                    "PAPI_L2_TCA" to Constants.PAPI_L2_TCA,
                    "PAPI_L3_TCA" to Constants.PAPI_L3_TCA,
                    "PAPI_L2_TCR" to Constants.PAPI_L2_TCR,
                    "PAPI_L3_TCR" to Constants.PAPI_L3_TCR,
                    "PAPI_L2_TCW" to Constants.PAPI_L2_TCW,
                    "PAPI_L3_TCW" to Constants.PAPI_L3_TCW,
                    "PAPI_SP_OPS" to Constants.PAPI_SP_OPS,
                    "PAPI_DP_OPS" to Constants.PAPI_DP_OPS,
                    "PAPI_VEC_SP" to Constants.PAPI_VEC_SP,
                    "PAPI_VEC_DP" to Constants.PAPI_VEC_DP,
                    "PAPI_REF_CYC" to Constants.PAPI_REF_CYC
            )

    val counters = counterSpec.values.toIntArray()

class PapiRunner() {
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
    fun emptyBenchmark(): MutableMap<String, List<Long>> {
        Papi.init()

        // For each counter,
        // we store the values for each run (10 runs)
        var data : MutableMap<String, List<Long>> = mutableMapOf()

        for (kvp in counterSpec) {
            val evset = EventSet.create(kvp.value)

            val current : MutableList<Long> = mutableListOf()

            for(warmup in 0 .. 100) {
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
                    if(acc % 2 == 1) {
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
            data.set(kvp.key, current)
        }
        return data
    }

    fun benchmark() {
        Papi.init()

        // Throws exception
        val evset = EventSet.create(*counters)

        val results = IntArray(10)

        // 9 warmup runs before measuring
        for (warmup in 10 downTo 0) {
            for (i in 0..9) {
                evset.start()

                // some weird code to measure
                for (k in 0..i * 10) {
                    results[i] += k * k
                }
                // done with the code

                evset.stop()
                val data = evset.counters

                // only print the 10th run
                if (warmup == 0) {
                    println("#" + i + ":\t" + data[0] + "\t" + data[1])
                }
            }
        }
    }

    /** Runs a set of programs (functions) without interleaving
     * (Performance should get better if there is JIT compilation)
Noric Couderc's avatar
Noric Couderc committed
147
     * @Returns A map from couples counter_program-name -> List<Long> over all runs
Noric Couderc's avatar
Noric Couderc committed
148
     */
149
    fun runWithoutInterleaving(numRuns : Int, functions : List<Pair<String,() -> Any>>):
Noric Couderc's avatar
Noric Couderc committed
150
151
152
            MutableMap<String, List<Long>> {
        Papi.init()

153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
        var data : MutableMap<String, List<Long>> = mutableMapOf()
        for (labelAndFunction in functions) {
            val function = labelAndFunction.second
            val label = labelAndFunction.first
            val values = runFunction(numRuns, function)
            for (counterAndValues in values) {
                data.put(counterAndValues.key + "_" + label,
                        counterAndValues.value)
            }
        }
        return data
    }

    /**
     * Runs a function several times
     * @return A map from PAPI counter names to list of values
     */
170
    inline fun runFunction(numRuns : Int, function : () -> Any): SortedMap<String, List<Long>> {
Noric Couderc's avatar
Noric Couderc committed
171
172
173
174
175
        var data : MutableMap<String, List<Long>> = mutableMapOf()

        for (kvp in counterSpec) {
            // We record only one counter
            val evset = EventSet.create(kvp.value)
176
177
178
179
180
181
182
183
184
185
186
            // We run the function n times
            var values = mutableListOf<Long>()
            for (run in 0..numRuns) {
                // We do the measurements
                evset.start()
                val result = function()
                evset.stop()

                // We record the data
                val data = evset.counters
                values.addAll(data.toList())
Noric Couderc's avatar
Noric Couderc committed
187
            }
188
            data[kvp.key] = values
Noric Couderc's avatar
Noric Couderc committed
189
        }
190
        return data.toSortedMap()
Noric Couderc's avatar
Noric Couderc committed
191
192
    }

Noric Couderc's avatar
Noric Couderc committed
193
194
195
196
197
198
199
    /**
     * Runs a function several times, gather the performance counters
     * and returns their median
     * @param numRuns Number of times the function should be ran
     * @param function The function to benchmark
     * @return A map from PAPI counter names to the median of their values over numRuns
     */
200
    fun runFunctionMedian(numRuns : Int, function : () -> Any) : SortedMap<String, Float> {
Noric Couderc's avatar
Noric Couderc committed
201
202
203
        val data = runFunction(numRuns, function)
        return data.mapValues {
            median(it.value.map { it.toFloat() })
204
        }.toSortedMap()
Noric Couderc's avatar
Noric Couderc committed
205
206
    }

207
208
    /**
     * Runs a list of generated applications without interleaving
Noric Couderc's avatar
Noric Couderc committed
209
     * @Returns A map from couple counter_program-name -> values over all runs
210
211
212
213
214
215
216
217
218
     */
    fun runListApplications(numRuns: Int, applications : List<Application<*>>): Map<String, List<Long>> {
        val apps = applications.map {
            Pair("App${it.seed}:${it.dataStructure.javaClass.canonicalName}", { it.benchmark() })
        }

        return runWithoutInterleaving(numRuns, apps).toMap()
    }

Noric Couderc's avatar
Noric Couderc committed
219
220
221
222
    /**
     * A class for feature vectors with the label of the app (seed), the fastest datastructure for that app,
     * and the performance counters for that app
     */
223
    data class FeatureVector(val appLabel : String, val dataStructure : String, val counters : SortedMap<String, Float>)
Noric Couderc's avatar
Noric Couderc committed
224
225
226
227

    /**
     * Runs a couple of generated applications and returns their feature vectors
     */
228
    fun getFeatures(numRuns: Int, applications : List<Application<*>>):
Noric Couderc's avatar
Noric Couderc committed
229
            List<FeatureVector> {
230
231

        val trainingSet = ApplicationRunner().runBenchmarks(applications)
232

Noric Couderc's avatar
Noric Couderc committed
233
234
235
236
237
238
239
        return trainingSet.map {
            FeatureVector(
                    it.application.seed.toString(),
                    it.dataStructure,
                    runFunctionMedian(numRuns) { it.application.benchmark() }
            )
        }
240
241
    }

242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
    fun featuresToCSV(vectors : List<FeatureVector>) : String {
        if (vectors.isEmpty()) return ""

        var header = mutableListOf(
                "application",
                "data_structure")

        val counters = vectors.map { it.counters.keys }
                .fold(setOf()) { s : Set<String>, v -> s.union(v)}

        header.addAll(counters)
        val headerText = header.joinToString(",")

        var values = mutableListOf<List<String>>()
        for (v in vectors) {
            var l = mutableListOf<String>()
            l.add(v.appLabel)
            l.add(v.dataStructure)
            for (c in counters) {
                l.add(v.counters[c]?.toString() ?: "None")
            }
            values.add(l)
        }

        val valuesTexts = values.map {
            it.joinToString(",")
        }

        val valuesText = valuesTexts.joinToString("\n")
        return "$headerText\n$valuesText"
    }

274
    data class BenchmarkId(val counter : String, val program : String)
Noric Couderc's avatar
Noric Couderc committed
275

276
    inline fun runWithInterleaving(numRuns : Int, functions : List<Pair<String, () -> Any>>):
277
278
279
280
281
282
283
284
            Map<String, List<Long>> {
        Papi.init()

        var data : MutableMap<BenchmarkId, MutableList<Long>> = mutableMapOf()

        for (kvp in counterSpec) {
            val evset = EventSet.create(kvp.value)

285
            println("Interleaved mode: " + "'" + kvp.key + "'")
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
            // For each run-number
            for (run in 0..numRuns) {
                // We run each program
                for (function in functions) {
                    val current = BenchmarkId(kvp.key, function.first)
                    if (!data.containsKey(current)) {
                        data[current] = mutableListOf()
                    }
                    // We do the measurements
                    evset.start()
                    val result = function.second()
                    evset.stop()

                    //println(result)
                    // We record the data
                    val counterdata = evset.counters
                    data[current]?.addAll(counterdata.toList())
                }
            }
        }

        return data.mapKeys { current ->
            current.key.counter + "_" + current.key.program }
                .mapValues { l -> l.value.toList() }
    }
Noric Couderc's avatar
Noric Couderc committed
311
312
313
314
}

fun main(args : Array<String>) {
    val r = PapiRunner()
315
    val gson = Gson()
316
    val apps = ApplicationGenerator.createMapApplications(4, 1000)
317
318
319
320
321
322
323
324
325
326
327
328
329
330
    val data = r.runListApplications(1000, apps)
    val splitted = data.toList().groupBy {
        it.first.split(":")[1] // Name of data structure
    }.mapValues { it.value.toMap() }
    for (kvp in splitted) {
        val data = kvp.value.mapKeys {
            it.key.split(":")[0] // We only save before the separator
        }
        // val suffix = kvp.value.hashCode()
        val file = File("benchmarkoutput-${kvp.key}.json")
        file.writeText(gson.toJson(data))
    }

    /*
Noric Couderc's avatar
Noric Couderc committed
331
332
333
334
335
    val functions = listOf(
            Pair("1", { test1() }),
            Pair("2", { test2() }),
            Pair("3", { test3() }))

336
    val data = r.runWithoutInterleaving(1000, functions)
337

338
339
    val suffix = data.hashCode()
    val file = File("benchmarkoutput-warmup-$suffix.json")
Noric Couderc's avatar
Noric Couderc committed
340
    file.writeText(gson.toJson(data))
341
342

    val data1 = r.runWithInterleaving(1000, functions)
343
344
    val suffix1 = data1.hashCode()
    val file1 = File("benchmarkoutput-interleaved-warmup-$suffix1.json")
345
    file1.writeText(gson.toJson(data1))
346
    */
Noric Couderc's avatar
Noric Couderc committed
347
}