PapiRunner.kt 8.87 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
3
4
package se.lth.cs

import papi.EventSet
import papi.Papi
5
import papi.PapiException
Noric Couderc's avatar
Noric Couderc committed
6
import se.lth.cs.ApplicationGeneration.ListApplicationGenerator
Noric Couderc's avatar
Noric Couderc committed
7
8
import java.io.File

9
open class PapiRunner(counters : CounterSpecification) {
10
11
12
    init {
        Papi.init()
    }
13
14
15

    val counterSpec = counters

Noric Couderc's avatar
Noric Couderc committed
16
17
18
19
20
21
22
23
24
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
    fun emptyBenchmark(): MutableMap<String, List<Long>> {
        // For each counter,
        // we store the values for each run (10 runs)
        var data : MutableMap<String, List<Long>> = mutableMapOf()

25
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
            val evset = EventSet.create(kvp.value)

            val current : MutableList<Long> = mutableListOf()

            for(warmup in 0 .. 100) {
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
                    if(acc % 2 == 1) {
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
            data.set(kvp.key, current)
        }
        return data
    }

53
54
55
56
    /**
     * Runs a function several times
     * @return A map from PAPI counter names to list of values
     */
Noric Couderc's avatar
Noric Couderc committed
57
    inline fun runFunction(numRuns : Int, counter : String, function : () -> Unit): MutableList<Long> {
58
        val counterId = counterSpec.getCounter(counter)!!
Noric Couderc's avatar
Noric Couderc committed
59
60
61
62
63
64
65
        // We record only one counter
        var evset = EventSet.create()
        try {
            evset = EventSet.create(counterId)
        } catch (e : PapiException) {
            error("Failed to sample counter: ${counter}")
        }
66

Noric Couderc's avatar
Noric Couderc committed
67
68
69
70
71
72
73
74
75
76
77
        // We run the function n times
        var values = mutableListOf<Long>()
        for (run in 0..numRuns) {
            // We do the measurements
            evset.start()
            val result = function()
            evset.stop()

            // We record the data
            val data = evset.counters
            values.addAll(data.toList())
78
        }
Noric Couderc's avatar
Noric Couderc committed
79
        return values
80
81
    }

82
83
84
85

    /** Runs a set of programs (functions) without interleaving
     * (Performance should get better if there is JIT compilation)
     * @Returns A map from couples (counter, program-name) -> values over all runs
Noric Couderc's avatar
Noric Couderc committed
86
     * TODO: Implement this using cartesianProduct from Guava instead
87
     */
88
89
    open fun runApplications(numRuns : Int, functions : List<Pair<String,() -> Any>>):
            Map<String, MutableMap<String, List<Long>>> {
90

Noric Couderc's avatar
Noric Couderc committed
91
92
93
94
95
96
        // We store a map from program names to map with counters and list of values
        var data = mutableMapOf<String, MutableMap<String, List<Long>>>()
        // We initialize data with empty maps
        for (f in functions) {
            data.put(f.first, mutableMapOf())
        }
97
98

        // For each counter that is available
99
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
100
101
102
            val counterName = kvp.key
            println("Streamlined mode: '$counterName'")

103
104
105
106
            // We record only one counter
            val evset = EventSet.create(kvp.value)
            // For each program...
            for (function in functions) {
Noric Couderc's avatar
Noric Couderc committed
107
108
109
                val appName = function.first

                val current = Pair(counterName, appName)
110
111
                // We run it n times
                var values = mutableListOf<Long>()
112
                for (run in 0 until numRuns) {
113
114
115
116
117
118
119
                    // We do the measurements
                    evset.start()
                    val result = function.second()
                    evset.stop()

                    //println(result)
                    // We record the data
Noric Couderc's avatar
Noric Couderc committed
120
                    values.addAll(evset.counters.toList())
121
                }
Noric Couderc's avatar
Noric Couderc committed
122
                data[appName]!![counterName] = values
123
124
125
126
127
128
            }
        }

        return data
    }

Noric Couderc's avatar
Noric Couderc committed
129
130
131
132
133
134
135
    /**
     * Runs a function several times, gather the performance counters
     * and returns their median
     * @param numRuns Number of times the function should be ran
     * @param function The function to benchmark
     * @return A map from PAPI counter names to the median of their values over numRuns
     */
Noric Couderc's avatar
Noric Couderc committed
136
137
138
    fun runFunctionMedian(numRuns : Int, counter : String, function : () -> Unit) : Double {
        val data = runFunction(numRuns, counter, function)
        return medianLong(data)
Noric Couderc's avatar
Noric Couderc committed
139
140
    }

141
142
    /**
     * Runs a list of generated applications without interleaving
Noric Couderc's avatar
Noric Couderc committed
143
     * @Returns A map from couple counter_program-name -> values over all runs
144
     */
145
    fun runListApplications(numRuns: Int, applications: List<Application<*>>):
146
            Map<String, MutableMap<String, List<Long>>> {
147
        val apps = applications.map {
Noric Couderc's avatar
Noric Couderc committed
148
            Pair(it.identifier, { it.benchmark() })
149
150
        }

151
        return runApplications(numRuns, apps)
152
153
    }

Noric Couderc's avatar
Noric Couderc committed
154
155
156
157
    /**
     * A class for feature vectors with the label of the app (seed), the fastest datastructure for that app,
     * and the performance counters for that app
     */
Noric Couderc's avatar
Noric Couderc committed
158
159
160
161
    data class FeatureVector(val appLabel : String,
                             val dataStructure : String,
                             val bestDataStructure : String,
                             val counters : Map<String, Double>)
Noric Couderc's avatar
Noric Couderc committed
162
163
164
165

    /**
     * Runs a couple of generated applications and returns their feature vectors
     */
166
    fun getFeatures(numRuns: Int, applications : List<Application<*>>):
Noric Couderc's avatar
Noric Couderc committed
167
            List<FeatureVector> {
Noric Couderc's avatar
Noric Couderc committed
168

169
170
        val trainingSet =
                ApplicationRunner().runBenchmarks(applications)
171

Noric Couderc's avatar
Noric Couderc committed
172
        val distribution = trainingSet.groupBy { it.bestDataStructure }.mapValues { it.value.size }
173
174
175
        println("Benchmark distribution : $distribution")

        val apps = trainingSet.map { it.application }
Noric Couderc's avatar
Noric Couderc committed
176
177
178
179
        val appsTocounters =
                runListApplications(numRuns, apps).mapValues {
                    it.value.mapValues { medianLong(it.value) }
                }
180

Noric Couderc's avatar
Noric Couderc committed
181
        val featureVectors = trainingSet.map {
182
            FeatureVector(
Noric Couderc's avatar
Noric Couderc committed
183
184
185
186
                    it.application.identifier,
                    it.dataStructure,
                    it.bestDataStructure,
                    appsTocounters[it.application.identifier]!!
Noric Couderc's avatar
Noric Couderc committed
187
188
            )
        }
Noric Couderc's avatar
Noric Couderc committed
189
        return featureVectors
190
191
    }

192
193
194
195
196
    fun featuresToCSV(vectors : List<FeatureVector>) : String {
        if (vectors.isEmpty()) return ""

        var header = mutableListOf(
                "application",
Noric Couderc's avatar
Noric Couderc committed
197
198
                "data_structure",
                "best_data_structure")
199
200
201
202
203
204
205
206
207
208
209
210

        val counters = vectors.map { it.counters.keys }
                .fold(setOf()) { s : Set<String>, v -> s.union(v)}

        header.addAll(counters)
        val headerText = header.joinToString(",")

        var values = mutableListOf<List<String>>()
        for (v in vectors) {
            var l = mutableListOf<String>()
            l.add(v.appLabel)
            l.add(v.dataStructure)
Noric Couderc's avatar
Noric Couderc committed
211
            l.add(v.bestDataStructure)
212
213
214
215
216
217
218
219
220
221
222
223
224
            for (c in counters) {
                l.add(v.counters[c]?.toString() ?: "None")
            }
            values.add(l)
        }

        val valuesTexts = values.map {
            it.joinToString(",")
        }

        val valuesText = valuesTexts.joinToString("\n")
        return "$headerText\n$valuesText"
    }
Noric Couderc's avatar
Noric Couderc committed
225
226
227
}

fun main(args : Array<String>) {
228
    val r = PapiRunner(CounterSpecification.fromFile(File("PAPI_FLAGS")))
Noric Couderc's avatar
Noric Couderc committed
229
230
231
    val apps = // ListApplicationGenerator().createApplications(0, 100, 100)
        ApplicationRunner().createListApplicationsSpread(20, 100, ListApplicationGenerator())
    val data = r.getFeatures(20, apps.map { it.application })
232
    val file = File("benchmarkoutput.csv")
233
    file.writeText(r.featuresToCSV(data))
234

235
    /*
236
    val gson = Gson()
237
238
239
    val data1 = r.runListApplications(100, apps)
    val splitted = data1.toList().groupBy {
        it.first.programName.split(":")[1] // Name of data structure
240
    }.mapValues { it.value.toMap() }
241

242
243
    for (kvp in splitted) {
        val data = kvp.value.mapKeys {
244
            it.key.toString().split(":")[0] // We only save before the separator
245
246
247
248
249
        }
        // val suffix = kvp.value.hashCode()
        val file = File("benchmarkoutput-${kvp.key}.json")
        file.writeText(gson.toJson(data))
    }
250
    */
251

252
    /* GENERATING A BENCHMARK
Noric Couderc's avatar
Noric Couderc committed
253
254
255
256
257
    val functions = listOf(
            Pair("1", { test1() }),
            Pair("2", { test2() }),
            Pair("3", { test3() }))

258
    val data = r.runWithoutInterleaving(1000, functions)
259

260
261
    val suffix = data.hashCode()
    val file = File("benchmarkoutput-warmup-$suffix.json")
Noric Couderc's avatar
Noric Couderc committed
262
    file.writeText(gson.toJson(data))
263
264

    val data1 = r.runWithInterleaving(1000, functions)
265
266
    val suffix1 = data1.hashCode()
    val file1 = File("benchmarkoutput-interleaved-warmup-$suffix1.json")
267
    file1.writeText(gson.toJson(data1))
268
    */
Noric Couderc's avatar
Noric Couderc committed
269
}