PapiRunner.kt 8.47 KB
Newer Older
Noric Couderc's avatar
Noric Couderc committed
1
2
3
4
package se.lth.cs

import papi.EventSet
import papi.Papi
5
import papi.PapiException
Noric Couderc's avatar
Noric Couderc committed
6
import se.lth.cs.ApplicationGeneration.ListApplicationGenerator
Noric Couderc's avatar
Noric Couderc committed
7
8
import java.io.File

Noric Couderc's avatar
Noric Couderc committed
9
open class PapiRunner(counters: CounterSpecification) {
10
11
12
    init {
        Papi.init()
    }
13
14
15

    val counterSpec = counters

Noric Couderc's avatar
Noric Couderc committed
16
17
18
19
20
21
22
    /**
     * Empty benchmark:
     * Test to see if the results are stable.
     */
    fun emptyBenchmark(): MutableMap<String, List<Long>> {
        // For each counter,
        // we store the values for each run (10 runs)
Noric Couderc's avatar
Noric Couderc committed
23
        var data: MutableMap<String, List<Long>> = mutableMapOf()
Noric Couderc's avatar
Noric Couderc committed
24

25
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
26
27
            val evset = EventSet.create(kvp.value)

Noric Couderc's avatar
Noric Couderc committed
28
            val current: MutableList<Long> = mutableListOf()
Noric Couderc's avatar
Noric Couderc committed
29

Noric Couderc's avatar
Noric Couderc committed
30
            for (warmup in 0..100) {
Noric Couderc's avatar
Noric Couderc committed
31
32
33
34
35
36
37
38
                val a = (0..warmup).toList().toTypedArray()
                val b = IntArray(warmup)

                evset.start()
                // Synthetic piece of code to see if counters run as expected
                var acc = 0
                for (i in 0 until warmup) {
                    acc += a[i]
Noric Couderc's avatar
Noric Couderc committed
39
                    if (acc % 2 == 1) {
Noric Couderc's avatar
Noric Couderc committed
40
41
42
43
44
45
46
47
48
49
50
51
52
                        b[i] = acc
                    }
                }
                evset.stop()
                // Get data
                val currentData = evset.counters
                current.addAll(currentData.toList())
            }
            data.set(kvp.key, current)
        }
        return data
    }

53
54
55
56
    /**
     * Runs a function several times
     * @return A map from PAPI counter names to list of values
     */
Noric Couderc's avatar
Noric Couderc committed
57
    inline fun runFunction(numRuns: Int, counter: String, function: () -> Unit): MutableList<Long> {
58
        val counterId = counterSpec.getCounter(counter)!!
Noric Couderc's avatar
Noric Couderc committed
59
60
61
62
        // We record only one counter
        var evset = EventSet.create()
        try {
            evset = EventSet.create(counterId)
Noric Couderc's avatar
Noric Couderc committed
63
        } catch (e: PapiException) {
Noric Couderc's avatar
Noric Couderc committed
64
65
            error("Failed to sample counter: ${counter}")
        }
66

Noric Couderc's avatar
Noric Couderc committed
67
68
69
70
71
72
73
74
75
76
77
        // We run the function n times
        var values = mutableListOf<Long>()
        for (run in 0..numRuns) {
            // We do the measurements
            evset.start()
            val result = function()
            evset.stop()

            // We record the data
            val data = evset.counters
            values.addAll(data.toList())
78
        }
Noric Couderc's avatar
Noric Couderc committed
79
        return values
80
81
    }

82
83
84
85

    /** Runs a set of programs (functions) without interleaving
     * (Performance should get better if there is JIT compilation)
     * @Returns A map from couples (counter, program-name) -> values over all runs
Noric Couderc's avatar
Noric Couderc committed
86
     * TODO: Implement this using cartesianProduct from Guava instead
87
     */
88
89
    open fun runApplications(numRuns: Int, applications: List<Application<*>>):
            Map<Application<*>, MutableMap<String, List<Long>>> {
90

Noric Couderc's avatar
Noric Couderc committed
91
        // We store a map from program names to map with counters and list of values
92
        var data = mutableMapOf<Application<*>, MutableMap<String, List<Long>>>()
Noric Couderc's avatar
Noric Couderc committed
93
        // We initialize data with empty maps
94
95
        for (app in applications) {
            data.put(app, mutableMapOf())
Noric Couderc's avatar
Noric Couderc committed
96
        }
97
98

        // For each counter that is available
99
        for (kvp in counterSpec.currentSpec) {
Noric Couderc's avatar
Noric Couderc committed
100
101
102
            val counterName = kvp.key
            println("Streamlined mode: '$counterName'")

103
104
105
            // We record only one counter
            val evset = EventSet.create(kvp.value)
            // For each program...
106
            for (app in applications) {
107
108
                // We run it n times
                var values = mutableListOf<Long>()
109
                for (run in 0 until numRuns) {
110
111
                    // We do the measurements
                    evset.start()
112
                    val result = app.benchmark()
113
114
115
116
                    evset.stop()

                    //println(result)
                    // We record the data
Noric Couderc's avatar
Noric Couderc committed
117
                    values.addAll(evset.counters.toList())
118
                }
119
                data[app]!![counterName] = values
120
121
122
123
124
125
            }
        }

        return data
    }

Noric Couderc's avatar
Noric Couderc committed
126
127
128
129
130
131
132
    /**
     * Runs a function several times, gather the performance counters
     * and returns their median
     * @param numRuns Number of times the function should be ran
     * @param function The function to benchmark
     * @return A map from PAPI counter names to the median of their values over numRuns
     */
Noric Couderc's avatar
Noric Couderc committed
133
    fun runFunctionMedian(numRuns: Int, counter: String, function: () -> Unit): Double {
Noric Couderc's avatar
Noric Couderc committed
134
135
        val data = runFunction(numRuns, counter, function)
        return medianLong(data)
Noric Couderc's avatar
Noric Couderc committed
136
137
138
139
140
141
    }

    /**
     * A class for feature vectors with the label of the app (seed), the fastest datastructure for that app,
     * and the performance counters for that app
     */
Noric Couderc's avatar
Noric Couderc committed
142
143
144
145
    data class FeatureVector(val appLabel: String,
                             val dataStructure: String,
                             val bestDataStructure: String,
                             val counters: Map<String, Double>)
Noric Couderc's avatar
Noric Couderc committed
146
147
148
149

    /**
     * Runs a couple of generated applications and returns their feature vectors
     */
Noric Couderc's avatar
Noric Couderc committed
150
    fun getFeatures(numRuns: Int, applications: List<Application<*>>):
Noric Couderc's avatar
Noric Couderc committed
151
            List<FeatureVector> {
Noric Couderc's avatar
Noric Couderc committed
152

153
154
        val trainingSet =
                ApplicationRunner().runBenchmarks(applications)
155

Noric Couderc's avatar
Noric Couderc committed
156
        val distribution = trainingSet.groupBy { it.bestDataStructure }.mapValues { it.value.size }
157
158
159
        println("Benchmark distribution : $distribution")

        val apps = trainingSet.map { it.application }
Noric Couderc's avatar
Noric Couderc committed
160
        val appsTocounters =
161
                runApplications(numRuns, apps).mapValues {
Noric Couderc's avatar
Noric Couderc committed
162
163
                    it.value.mapValues { medianLong(it.value) }
                }
164

Noric Couderc's avatar
Noric Couderc committed
165
        val featureVectors = trainingSet.map {
166
            FeatureVector(
Noric Couderc's avatar
Noric Couderc committed
167
168
169
                    it.application.identifier,
                    it.dataStructure,
                    it.bestDataStructure,
170
                    appsTocounters.getValue(it.application)
Noric Couderc's avatar
Noric Couderc committed
171
172
            )
        }
Noric Couderc's avatar
Noric Couderc committed
173
        return featureVectors
174
175
    }

Noric Couderc's avatar
Noric Couderc committed
176
    fun featuresToCSV(vectors: List<FeatureVector>): String {
177
178
179
180
        if (vectors.isEmpty()) return ""

        var header = mutableListOf(
                "application",
Noric Couderc's avatar
Noric Couderc committed
181
182
                "data_structure",
                "best_data_structure")
183
184

        val counters = vectors.map { it.counters.keys }
Noric Couderc's avatar
Noric Couderc committed
185
                .fold(setOf()) { s: Set<String>, v -> s.union(v) }
186
187
188
189
190
191
192
193
194

        header.addAll(counters)
        val headerText = header.joinToString(",")

        var values = mutableListOf<List<String>>()
        for (v in vectors) {
            var l = mutableListOf<String>()
            l.add(v.appLabel)
            l.add(v.dataStructure)
Noric Couderc's avatar
Noric Couderc committed
195
            l.add(v.bestDataStructure)
196
197
198
199
200
201
202
203
204
205
206
207
208
            for (c in counters) {
                l.add(v.counters[c]?.toString() ?: "None")
            }
            values.add(l)
        }

        val valuesTexts = values.map {
            it.joinToString(",")
        }

        val valuesText = valuesTexts.joinToString("\n")
        return "$headerText\n$valuesText"
    }
209
210
211
212

    fun processJMHData(jmhData : MutableList<MutableList<String>>): MutableList<PapiRunner.FeatureVector> {
        return listOf<FeatureVector>().toMutableList()
    }
Noric Couderc's avatar
Noric Couderc committed
213
214
}

Noric Couderc's avatar
Noric Couderc committed
215
fun main(args: Array<String>) {
216
    val r = PapiRunner(CounterSpecification.fromFile(File("PAPI_FLAGS")))
Noric Couderc's avatar
Noric Couderc committed
217
    val apps = // ListApplicationGenerator().createApplications(0, 100, 100)
Noric Couderc's avatar
Noric Couderc committed
218
            ApplicationRunner().createListApplicationsSpread(20, 100, ListApplicationGenerator())
Noric Couderc's avatar
Noric Couderc committed
219
    val data = r.getFeatures(20, apps.map { it.application })
220
    val file = File("benchmarkoutput.csv")
221
    file.writeText(r.featuresToCSV(data))
222

223
    /*
224
    val gson = Gson()
225
226
227
    val data1 = r.runListApplications(100, apps)
    val splitted = data1.toList().groupBy {
        it.first.programName.split(":")[1] // Name of data structure
228
    }.mapValues { it.value.toMap() }
229

230
231
    for (kvp in splitted) {
        val data = kvp.value.mapKeys {
232
            it.key.toString().split(":")[0] // We only save before the separator
233
234
235
236
237
        }
        // val suffix = kvp.value.hashCode()
        val file = File("benchmarkoutput-${kvp.key}.json")
        file.writeText(gson.toJson(data))
    }
238
    */
239

240
    /* GENERATING A BENCHMARK
Noric Couderc's avatar
Noric Couderc committed
241
242
243
244
245
    val functions = listOf(
            Pair("1", { test1() }),
            Pair("2", { test2() }),
            Pair("3", { test3() }))

246
    val data = r.runWithoutInterleaving(1000, functions)
247

248
249
    val suffix = data.hashCode()
    val file = File("benchmarkoutput-warmup-$suffix.json")
Noric Couderc's avatar
Noric Couderc committed
250
    file.writeText(gson.toJson(data))
251
252

    val data1 = r.runWithInterleaving(1000, functions)
253
254
    val suffix1 = data1.hashCode()
    val file1 = File("benchmarkoutput-interleaved-warmup-$suffix1.json")
255
    file1.writeText(gson.toJson(data1))
256
    */
Noric Couderc's avatar
Noric Couderc committed
257
}