TracingCollectionRunnerTest.kt 9.14 KB
Newer Older
1
2
import com.google.common.graph.GraphBuilder
import com.google.common.graph.MutableGraph
3
import org.apache.commons.math3.stat.regression.SimpleRegression
Noric Couderc's avatar
Noric Couderc committed
4
import org.junit.jupiter.api.*
Noric Couderc's avatar
Noric Couderc committed
5
6
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.ValueSource
7
import org.openjdk.jmh.infra.Blackhole
Noric Couderc's avatar
Noric Couderc committed
8
import papi.Constants
9
import papi.EventSet
10
import papi.Papi
11
import se.lth.cs.bcgen.BCBenchmarkPackage
12
import se.lth.cs.papicounters.PAPICounter
13
import se.lth.cs.papicounters.PapiBenchmarkAnalyzer
14
import se.lth.cs.papicounters.PapiTracerRunner
15
import se.lth.cs.smartmodules.tracer.Tracer
16
import se.lth.cs.timing.OperationType
Noric Couderc's avatar
Noric Couderc committed
17
import se.lth.cs.util.FeatureSet
18
19
import se.lth.util.ArrayListTracer
import java.io.ByteArrayOutputStream
20
import java.io.File
21
import java.io.PrintStream
22
23
24
25
import java.util.*
import kotlin.collections.ArrayList
import kotlin.collections.HashMap
import kotlin.collections.HashSet
26
import kotlin.system.measureTimeMillis
27
28
29
30
31
32

class TracingCollectionRunnerTest {
    // In this file, we will test running benchmarks with tracing collections
    // We create a benchmark, run it with tracing collections, and obtain the counters afterwards
    companion object {
        val blackhole = Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.")
Noric Couderc's avatar
Noric Couderc committed
33

34
35
        var runner: PapiTracerRunner? = null

Noric Couderc's avatar
Noric Couderc committed
36
37
        @BeforeAll()
        @JvmStatic
38
        fun initTracer() {
39
            // EventSet.logCalls = false
Noric Couderc's avatar
Noric Couderc committed
40
        }
41
42
    }

43
44
45
46
47
48
49
    @BeforeEach
    fun setup() {
        val papiAvailableCounters = File("../papi_avail")
        Assertions.assertTrue(papiAvailableCounters.exists())
        runner = PapiTracerRunner()
    }

50
    @Test
Noric Couderc's avatar
Noric Couderc committed
51
    @Disabled("Results differ depending of Java version")
52
53
    fun testOneBenchmark() {
        val l = ArrayListTracer<Any>("loc1")
Noric Couderc's avatar
Noric Couderc committed
54

55
56
57
58
59
        val b = BCBenchmarkPackage.LIST(40 , 50, 0, l)

        b.runBenchmark(blackhole)

        val s = ByteArrayOutputStream()
Noric Couderc's avatar
Noric Couderc committed
60
        Tracer.printRecords(1, PrintStream(s))
Noric Couderc's avatar
Noric Couderc committed
61
62
63
64
65
        // We're gonna clean up the lines where we have 0 invocations
        val partitioned = s.toString().lines().partition {
            it.contains("invocations\t0")
        }

Noric Couderc's avatar
Noric Couderc committed
66
        // We should have dropped 19 lines (zero invocations)
Noric Couderc's avatar
Noric Couderc committed
67
68
        // We get different results depending of the java version...
        Assertions.assertTrue(partitioned.first.size in listOf(18, 19))
Noric Couderc's avatar
Noric Couderc committed
69

70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
        // We analyze the results with a graph.
        val g : MutableGraph<String> = GraphBuilder.directed().build<String>()

        for (line in partitioned.second) {
            if (line.isEmpty()) { continue}
            val splitted = line.split("\t")
            // Source is the pair of location + method name.
            val source = Pair(splitted[0], splitted[4]).toString()
            // Target is the name of the feature.
            val target = splitted[1]
            g.putEdge(source, target)
        }

        for (v in g.nodes()) {
            val next = g.successors(v)
            val pred = g.predecessors(v)
            if (next.isEmpty()) {
                // We have a feature node
Noric Couderc's avatar
Noric Couderc committed
88
89
90
                // We should have 18 predecessors (for java > 8)
                // 19 for java 8
                Assertions.assertTrue(pred.size in listOf(18, 19))
91
92
93
94
95
96
97
98
            } else {
                Assertions.assertEquals(3, next.size)
                Assertions.assertTrue(next.contains("0x8000003b"))
                Assertions.assertTrue(next.contains("0x80000032"))
                Assertions.assertTrue(next.contains("invocations"))
            }
        }

Noric Couderc's avatar
Noric Couderc committed
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
        val filtered = partitioned.second.filter {
            it.contains("invocations")
        }

        val expectedText = """
            loc1	invocations	4	java.util.ArrayList	java.lang.String toString()
            loc1	invocations	4	java.util.ArrayList	void sort(java.util.Comparator<? super E>)
            loc1	invocations	3	java.util.ArrayList	boolean equals(java.lang.Object)
            loc1	invocations	3	java.util.ArrayList	void clear()
            loc1	invocations	2	java.util.ArrayList	void add(int, E)
            loc1	invocations	7	java.util.ArrayList	java.util.ListIterator<E> listIterator(int)
            loc1	invocations	3	java.util.ArrayList	java.util.Iterator<E> iterator()
            loc1	invocations	1	java.util.ArrayList	T[] toArray(T[])
            loc1	invocations	1	java.util.ArrayList	E set(int, E)
            loc1	invocations	2	java.util.ArrayList	E remove(int)
            loc1	invocations	9	java.util.ArrayList	int size()
            loc1	invocations	6	java.util.ArrayList	int indexOf(java.lang.Object)
            loc1	invocations	2	java.util.ArrayList	boolean retainAll(java.util.Collection<?>)
            loc1	invocations	3	java.util.ArrayList	boolean removeAll(java.util.Collection<?>)
            loc1	invocations	3	java.util.ArrayList	boolean remove(java.lang.Object)
            loc1	invocations	2	java.util.ArrayList	boolean isEmpty()
            loc1	invocations	1	java.util.ArrayList	boolean containsAll(java.util.Collection<?>)
            loc1	invocations	3	java.util.ArrayList	boolean addAll(java.util.Collection<? extends E>)
            """.trimIndent()

        Assertions.assertEquals(expectedText, java.lang.String.join("\n", filtered))
125
126
    }

127
128
    // What we need: A drop-in replacement for PapiRunner, which uses these collections instead.

129

130
131
132
133
134
135

    @Test
    fun testRunner() {
        val benches = listOf(
            BCBenchmarkPackage.LIST(10, 10, 0, ArrayList<Int>())
        )
136

Noric Couderc's avatar
Noric Couderc committed
137
138
139
        val counters = FeatureSet(PAPICounter("PAPI_TOT_CYC"),
            PAPICounter("PAPI_TOT_INS"),
            PAPICounter("PAPI_L1_DCM"))
140
        val res = runner!!.runApplications(10, counters,  benches)
141

142
        // We only tried one benchmark, but we should get several counters.
143
        // One for each number
144
        Assertions.assertEquals(10, res.size)
145

146
        for (t in res) {
147
            Assertions.assertEquals(benches.get(0), t.benchmark)
148
            Assertions.assertTrue(t.iteration < 10)
149
            Assertions.assertEquals(
150
                counters.toSet() ,
151
                t.samples.map { it.counter }.toSet())
152
153
154
        }
    }

Noric Couderc's avatar
Noric Couderc committed
155
156
157
158
    @ParameterizedTest(name = "Running a spec returns the expected report (seed {0})")
    @ValueSource(ints = [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
    fun testRunSpec(seed : Long) {
        val bench = BCBenchmarkPackage.LIST(seed, 10, 0, ArrayList<Int>())
159

160
        val spec = PapiBenchmarkAnalyzer.RunSpec(
161
            10,
Noric Couderc's avatar
Noric Couderc committed
162
            FeatureSet(PAPICounter("PAPI_TOT_CYC")),
Noric Couderc's avatar
Noric Couderc committed
163
            bench)
164
165

        val iterations = runner!!.runSpec(spec)
166

Noric Couderc's avatar
Noric Couderc committed
167
        // TODO: Create a mockup class for this test.
168
169
170
        Assertions.assertEquals(10, iterations.size)

        for (iteration in iterations) {
171
            for (sample in iteration.samples) {
172
                Assertions.assertEquals(PAPICounter("PAPI_TOT_CYC"), sample.counter)
173
                Assertions.assertTrue(sample.value > 0)
174
                Assertions.assertTrue(sample.value < 1000000)
175
            }
176
        }
177
    }
178
179

    // We now test getting the number of cycles per operation type
Noric Couderc's avatar
Noric Couderc committed
180
181
182
    @ParameterizedTest(name = "Test the gathering of cycles works.")
    @ValueSource(ints = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
    fun testCyclesPerOpType(seed : Long) {
183
        val benches = listOf(
Noric Couderc's avatar
Noric Couderc committed
184
            BCBenchmarkPackage.MAP(seed, 20, 0, HashMap<Int, Int>())
185
186
        )

Noric Couderc's avatar
Noric Couderc committed
187
188
189
        val counters = FeatureSet(PAPICounter("PAPI_TOT_CYC"),
            PAPICounter("PAPI_TOT_INS"),
            PAPICounter("PAPI_L1_DCM"))
190
        val iterations = runner!!.createRunSpecs(10, counters, benches)
191
192
193
194
195

        val samples = iterations.flatMap {
            runner!!.runIterations(it)
        }

196
        val cycles = samples.map { runner!!.getCyclesPerOpType(it) }
197

198
        val expectedCounters = OperationType.values().toSet()
199

200
        Assertions.assertEquals(10, cycles.size)
201
202

        Assertions.assertEquals(expectedCounters, cycles.flatMap { it.keys }.toSet())
203
204

    }
205
206
207
208
209
210

    @ParameterizedTest(name = "Test we only run the benchmark once when getting the counters (seed = {0})")
    @ValueSource(ints = [100, 101, 102, 103, 104, 105, 106])
    fun testBenchmarkRunsOnce(seed : Long) {
        val bench = BCBenchmarkPackage.SET(seed, 100, 0, HashSet<Int>())

Noric Couderc's avatar
Noric Couderc committed
211
        val results = runner!!.runApplications(10, FeatureSet(), listOf(bench))
212

213
        // We reset numberRuns for each iteration, so the benchmarks is run "once".
214
215
216
217
        for (r in results) {
            Assertions.assertEquals(1, r.benchmark.numberRuns)
        }
    }
218
219
220
221
222
223

    @Test
    fun testPerformance() {
        // TODO: Do the same, but create a lot of benchmarks!
        val bench = BCBenchmarkPackage.LIST(12345, 1000, 0, ArrayList<Int>())

Noric Couderc's avatar
Noric Couderc committed
224
225
        val features = FeatureSet(PAPICounter("PAPI_TOT_CYC"),
            PAPICounter("PAPI_TOT_INS"))
226
227
228
229
230
231

        val times = mutableListOf<Double>()
        val max = 100
        for (i in 0..max) {
            val timeSpent = measureTimeMillis {
                runner!!.runIterations(
Noric Couderc's avatar
Noric Couderc committed
232
                    PapiBenchmarkAnalyzer.RunSpec(10, features, bench)
233
234
235
236
237
238
239
240
241
242
243
244
245
                )
            }
            times.add(timeSpent / 1000.0)
        }

        val reg = SimpleRegression()
        for (i in 0..max) {
            reg.addData(i.toDouble(), times[i])
        }

        Assertions.assertEquals(0.0, reg.slope, 0.01)

    }
246
}