TracingCollectionRunnerTest.kt 9.11 KB
Newer Older
1
2
import com.google.common.graph.GraphBuilder
import com.google.common.graph.MutableGraph
3
import org.apache.commons.math3.stat.regression.SimpleRegression
Noric Couderc's avatar
Noric Couderc committed
4
import org.junit.jupiter.api.*
Noric Couderc's avatar
Noric Couderc committed
5
6
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.ValueSource
7
import org.openjdk.jmh.infra.Blackhole
Noric Couderc's avatar
Noric Couderc committed
8
import papi.Constants
9
import papi.EventSet
10
import se.lth.cs.average
11
import se.lth.cs.bcgen.BCBenchmarkPackage
12
import se.lth.cs.papicounters.PAPICounter
13
import se.lth.cs.papicounters.PapiBenchmarkAnalyzer
14
import se.lth.cs.papicounters.PapiTracerRunner
15
import se.lth.cs.smartmodules.tracer.Tracer
16
import se.lth.cs.timing.OperationType
Noric Couderc's avatar
Noric Couderc committed
17
import se.lth.cs.util.FeatureSet
18
import se.lth.cs.variance
19
20
import se.lth.util.ArrayListTracer
import java.io.ByteArrayOutputStream
21
import java.io.File
22
import java.io.PrintStream
23
import kotlin.system.measureTimeMillis
24
25
26
27
28
29
30

class TracingCollectionRunnerTest {
    // In this file, we will test running benchmarks with tracing collections
    // We create a benchmark, run it with tracing collections, and obtain the counters afterwards

    companion object {
        val blackhole = Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.")
Noric Couderc's avatar
Noric Couderc committed
31

32
33
        var runner: PapiTracerRunner? = null

Noric Couderc's avatar
Noric Couderc committed
34
35
        @BeforeAll()
        @JvmStatic
36
        fun initTracer() {
Noric Couderc's avatar
Noric Couderc committed
37
            System.out.println("Initialize PAPI")
Noric Couderc's avatar
Noric Couderc committed
38
            Tracer.JPAPI = true
39
40
            Tracer.addCounter(Constants.PAPI_TOT_CYC)
            Tracer.addCounter(Constants.PAPI_TOT_INS)
Noric Couderc's avatar
Noric Couderc committed
41
        }
42
43
44
    }

    @Test
Noric Couderc's avatar
Noric Couderc committed
45
    @Disabled("Results differ depending of Java version")
46
47
    fun testOneBenchmark() {
        val l = ArrayListTracer<Any>("loc1")
Noric Couderc's avatar
Noric Couderc committed
48

49
50
51
52
53
        val b = BCBenchmarkPackage.LIST(40 , 50, 0, l)

        b.runBenchmark(blackhole)

        val s = ByteArrayOutputStream()
Noric Couderc's avatar
Noric Couderc committed
54
        Tracer.printRecords(1, PrintStream(s))
Noric Couderc's avatar
Noric Couderc committed
55
56
57
58
59
        // We're gonna clean up the lines where we have 0 invocations
        val partitioned = s.toString().lines().partition {
            it.contains("invocations\t0")
        }

Noric Couderc's avatar
Noric Couderc committed
60
        // We should have dropped 19 lines (zero invocations)
Noric Couderc's avatar
Noric Couderc committed
61
62
        // We get different results depending of the java version...
        Assertions.assertTrue(partitioned.first.size in listOf(18, 19))
Noric Couderc's avatar
Noric Couderc committed
63

64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
        // We analyze the results with a graph.
        val g : MutableGraph<String> = GraphBuilder.directed().build<String>()

        for (line in partitioned.second) {
            if (line.isEmpty()) { continue}
            val splitted = line.split("\t")
            // Source is the pair of location + method name.
            val source = Pair(splitted[0], splitted[4]).toString()
            // Target is the name of the feature.
            val target = splitted[1]
            g.putEdge(source, target)
        }

        for (v in g.nodes()) {
            val next = g.successors(v)
            val pred = g.predecessors(v)
            if (next.isEmpty()) {
                // We have a feature node
Noric Couderc's avatar
Noric Couderc committed
82
83
84
                // We should have 18 predecessors (for java > 8)
                // 19 for java 8
                Assertions.assertTrue(pred.size in listOf(18, 19))
85
86
87
88
89
90
91
92
            } else {
                Assertions.assertEquals(3, next.size)
                Assertions.assertTrue(next.contains("0x8000003b"))
                Assertions.assertTrue(next.contains("0x80000032"))
                Assertions.assertTrue(next.contains("invocations"))
            }
        }

Noric Couderc's avatar
Noric Couderc committed
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
        val filtered = partitioned.second.filter {
            it.contains("invocations")
        }

        val expectedText = """
            loc1	invocations	4	java.util.ArrayList	java.lang.String toString()
            loc1	invocations	4	java.util.ArrayList	void sort(java.util.Comparator<? super E>)
            loc1	invocations	3	java.util.ArrayList	boolean equals(java.lang.Object)
            loc1	invocations	3	java.util.ArrayList	void clear()
            loc1	invocations	2	java.util.ArrayList	void add(int, E)
            loc1	invocations	7	java.util.ArrayList	java.util.ListIterator<E> listIterator(int)
            loc1	invocations	3	java.util.ArrayList	java.util.Iterator<E> iterator()
            loc1	invocations	1	java.util.ArrayList	T[] toArray(T[])
            loc1	invocations	1	java.util.ArrayList	E set(int, E)
            loc1	invocations	2	java.util.ArrayList	E remove(int)
            loc1	invocations	9	java.util.ArrayList	int size()
            loc1	invocations	6	java.util.ArrayList	int indexOf(java.lang.Object)
            loc1	invocations	2	java.util.ArrayList	boolean retainAll(java.util.Collection<?>)
            loc1	invocations	3	java.util.ArrayList	boolean removeAll(java.util.Collection<?>)
            loc1	invocations	3	java.util.ArrayList	boolean remove(java.lang.Object)
            loc1	invocations	2	java.util.ArrayList	boolean isEmpty()
            loc1	invocations	1	java.util.ArrayList	boolean containsAll(java.util.Collection<?>)
            loc1	invocations	3	java.util.ArrayList	boolean addAll(java.util.Collection<? extends E>)
            """.trimIndent()

        Assertions.assertEquals(expectedText, java.lang.String.join("\n", filtered))
119
120
    }

121
122
    // What we need: A drop-in replacement for PapiRunner, which uses these collections instead.

123
124
125
126
    @BeforeEach
    fun setup() {
        val papiAvailableCounters = File("../papi_avail")
        Assertions.assertTrue(papiAvailableCounters.exists())
127
        runner = PapiTracerRunner()
128
129
130
131
132
133
134
    }

    @Test
    fun testRunner() {
        val benches = listOf(
            BCBenchmarkPackage.LIST(10, 10, 0, ArrayList<Int>())
        )
135

Noric Couderc's avatar
Noric Couderc committed
136
137
138
        val counters = FeatureSet(PAPICounter("PAPI_TOT_CYC"),
            PAPICounter("PAPI_TOT_INS"),
            PAPICounter("PAPI_L1_DCM"))
139
        val res = runner!!.runApplications(10, counters,  benches)
140

141
        // We only tried one benchmark, but we should get several counters.
142
        // One for each number
143
        Assertions.assertEquals(10, res.size)
144

145
        for (t in res) {
146
            Assertions.assertEquals(benches.get(0), t.benchmark)
147
            Assertions.assertTrue(t.iteration < 10)
148
            Assertions.assertEquals(
149
                counters.toSet() ,
150
                t.samples.map { it.counter }.toSet())
151
152
153
        }
    }

Noric Couderc's avatar
Noric Couderc committed
154
155
156
157
    @ParameterizedTest(name = "Running a spec returns the expected report (seed {0})")
    @ValueSource(ints = [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
    fun testRunSpec(seed : Long) {
        val bench = BCBenchmarkPackage.LIST(seed, 10, 0, ArrayList<Int>())
158

159
        val spec = PapiBenchmarkAnalyzer.RunSpec(
160
            10,
Noric Couderc's avatar
Noric Couderc committed
161
            FeatureSet(PAPICounter("PAPI_TOT_CYC")),
Noric Couderc's avatar
Noric Couderc committed
162
            bench)
163
164

        val iterations = runner!!.runSpec(spec)
165

Noric Couderc's avatar
Noric Couderc committed
166
        // TODO: Create a mockup class for this test.
167
168
169
        Assertions.assertEquals(10, iterations.size)

        for (iteration in iterations) {
170
            for (sample in iteration.samples) {
171
                Assertions.assertEquals(PAPICounter("PAPI_TOT_CYC"), sample.counter)
172
                Assertions.assertTrue(sample.value > 0)
173
                Assertions.assertTrue(sample.value < 1000000)
174
            }
175
        }
176
    }
177
178

    // We now test getting the number of cycles per operation type
Noric Couderc's avatar
Noric Couderc committed
179
180
181
    @ParameterizedTest(name = "Test the gathering of cycles works.")
    @ValueSource(ints = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
    fun testCyclesPerOpType(seed : Long) {
182
        val benches = listOf(
Noric Couderc's avatar
Noric Couderc committed
183
            BCBenchmarkPackage.MAP(seed, 20, 0, HashMap<Int, Int>())
184
185
        )

Noric Couderc's avatar
Noric Couderc committed
186
187
188
        val counters = FeatureSet(PAPICounter("PAPI_TOT_CYC"),
            PAPICounter("PAPI_TOT_INS"),
            PAPICounter("PAPI_L1_DCM"))
189
        val iterations = runner!!.createRunSpecs(10, counters, benches)
190
191
192
193
194

        val samples = iterations.flatMap {
            runner!!.runIterations(it)
        }

195
        val cycles = samples.map { runner!!.getCyclesPerOpType(it) }
196

197
        val expectedCounters = OperationType.values().toSet()
198

199
        Assertions.assertEquals(10, cycles.size)
200
201

        Assertions.assertEquals(expectedCounters, cycles.flatMap { it.keys }.toSet())
202
203

    }
204
205
206
207
208
209

    @ParameterizedTest(name = "Test we only run the benchmark once when getting the counters (seed = {0})")
    @ValueSource(ints = [100, 101, 102, 103, 104, 105, 106])
    fun testBenchmarkRunsOnce(seed : Long) {
        val bench = BCBenchmarkPackage.SET(seed, 100, 0, HashSet<Int>())

Noric Couderc's avatar
Noric Couderc committed
210
        val results = runner!!.runApplications(10, FeatureSet(), listOf(bench))
211
212
213
214
215

        for (r in results) {
            Assertions.assertEquals(1, r.benchmark.numberRuns)
        }
    }
216
217
218
219
220
221

    @Test
    fun testPerformance() {
        // TODO: Do the same, but create a lot of benchmarks!
        val bench = BCBenchmarkPackage.LIST(12345, 1000, 0, ArrayList<Int>())

Noric Couderc's avatar
Noric Couderc committed
222
223
        val features = FeatureSet(PAPICounter("PAPI_TOT_CYC"),
            PAPICounter("PAPI_TOT_INS"))
224
225
226
227
228
229

        val times = mutableListOf<Double>()
        val max = 100
        for (i in 0..max) {
            val timeSpent = measureTimeMillis {
                runner!!.runIterations(
Noric Couderc's avatar
Noric Couderc committed
230
                    PapiBenchmarkAnalyzer.RunSpec(10, features, bench)
231
232
233
234
235
236
237
238
239
240
241
242
243
                )
            }
            times.add(timeSpent / 1000.0)
        }

        val reg = SimpleRegression()
        for (i in 0..max) {
            reg.addData(i.toDouble(), times[i])
        }

        Assertions.assertEquals(0.0, reg.slope, 0.01)

    }
244
}