TracingCollectionRunnerTest.kt

import com.google.common.graph.GraphBuilder
import com.google.common.graph.MutableGraph
import org.apache.commons.math3.stat.regression.SimpleRegression
import org.junit.jupiter.api.*
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.ValueSource
import org.openjdk.jmh.infra.Blackhole
import papi.Constants
import papi.EventSet
import papi.Papi
import se.lth.cs.bcgen.BCBenchmarkPackage
import se.lth.cs.papicounters.PAPICounter
import se.lth.cs.papicounters.PapiBenchmarkAnalyzer
import se.lth.cs.papicounters.PapiTracerRunner
import se.lth.cs.smartmodules.tracer.Tracer
import se.lth.cs.timing.OperationType
import se.lth.cs.util.FeatureSet
import se.lth.util.ArrayListTracer
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.PrintStream
import java.util.*
import kotlin.collections.ArrayList
import kotlin.collections.HashMap
import kotlin.collections.HashSet
import kotlin.system.measureTimeMillis

class TracingCollectionRunnerTest {
    // In this file, we will test running benchmarks with tracing collections
    // We create a benchmark, run it with tracing collections, and obtain the counters afterwards
    companion object {
        val blackhole = Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.")

        var runner: PapiTracerRunner? = null

        @BeforeAll()
        @JvmStatic
        fun initTracer() {
            // EventSet.logCalls = false
        }
    }

    @BeforeEach
    fun setup() {
        val papiAvailableCounters = File("../papi_avail")
        Assertions.assertTrue(papiAvailableCounters.exists())
        runner = PapiTracerRunner()
    }

    @Test
    @Disabled("Results differ depending of Java version")
    fun testOneBenchmark() {
        val l = ArrayListTracer<Any>("loc1")

        val b = BCBenchmarkPackage.LIST(40 , 50, 0, l)

        b.runBenchmark(blackhole)

        val s = ByteArrayOutputStream()
        Tracer.printRecords(1, PrintStream(s))
        // We're gonna clean up the lines where we have 0 invocations
        val partitioned = s.toString().lines().partition {
            it.contains("invocations\t0")
        }

        // We should have dropped 19 lines (zero invocations)
        // We get different results depending of the java version...
        Assertions.assertTrue(partitioned.first.size in listOf(18, 19))

        // We analyze the results with a graph.
        val g : MutableGraph<String> = GraphBuilder.directed().build<String>()

        for (line in partitioned.second) {
            if (line.isEmpty()) { continue}
            val splitted = line.split("\t")
            // Source is the pair of location + method name.
            val source = Pair(splitted[0], splitted[4]).toString()
            // Target is the name of the feature.
            val target = splitted[1]
            g.putEdge(source, target)
        }

        for (v in g.nodes()) {
            val next = g.successors(v)
            val pred = g.predecessors(v)
            if (next.isEmpty()) {
                // We have a feature node
                // We should have 18 predecessors (for java > 8)
                // 19 for java 8
                Assertions.assertTrue(pred.size in listOf(18, 19))
            } else {
                Assertions.assertEquals(3, next.size)
                Assertions.assertTrue(next.contains("0x8000003b"))
                Assertions.assertTrue(next.contains("0x80000032"))
                Assertions.assertTrue(next.contains("invocations"))
            }
        }

        val filtered = partitioned.second.filter {
            it.contains("invocations")
        }

        val expectedText = """
            loc1	invocations	4	java.util.ArrayList	java.lang.String toString()
            loc1	invocations	4	java.util.ArrayList	void sort(java.util.Comparator<? super E>)
            loc1	invocations	3	java.util.ArrayList	boolean equals(java.lang.Object)
            loc1	invocations	3	java.util.ArrayList	void clear()
            loc1	invocations	2	java.util.ArrayList	void add(int, E)
            loc1	invocations	7	java.util.ArrayList	java.util.ListIterator<E> listIterator(int)
            loc1	invocations	3	java.util.ArrayList	java.util.Iterator<E> iterator()
            loc1	invocations	1	java.util.ArrayList	T[] toArray(T[])
            loc1	invocations	1	java.util.ArrayList	E set(int, E)
            loc1	invocations	2	java.util.ArrayList	E remove(int)
            loc1	invocations	9	java.util.ArrayList	int size()
            loc1	invocations	6	java.util.ArrayList	int indexOf(java.lang.Object)
            loc1	invocations	2	java.util.ArrayList	boolean retainAll(java.util.Collection<?>)
            loc1	invocations	3	java.util.ArrayList	boolean removeAll(java.util.Collection<?>)
            loc1	invocations	3	java.util.ArrayList	boolean remove(java.lang.Object)
            loc1	invocations	2	java.util.ArrayList	boolean isEmpty()
            loc1	invocations	1	java.util.ArrayList	boolean containsAll(java.util.Collection<?>)
            loc1	invocations	3	java.util.ArrayList	boolean addAll(java.util.Collection<? extends E>)
            """.trimIndent()

        Assertions.assertEquals(expectedText, java.lang.String.join("\n", filtered))
    }

    // What we need: A drop-in replacement for PapiRunner, which uses these collections instead.



    @Test
    fun testRunner() {
        val benches = listOf(
            BCBenchmarkPackage.LIST(10, 10, 0, ArrayList<Int>())
        )

        val counters = FeatureSet(PAPICounter("PAPI_TOT_CYC"),
            PAPICounter("PAPI_TOT_INS"),
            PAPICounter("PAPI_L1_DCM"))
        val res = runner!!.runApplications(10, counters,  benches)

        // We only tried one benchmark, but we should get several counters.
        // One for each number
        Assertions.assertEquals(10, res.size)

        for (t in res) {
            Assertions.assertEquals(benches.get(0), t.benchmark)
            Assertions.assertTrue(t.iteration < 10)
            Assertions.assertEquals(
                counters.toSet() ,
                t.samples.map { it.counter }.toSet())
        }
    }

    @ParameterizedTest(name = "Running a spec returns the expected report (seed {0})")
    @ValueSource(ints = [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
    fun testRunSpec(seed : Long) {
        val bench = BCBenchmarkPackage.LIST(seed, 10, 0, ArrayList<Int>())

        val spec = PapiBenchmarkAnalyzer.RunSpec(
            10,
            FeatureSet(PAPICounter("PAPI_TOT_CYC")),
            bench)

        val iterations = runner!!.runSpec(spec)

        // TODO: Create a mockup class for this test.
        Assertions.assertEquals(10, iterations.size)

        for (iteration in iterations) {
            for (sample in iteration.samples) {
                Assertions.assertEquals(PAPICounter("PAPI_TOT_CYC"), sample.counter)
                Assertions.assertTrue(sample.value > 0)
                Assertions.assertTrue(sample.value < 1000000)
            }
        }
    }

    // We now test getting the number of cycles per operation type
    @ParameterizedTest(name = "Test the gathering of cycles works.")
    @ValueSource(ints = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
    fun testCyclesPerOpType(seed : Long) {
        val benches = listOf(
            BCBenchmarkPackage.MAP(seed, 20, 0, HashMap<Int, Int>())
        )

        val counters = FeatureSet(PAPICounter("PAPI_TOT_CYC"),
            PAPICounter("PAPI_TOT_INS"),
            PAPICounter("PAPI_L1_DCM"))
        val iterations = runner!!.createRunSpecs(10, counters, benches)

        val samples = iterations.flatMap {
            runner!!.runIterations(it)
        }

        val cycles = samples.map { runner!!.getCyclesPerOpType(it) }

        val expectedCounters = OperationType.values().toSet()

        Assertions.assertEquals(10, cycles.size)

        Assertions.assertEquals(expectedCounters, cycles.flatMap { it.keys }.toSet())

    }

    @ParameterizedTest(name = "Test we only run the benchmark once when getting the counters (seed = {0})")
    @ValueSource(ints = [100, 101, 102, 103, 104, 105, 106])
    fun testBenchmarkRunsOnce(seed : Long) {
        val bench = BCBenchmarkPackage.SET(seed, 100, 0, HashSet<Int>())

        val results = runner!!.runApplications(10, FeatureSet(), listOf(bench))

        // We reset numberRuns for each iteration, so the benchmarks is run "once".
        for (r in results) {
            Assertions.assertEquals(1, r.benchmark.numberRuns)
        }
    }

    @Test
    fun testPerformance() {
        // TODO: Do the same, but create a lot of benchmarks!
        val bench = BCBenchmarkPackage.LIST(12345, 1000, 0, ArrayList<Int>())

        val features = FeatureSet(PAPICounter("PAPI_TOT_CYC"),
            PAPICounter("PAPI_TOT_INS"))

        val times = mutableListOf<Double>()
        val max = 100
        for (i in 0..max) {
            val timeSpent = measureTimeMillis {
                runner!!.runIterations(
                    PapiBenchmarkAnalyzer.RunSpec(10, features, bench)
                )
            }
            times.add(timeSpent / 1000.0)
        }

        val reg = SimpleRegression()
        for (i in 0..max) {
            reg.addData(i.toDouble(), times[i])
        }

        Assertions.assertEquals(0.0, reg.slope, 0.01)

    }
}