Commit 8ed8f1b8 authored by Noric Couderc's avatar Noric Couderc
Browse files

Refactoring: Experiment takes a PapiRunner as param

I removed the parameters numRuns and counters from the constructor of
PapiRunner, this parameter is passed in the specs instead and as a
parameter of the specific functions that need it.

This is so I can pass a PapiRunner without the number of runs to an
Experiment and run it. Otherwise you might have to check the papiRunner
has the same number of runs than the Experiment...
parent 875f1a56
package se.lth.cs
import se.lth.cs.bcgen.BCBenchmarkPackage
import se.lth.cs.papicounters.PAPICounter
import se.lth.cs.papicounters.PapiBenchmarkAnalyzer
import se.lth.cs.timing.OperationTypeTable
import se.lth.cs.timing.StopWatchRunner
import se.lth.cs.util.Experiment
import java.io.Writer
import java.util.*
class SyntheticBenchmarkFeaturePrinter(out: Writer,
papiRunner: PapiRunner,
methodPrintFormat: String,
normalizeFeatures: Boolean)
normalizeFeatures: Boolean,
val numberRuns : Int,
val papiCounters : List<PAPICounter>)
: SyntheticBenchmarkDataPrinter(out, methodPrintFormat) {
private val papiRunner: PapiRunner = papiRunner
......@@ -90,10 +93,10 @@ class SyntheticBenchmarkFeaturePrinter(out: Writer,
private fun runBenchmarks(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) {
benchmarkResults = when (normalizeFeatures) {
true -> papiRunner.runApplicationsNormalized(syntheticBenchmarks).groupBy {
true -> papiRunner.runApplicationsNormalized(numberRuns, papiCounters, syntheticBenchmarks).groupBy {
it.benchmark.benchmarkIdentifier
}
false -> papiRunner.runApplications(syntheticBenchmarks).groupBy {
false -> papiRunner.runApplications(numberRuns, papiCounters, syntheticBenchmarks).groupBy {
it.benchmark.benchmarkIdentifier
}
}
......
......@@ -6,7 +6,8 @@ import se.lth.cs.timing.OperationType
import se.lth.cs.util.Feature
interface PapiBenchmarkAnalyzer {
class RunSpec(val features : List<Feature>,
class RunSpec(val numberRuns : Int,
val features : List<Feature>,
val eventSet: EventSet,
val syntheticBenchmark: BCBenchmarkPackage<*>) {
......@@ -52,5 +53,7 @@ interface PapiBenchmarkAnalyzer {
fun runSpec(spec : RunSpec) : List<BenchmarkRunData>
fun runApplications(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<BenchmarkRunData>
fun runApplications(numberRuns: Int,
papiCounters : List<PAPICounter>,
syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<BenchmarkRunData>
}
\ No newline at end of file
......@@ -9,7 +9,7 @@ import se.lth.cs.papicounters.PapiBenchmarkAnalyzer
import se.lth.cs.timing.OperationType
import java.io.FileWriter
open class PapiRunner(val numRuns: Int, val counters: List<PAPICounter>) : PapiBenchmarkAnalyzer {
open class PapiRunner() : PapiBenchmarkAnalyzer {
init {
Papi.init()
......@@ -21,7 +21,7 @@ open class PapiRunner(val numRuns: Int, val counters: List<PAPICounter>) : PapiB
* Empty benchmark:
* Test to see if the results are stable.
*/
fun emptyBenchmark(): Map<PAPICounter, List<Long>> {
fun emptyBenchmark(counters : List<PAPICounter>): Map<PAPICounter, List<Long>> {
// For each counter,
// we store the values for each run (10 runs)
var data: MutableMap<PAPICounter, List<Long>> = mutableMapOf()
......@@ -57,14 +57,15 @@ open class PapiRunner(val numRuns: Int, val counters: List<PAPICounter>) : PapiB
/**
* Creates a list of specifications of what counter to get from what benchmarks
*/
fun createRunSpecs(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<PapiBenchmarkAnalyzer.RunSpec> {
fun createRunSpecs(numRuns: Int, counters: List<PAPICounter>,
syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<PapiBenchmarkAnalyzer.RunSpec> {
// Let's say we want to minimize the counter change
// We put it in the outer loop
val specification = mutableListOf<PapiBenchmarkAnalyzer.RunSpec>()
val counterValues = counters.map { it.toPAPIConstant()!! }
val eventSet = EventSet.create(*counterValues.toIntArray())
for (b in syntheticBenchmarks) {
specification.add(PapiBenchmarkAnalyzer.RunSpec(counters, eventSet, b))
specification.add(PapiBenchmarkAnalyzer.RunSpec(numRuns, counters, eventSet, b))
}
return specification.toList()
}
......@@ -74,7 +75,7 @@ open class PapiRunner(val numRuns: Int, val counters: List<PAPICounter>) : PapiB
val writer = FileWriter("/dev/null")
val samples = mutableListOf<PapiBenchmarkAnalyzer.BenchmarkRunData>()
for (i in 0 until numRuns) {
for (i in 0 until spec.numberRuns) {
val app = spec.syntheticBenchmark
val evset = spec.eventSet
// We do the measurements
......@@ -104,59 +105,11 @@ open class PapiRunner(val numRuns: Int, val counters: List<PAPICounter>) : PapiB
return samples
}
/**
* A function for running one benchmark, gathering all the hardware counters
* Present in the counter specification we have.
*/
@Deprecated("Used only in tests??")
fun runApplication(app: BCBenchmarkPackage<*>) : Map<PAPICounter, List<Long>> {
// We will write the aggregators to nowhere
val writer = FileWriter("/dev/null")
// We get integers for all the counters.
var result = mutableMapOf<PAPICounter, List<Long>>()
for (counter in counters) {
val counterId = counter.toPAPIConstant()!!
val evset = EventSet.create(counterId)
val samples = mutableListOf<Long>()
for (run in 0 until numRuns) {
// We do the measurements
app.reset(app.getDatastructureSize())
var accumulator = 0
evset.start()
app.runBenchmark(blackhole)
evset.stop()
System.out.println("missed trace methods count = " + app.getMissedMethods())
if (app.getMissedMethods() > 0) {
System.out.println("[XWARN:SIMMISS]: missed "
+ Integer.toString(app.getMissedMethods())
+ " out of "
+ Integer.toString(app.getTrace().size) + " trace methods")
}
val resultDataStructure = app.getDatastructure()
// We reset the benchmark
app.reset(0)
// Write the result somewhere to prevent dead code elimination
writer.write(accumulator)
writer.write(resultDataStructure.toString())
// We record the data
samples.addAll(evset.counters.toList())
}
result[counter] = ArrayList(samples)
samples.clear()
}
app.clearAll();
writer.close()
return result.toMap()
}
override fun runApplications(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
override fun runApplications(numberRuns: Int,
papiCounters: List<PAPICounter>,
syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
// return syntheticBenchmarks.map { b -> runApplication(iterations, b) }
val specs = createRunSpecs(syntheticBenchmarks)
val specs = createRunSpecs(numberRuns, papiCounters, syntheticBenchmarks)
val numberBenchmarks = specs.size
var i = 0
val samples = specs.flatMap {
......@@ -169,8 +122,9 @@ open class PapiRunner(val numRuns: Int, val counters: List<PAPICounter>) : PapiB
return samples
}
fun runApplicationsNormalized(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
val benchmarkRuns = runApplications(syntheticBenchmarks)
fun runApplicationsNormalized(numberRuns : Int, papiCounters : List<PAPICounter>,
syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
val benchmarkRuns = runApplications(numberRuns, papiCounters, syntheticBenchmarks)
return benchmarkRuns.map {
it.normalized()
......@@ -193,7 +147,7 @@ open class PapiRunner(val numRuns: Int, val counters: List<PAPICounter>) : PapiB
/**
* A mockup class which returns deterministic results when you "run" a spec.
*/
class MockupPapiRunner(numRuns : Int, counters : List<PAPICounter>) : PapiRunner(numRuns, counters) {
class MockupPapiRunner() : PapiRunner() {
override fun runSpec(spec : PapiBenchmarkAnalyzer.RunSpec) : List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
// We map the counters to integers
// val counterToInt = mapOf<String, Long>(
......@@ -203,7 +157,7 @@ class MockupPapiRunner(numRuns : Int, counters : List<PAPICounter>) : PapiRunner
// Pair("PAPI_TOT_INS", 3)
// )
// val n = seed + counterToInt[spec.counter]!!
val iterations = 0 until numRuns
val iterations = 0 until spec.numberRuns
return iterations.map {
val samples = spec.counters().map {
PapiBenchmarkAnalyzer.BenchmarkRunSample(it, 10000.0)
......
......@@ -3,19 +3,15 @@ package se.lth.cs.papicounters
import com.google.common.graph.MutableValueGraph
import com.google.common.graph.ValueGraph
import com.google.common.graph.ValueGraphBuilder
import papi.Constants
import papi.EventSet
import se.lth.cs.CounterSpecification
import se.lth.cs.PapiRunner
import se.lth.cs.bcgen.BCBenchmarkPackage
import se.lth.cs.methodNameTable2
import se.lth.cs.smartmodules.tracer.Tracer
import se.lth.cs.timing.CollectionTimer
import se.lth.cs.timing.OperationType
import se.lth.cs.timing.OperationTypeTable
import se.lth.util.*
import java.io.ByteArrayOutputStream
import java.io.PrintStream
import java.lang.RuntimeException
/**
* This is a class that works exactly like the PAPI Runner, except it uses
......@@ -23,7 +19,7 @@ import java.lang.RuntimeException
* The tracing collections allow to get more information (breakdown of the features per each method
* instead of each run)
*/
class PapiTracerRunner(numRuns : Int, counters : List<PAPICounter>) : PapiRunner(numRuns, counters) {
open class PapiTracerRunner() : PapiRunner() {
private fun createTracerCollection(bench: BCBenchmarkPackage<*>): Any? {
// Sometimes we used a tracer collection, we want to replace it anyway.
......@@ -110,14 +106,14 @@ class PapiTracerRunner(numRuns : Int, counters : List<PAPICounter>) : PapiRunner
// Shouldn't happen with more elements
}
val rangeIterations = 0 until numRuns
val rangeIterations = 0 until spec.numberRuns
return rangeIterations.map {
runIteration(spec.syntheticBenchmark)
}
}
private fun runIteration(syntheticBenchmark: BCBenchmarkPackage<*>): List<TraceRecord> {
open fun runIteration(syntheticBenchmark: BCBenchmarkPackage<*>): List<TraceRecord> {
Tracer.reset()
Tracer.JPAPI = true
......@@ -144,6 +140,7 @@ class PapiTracerRunner(numRuns : Int, counters : List<PAPICounter>) : PapiRunner
val iterations = runIterations(spec)
// Some post-processing
var iterationIndex = 0
iterations.forEach {
// Analyze text for this iteration
......
......@@ -9,7 +9,8 @@ import se.lth.cs.timing.OperationTypeTable
class Experiment(val numberIterations : Int,
val benchmark : BCBenchmarkPackage<*>,
val features : FeatureSet) : FeatureVisitor {
val features : FeatureSet,
val papiRunner : PapiTracerRunner) : FeatureVisitor {
data class Result(val iterationNumber : Int,
val benchmark : BCBenchmarkPackage<*>,
......@@ -22,13 +23,14 @@ class Experiment(val numberIterations : Int,
eventSetBuilder.visitFeatureSet(features)
val counters = eventSetBuilder.getCounterSet()
val papiRunner = PapiTracerRunner(numberIterations, counters.toList())
val evSet = eventSetBuilder.getEventSet()
val iterations = papiRunner.runIterations(
PapiBenchmarkAnalyzer.RunSpec(counters.toList(), evSet, benchmark)
PapiBenchmarkAnalyzer.RunSpec(numberIterations, counters.toList(), evSet, benchmark)
)
evSet.destroy()
var iterationNumber = 0
val results = mutableListOf<Result>()
for (i in iterations) {
......
......@@ -27,13 +27,17 @@ public class PapiRunnerTest {
CounterSpecification specification = null;
PapiRunner runner = null;
List<PAPICounter> counters = listOf(new PAPICounter("PAPI_L1_DCM"),
new PAPICounter("PAPI_TOT_CYC"),
new PAPICounter("PAPI_BR_MSP"));
int numberRuns = 10;
@BeforeEach
public void setup() {
File papiAvailableCounters = new File("../papi_avail");
Assertions.assertTrue(papiAvailableCounters.exists());
runner = new PapiRunner(10, listOf(new PAPICounter("PAPI_L1_DCM"),
new PAPICounter("PAPI_TOT_CYC"),
new PAPICounter("PAPI_BR_MSP")));
runner = new PapiRunner();
}
/**
* Tests that all performance counters can be put in an eventSet
......@@ -84,14 +88,14 @@ public class PapiRunnerTest {
new PAPICounter("PAPI_L1_DCM"),
new PAPICounter("PAPI_TOT_CYC"));
List<PapiRunner.BenchmarkRunData> data = runner.runApplications(syntheticBenchmarks);
List<PapiRunner.BenchmarkRunData> data = runner.runApplications(numberRuns, counters, syntheticBenchmarks);
// We check all known Papi counters are in the map
Assertions.assertFalse(data.isEmpty());
for (PapiBenchmarkAnalyzer.BenchmarkRunData runData : data) {
Assertions.assertTrue(syntheticBenchmarks.contains(runData.getBenchmark()));
Assertions.assertTrue(runData.getIteration() < runner.getNumRuns());
Assertions.assertTrue(runData.getIteration() < numberRuns);
for (PapiBenchmarkAnalyzer.BenchmarkRunSample s : runData.getSamples()) {
Assertions.assertTrue(counters.contains(s.getCounter()));
Assertions.assertTrue(s.getValue() > 0);
......@@ -100,17 +104,6 @@ public class PapiRunnerTest {
}
}
@Test
public void testPapiRunBenchmark() throws PapiException {
BCBenchmarkPackage<Map<Object, Object>> bench = BCBenchmarkPackage.MAP(0, 100, 0, new HashMap());
Map<PAPICounter, List<Long>> results = runner.runApplication(bench);
Assertions.assertFalse(results.isEmpty());
for (PAPICounter c : results.keySet()) {
Assertions.assertFalse(results.get(c).isEmpty());
}
}
@Test
public void testRunManyBenchmarks() {
// We want to test our setup can handle many benchmarks without crashing.
......@@ -121,14 +114,14 @@ public class PapiRunnerTest {
benchmarks.add(bench);
}
List<PapiRunner.BenchmarkRunData> results = runner.runApplications(benchmarks);
List<PapiRunner.BenchmarkRunData> results = runner.runApplications(numberRuns, counters, benchmarks);
Assertions.assertFalse(results.isEmpty());
}
@Test
public void testEmptyBenchmark() throws PapiException {
runner.emptyBenchmark();
runner.emptyBenchmark(counters);
}
@Test
......@@ -157,7 +150,9 @@ public class PapiRunnerTest {
);
}
List<PapiBenchmarkAnalyzer.RunSpec> specs = runner.createRunSpecs(benchmarks);
List<PapiBenchmarkAnalyzer.RunSpec> specs = runner.createRunSpecs(numberRuns,
counters,
benchmarks);
for(BCBenchmarkPackage<?> b : benchmarks) {
// Each spec has one benchmark, each benchmark has one spec
......@@ -169,7 +164,7 @@ public class PapiRunnerTest {
for (PapiBenchmarkAnalyzer.RunSpec r : specs) {
Assertions.assertEquals(
runner.getCounters(),
counters,
r.counters()
);
......@@ -182,7 +177,9 @@ public class PapiRunnerTest {
public void testRunBenchOnce(int seed) {
BCBenchmarkPackage<List<Object>> bench = BCBenchmarkPackage.LIST(seed, 100, 0, new ArrayList<>());
List<PapiBenchmarkAnalyzer.BenchmarkRunData> results = runner.runApplications(listOf(bench));
List<PapiBenchmarkAnalyzer.BenchmarkRunData> results = runner.runApplications(numberRuns,
counters,
listOf(bench));
for (PapiBenchmarkAnalyzer.BenchmarkRunData r : results) {
Assertions.assertEquals(1, r.getBenchmark().numberRuns);
......
......@@ -2,17 +2,21 @@ import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Test
import se.lth.cs.bcgen.BCBenchmarkPackage
import se.lth.cs.papicounters.MockupPapiTracerRunner
import se.lth.cs.papicounters.PAPICounter
import se.lth.cs.papicounters.PapiTracerRunner
import se.lth.cs.timing.OperationType
import se.lth.cs.util.*
class ExperimentTest {
val runner = MockupPapiTracerRunner()
@Test
fun testExperimentNoFeatures() {
val bench = BCBenchmarkPackage.LIST(1234, 100, 0, ArrayList<Int>())
val exp = Experiment(10, bench, FeatureSet(listOf()))
val exp = Experiment(10, bench, FeatureSet(listOf()), runner)
val results = exp.run()
Assertions.assertTrue(results.isEmpty())
......@@ -24,7 +28,8 @@ class ExperimentTest {
val exp = Experiment(10, bench,
FeatureSet(
listOf(PAPICounter("PAPI_TOT_CYC"))
)
),
runner
)
val results = exp.run()
......@@ -60,7 +65,8 @@ class ExperimentTest {
costPerDelete,
cacheMissRate
)
)
),
runner
)
val results = exp.run()
......@@ -79,7 +85,7 @@ class ExperimentTest {
fun testExperimentBrainyFeatures() {
val bench = BCBenchmarkPackage.LIST(1234, 100, 0, ArrayList<Int>())
val exp = Experiment(10, bench, brainyFeatures())
val exp = Experiment(10, bench, brainyFeatures(), runner)
val results = exp.run()
Assertions.assertEquals(listOf<Map<Feature, Double>>(), results)
......
......@@ -27,7 +27,9 @@ class SyntheticBenchmarkFeaturePrinterTest {
val methodOutputFormat = "JAVA-STANDARD-FORMAT"
val normalizeFeatures = false
val papiRunner = MockupPapiRunner( 3, counters)
val numRuns = 3
val papiRunner = MockupPapiRunner()
@Test
fun testEmptyBenchmark() {
......@@ -36,7 +38,9 @@ class SyntheticBenchmarkFeaturePrinterTest {
val writer = StringWriter()
val csvPrinter = SyntheticBenchmarkFeaturePrinter(writer, papiRunner,
methodOutputFormat, normalizeFeatures)
methodOutputFormat, normalizeFeatures,
numRuns,
counters)
val toPrint : List<BCBenchmarkPackage<*>> = listOf<BCBenchmarkPackage<*>>(benchmark)
csvPrinter.printToCSV(toPrint)
......@@ -91,7 +95,7 @@ class SyntheticBenchmarkFeaturePrinterTest {
val writer = StringWriter()
val csvPrinter = SyntheticBenchmarkFeaturePrinter(writer, papiRunner,
"SYNTHETIC-RUN-FORMAT", normalizeFeatures)
"SYNTHETIC-RUN-FORMAT", normalizeFeatures, numRuns, counters)
csvPrinter.printToCSV(listOf<BCBenchmarkPackage<*>>(benchmark))
......@@ -163,7 +167,7 @@ class SyntheticBenchmarkFeaturePrinterTest {
val csvPrinter = SyntheticBenchmarkFeaturePrinter(writer,
papiRunner,
"SYNTHETIC-RUN-FORMAT",
true)
true, numRuns, counters)
val expectedText =
"""
......@@ -220,7 +224,7 @@ class SyntheticBenchmarkFeaturePrinterTest {
val writer = StringWriter()
val csvPrinter = SyntheticBenchmarkFeaturePrinter(writer, papiRunner,
"SYNTHETIC-RUN-FORMAT",
false)
false, numRuns, counters)
csvPrinter.printToCSV(benchmarks)
......@@ -352,9 +356,9 @@ class SyntheticBenchmarkFeaturePrinterTest {
val sw = StringWriter()
val syntheticBenchmarkFeaturePrinter = SyntheticBenchmarkFeaturePrinter(sw,
PapiRunner(100, counters),
PapiRunner(),
methodOutputFormat,
true)
true, 100, counters)
syntheticBenchmarkFeaturePrinter.printToCSV(listOf(benchmark))
......
......@@ -122,8 +122,7 @@ class TracingCollectionRunnerTest {
fun setup() {
val papiAvailableCounters = File("../papi_avail")
Assertions.assertTrue(papiAvailableCounters.exists())
val counters = listOf("PAPI_TOT_CYC", "PAPI_TOT_INS", "PAPI_L1_DCM").map { PAPICounter(it) }
runner = PapiTracerRunner(10, counters)
runner = PapiTracerRunner()
}
@Test
......@@ -132,17 +131,18 @@ class TracingCollectionRunnerTest {
BCBenchmarkPackage.LIST(10, 10, 0, ArrayList<Int>())
)
val res = runner!!.runApplications(benches)
val counters = listOf("PAPI_TOT_CYC", "PAPI_TOT_INS", "PAPI_L1_DCM").map { PAPICounter(it) }
val res = runner!!.runApplications(10, counters, benches)
// We only tried one benchmark, but we should get several counters.
// One for each number
Assertions.assertEquals(runner!!.numRuns, res.size)
Assertions.assertEquals(10, res.size)
for (t in res) {
Assertions.assertEquals(benches.get(0), t.benchmark)
Assertions.assertTrue(t.iteration < runner!!.numRuns)
Assertions.assertTrue(t.iteration < 10)
Assertions.assertEquals(
runner!!.counters.toSet() ,
counters.toSet() ,
t.samples.map { it.counter }.toSet())
}
}
......@@ -153,6 +153,7 @@ class TracingCollectionRunnerTest {
val bench = BCBenchmarkPackage.LIST(seed, 10, 0, ArrayList<Int>())
val spec = PapiBenchmarkAnalyzer.RunSpec(
10,
listOf(PAPICounter("PAPI_TOT_CYC")),
EventSet.create(Constants.PAPI_TOT_CYC),
bench)
......@@ -179,7 +180,8 @@ class TracingCollectionRunnerTest {
BCBenchmarkPackage.MAP(seed, 20, 0, HashMap<Int, Int>())
)
val iterations = runner!!.createRunSpecs(benches)
val counters = listOf("PAPI_TOT_CYC", "PAPI_TOT_INS", "PAPI_L1_DCM").map { PAPICounter(it) }
val iterations = runner!!.createRunSpecs(10, counters, benches)
val samples = iterations.flatMap {
runner!!.runIterations(it)
......@@ -189,7 +191,7 @@ class TracingCollectionRunnerTest {
val expectedCounters = OperationType.values().toSet()
Assertions.assertEquals(runner!!.numRuns, cycles.size)
Assertions.assertEquals(10, cycles.size)
Assertions.assertEquals(expectedCounters, cycles.flatMap { it.keys }.toSet())
......@@ -200,7 +202,7 @@ class TracingCollectionRunnerTest {
fun testBenchmarkRunsOnce(seed : Long) {
val bench = BCBenchmarkPackage.SET(seed, 100, 0, HashSet<Int>())
val results = runner!!.runApplications(listOf(bench))
val results = runner!!.runApplications(10, listOf(), listOf(bench))
for (r in results) {
Assertions.assertEquals(1, r.benchmark.numberRuns)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment