Commit 1ca5ea1e authored by Noric Couderc's avatar Noric Couderc
Browse files

Made PapiRunner gather all counters in one pass

The papiRunner class now gathers all the counters in one single run.
For that, we need to gather a limited number of counters: PAPI_TOT_CYC +
PAPI_TOT_INS + max 2 counters.

Because we don't do any aggregation either, I needed to change the
functions so they return a list of values for each iteration.
parent 35877094
......@@ -2,6 +2,7 @@ package se.lth.cs.timing;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
public class OperationTypeTable {
......@@ -11,8 +12,12 @@ public class OperationTypeTable {
initialize();
}
public static OperationType getType(String method) {
return operationTypeMap.getOrDefault(method, null);
public static Optional<OperationType> getType(String method) {
if (operationTypeMap.containsKey(method)) {
return Optional.of(operationTypeMap.get(method));
} else {
return Optional.empty();
}
}
public static String getOperationTypeFeatureName(OperationType op) {
......
......@@ -13,6 +13,10 @@ class CounterSpecification(strings : List<String>) {
return currentSpec[name]
}
fun getCounterNames() : List<String> {
return currentSpec.keys.toList()
}
fun getCounterValues() : List<Int> {
return currentSpec.values.toList()
}
......
......@@ -39,20 +39,13 @@ class SyntheticBenchmarkFeaturePrinter(out: Writer,
val runResults = benchmarkResults[identifier]!!.sortedBy { it.iteration }
// For some features, we don't want to print them once for each counter
// We just want to print them once
val selectedCounter = runResults.map { it.counter }.first()
// For the first record:
for (r in runResults) {
if (r.counter == selectedCounter) {
val collection = syntheticBenchmark.dataStructureSimpleName
printer.printRecord(identifier, r.iteration, "collection", "collection", collection)
printSoftwareCounters(r)
printStopWatchCounters(r)
}
val collection = syntheticBenchmark.dataStructureSimpleName
printer.printRecord(identifier, r.iteration, "collection", "collection", collection)
printSoftwareCounters(r)
printStopWatchCounters(r)
printHardwareCounters(r)
}
}
......@@ -72,11 +65,13 @@ class SyntheticBenchmarkFeaturePrinter(out: Writer,
}
private fun printHardwareCounters(runData : PapiBenchmarkAnalyzer.BenchmarkRunData) {
printer.printRecord(runData.benchmark.benchmarkIdentifier,
runData.iteration,
runData.counter,
"hardware",
runData.value)
for (sample in runData.samples) {
printer.printRecord(runData.benchmark.benchmarkIdentifier,
runData.iteration,
sample.counter,
"hardware",
sample.value)
}
}
private fun printSoftwareCounters(runData : PapiBenchmarkAnalyzer.BenchmarkRunData) {
......
......@@ -4,17 +4,51 @@ import papi.EventSet
import se.lth.cs.bcgen.BCBenchmarkPackage
interface PapiBenchmarkAnalyzer {
data class RunSpec(val numRuns: Int,
val counter : String,
val eventSet: EventSet,
val syntheticBenchmark: BCBenchmarkPackage<*>)
class RunSpec(val numRuns: Int,
val counters : List<String>,
val eventSet: EventSet,
val syntheticBenchmark: BCBenchmarkPackage<*>) {
data class BenchmarkRunData(val benchmark : BCBenchmarkPackage<*>,
val iteration : Int,
val counter : String,
val value : Double)
fun specialCounters(): List<String> {
return listOf<String>("PAPI_TOT_CYC", "PAPI_TOT_INS")
}
fun runSpec(spec : RunSpec) : List<Long>
private fun isSpecial(counter : String) : Boolean {
return specialCounters().contains(counter)
}
/**
Tells you if a RunSpec can be sampled in one single pass
for that, the counters have to be 2 counters + (optionally) PAPI_TOT_CYC and PAPI_TOT_INS
*/
fun canBeSampled(): Boolean {
return counters.filter { ! this.isSpecial(it) }.size <= 2
}
}
data class BenchmarkRunSample(val counter : String, var value : Double)
class BenchmarkRunData(val benchmark : BCBenchmarkPackage<*>,
val iteration : Int,
val samples : List<BenchmarkRunSample>) {
fun normalizationCounter(): String {
return "PAPI_TOT_CYC"
}
fun normalized(): BenchmarkRunData {
val cycles = samples.find { it.counter == normalizationCounter() }!!.value
val newSamples = this.samples.map {
BenchmarkRunSample(it.counter, it.value / cycles)
}
return BenchmarkRunData(this.benchmark, this.iteration, newSamples)
}
}
fun runSpec(spec : RunSpec) : List<List<BenchmarkRunSample>>
fun runApplications(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<BenchmarkRunData>
}
\ No newline at end of file
......@@ -58,24 +58,22 @@ open class PapiRunner(val numRuns: Int, counters: CounterSpecification) : PapiBe
* Creates a list of specifications of what counter to get from what benchmarks
*/
fun createRunSpecs(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<PapiBenchmarkAnalyzer.RunSpec> {
val counters = counterSpec.currentSpec
// Let's say we want to minimize the counter change
// We put it in the outer loop
val specification = mutableListOf<PapiBenchmarkAnalyzer.RunSpec>()
for (c in counters) {
val eventSet = EventSet.create(c.value)
for (b in syntheticBenchmarks) {
specification.add(PapiBenchmarkAnalyzer.RunSpec(numRuns, c.key, eventSet, b))
}
val counterNames = counterSpec.getCounterNames()
val counterValues = counterSpec.getCounterValues()
val eventSet = EventSet.create(*counterValues.toIntArray())
for (b in syntheticBenchmarks) {
specification.add(PapiBenchmarkAnalyzer.RunSpec(numRuns, counterNames, eventSet, b))
}
return specification.toList()
}
override fun runSpec(spec : PapiBenchmarkAnalyzer.RunSpec): List<Long> {
override fun runSpec(spec : PapiBenchmarkAnalyzer.RunSpec): List<List<PapiBenchmarkAnalyzer.BenchmarkRunSample>> {
// println("Running benchmark: " + spec.syntheticBenchmark.benchmarkIdentifier)
val writer = FileWriter("/dev/null")
val samples = mutableListOf<Long>()
val samples = mutableListOf<List<PapiBenchmarkAnalyzer.BenchmarkRunSample>>()
for (i in 0 until spec.numRuns) {
val app = spec.syntheticBenchmark
val evset = spec.eventSet
......@@ -94,7 +92,10 @@ open class PapiRunner(val numRuns: Int, counters: CounterSpecification) : PapiBe
writer.write(accumulator)
writer.write(resultDataStructure.toString())
// We record the data
samples.addAll(evset.counters.toList())
val runData = spec.counters.zip(evset.counters.toList()).map {
PapiBenchmarkAnalyzer.BenchmarkRunSample(it.first, it.second.toDouble())
}
samples.add(runData)
}
writer.close()
return samples
......@@ -160,17 +161,18 @@ open class PapiRunner(val numRuns: Int, counters: CounterSpecification) : PapiBe
print("Running spec: $i / $numberBenchmarks\r")
i++
runSpec(it) }
val spectToCounters = specs.zip(samples)
val spectToSamples = specs.zip(samples)
println("Finished running specs.")
// We create a report for a run for each individual sample
return spectToCounters.flatMap {
return spectToSamples.flatMap {
it.second.mapIndexed { index, sample ->
PapiBenchmarkAnalyzer.BenchmarkRunData(it.first.syntheticBenchmark,
PapiBenchmarkAnalyzer.BenchmarkRunData(
it.first.syntheticBenchmark,
index,
it.first.counter,
sample.toDouble())
sample)
}
}
......@@ -178,17 +180,9 @@ open class PapiRunner(val numRuns: Int, counters: CounterSpecification) : PapiBe
fun runApplicationsNormalized(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<PapiBenchmarkAnalyzer.BenchmarkRunData> {
val benchmarkRuns = runApplications(syntheticBenchmarks)
val instructionsForRun = benchmarkRuns.filter { it.counter == "PAPI_TOT_INS" }
.associate {
val k = Pair(it.benchmark, it.iteration)
val v = it.value
Pair(k, v)
}
return benchmarkRuns.map {
val k = Pair(it.benchmark, it.iteration)
val numberInstructions = instructionsForRun[k]
PapiBenchmarkAnalyzer.BenchmarkRunData(it.benchmark, it.iteration, it.counter, it.value / numberInstructions!!)
it.normalized()
}
}
......@@ -209,7 +203,7 @@ open class PapiRunner(val numRuns: Int, counters: CounterSpecification) : PapiBe
* A mockup class which returns deterministic results when you "run" a spec.
*/
class MockupPapiRunner(numRuns : Int, counters : CounterSpecification) : PapiRunner(numRuns, counters) {
override fun runSpec(spec : PapiBenchmarkAnalyzer.RunSpec) : List<Long> {
override fun runSpec(spec : PapiBenchmarkAnalyzer.RunSpec) : List<List<PapiBenchmarkAnalyzer.BenchmarkRunSample>> {
val seed = spec.syntheticBenchmark.seed
// We map the counters to integers
// val counterToInt = mapOf<String, Long>(
......@@ -220,6 +214,11 @@ class MockupPapiRunner(numRuns : Int, counters : CounterSpecification) : PapiRun
// )
// val n = seed + counterToInt[spec.counter]!!
val n = seed
return listOf(n, n, n)
val iterations = 0 until spec.numRuns
return iterations.map {
spec.counters.map {
PapiBenchmarkAnalyzer.BenchmarkRunSample(it, n.toDouble())
}
}
}
}
......@@ -15,6 +15,7 @@ import se.lth.cs.timing.OperationTypeTable
import se.lth.util.*
import java.io.ByteArrayOutputStream
import java.io.PrintStream
import java.lang.RuntimeException
/**
* This is a class that works exactly like the PAPI Runner, except it uses
......@@ -43,35 +44,69 @@ class PapiTracerRunner(numRuns : Int, counters : CounterSpecification) : PapiRun
}
}
private fun analyzeOutput(text : String): ValueGraph<String, Int> {
data class TraceRecord (val allocationSite : String,
val featureName : String,
val featureValue : Int,
val collection : String,
val method : String)
fun makeRecords(text : String): List<TraceRecord> {
val cleanedLines = text.lines().filter { !it.isEmpty() }
return cleanedLines.map { line ->
val splitted = line.split("\t")
TraceRecord(splitted[0],
splitted[1],
Integer.parseInt(splitted[2]),
splitted[3],
splitted[4]
)
}
}
/**
* Analyzes the output of running an iteration, line by line
* returns a graph where for each edge :
* - the source is an allocation site
* - the target is a feature
* - the label is the value of that feature
*/
private fun analyzeOutput(records : List<TraceRecord>): ValueGraph<String, Int> {
// We can analyze the output to get what we want.
val g: MutableValueGraph<String, Int> = ValueGraphBuilder.directed().build()
for (line in text.lines().filter { !it.isEmpty() }) {
val splitted = line.split("\t")
val feature = splitted[1]
val counterValue = Integer.parseInt(splitted[2])
for (r in records) {
val hasZeroInvokes = r.featureName == "invocations" && r.featureValue == 0
val hasZeroInvokes = feature == "invocations" && counterValue == 0
if (hasZeroInvokes ) {
continue
}
val allocSite = splitted[0] // Allocation site / Benchmark ID.
val value = g.edgeValueOrDefault(r.allocationSite, r.featureName, 0)?.plus(r.featureValue) // Number
g.putEdgeValue(r.allocationSite, r.featureName, value)
val value = g.edgeValueOrDefault(allocSite, feature, 0)?.plus(counterValue) // Number
g.putEdgeValue(allocSite, feature, value)
// We also relate methods to their type (insertionCycles, etc)
val methodType = OperationTypeTable.getType(r.method)
if (methodType.isPresent) {
g.putEdgeValue("methodType", methodType.get().toString(), 0)
g.putEdgeValue(r.method, methodType.get().toString(), 0)
}
}
return g
}
private fun runIterations(spec : PapiBenchmarkAnalyzer.RunSpec) : List<String> {
val counter = counterSpec.getCounter(spec.counter)
Tracer.setCounters(counter!!.toInt())
fun runIterations(spec : PapiBenchmarkAnalyzer.RunSpec) : List<List<TraceRecord>> {
val counters = spec.counters
.filter { ! spec.specialCounters().contains(it) } // These are active by default.
.map { counterSpec.getCounter(it)!! }
when (counters.size) {
1 -> Tracer.setCounters(counters.first())
2 -> Tracer.setCounters(counters[0], counters[1])
// Shouldn't happen with more elements
}
val rangeIterations = 0 until spec.numRuns
......@@ -80,7 +115,7 @@ class PapiTracerRunner(numRuns : Int, counters : CounterSpecification) : PapiRun
}
}
private fun runIteration(syntheticBenchmark: BCBenchmarkPackage<*>): String {
private fun runIteration(syntheticBenchmark: BCBenchmarkPackage<*>): List<TraceRecord> {
Tracer.reset()
Tracer.JPAPI = true
......@@ -95,72 +130,76 @@ class PapiTracerRunner(numRuns : Int, counters : CounterSpecification) : PapiRun
// We get the output from the run
val stream = ByteArrayOutputStream()
Tracer.printRecords(PrintStream(stream))
return stream.toString()
return makeRecords(stream.toString())
}
override fun runSpec(spec: PapiBenchmarkAnalyzer.RunSpec): List<Long> {
val result = mutableListOf<Long>()
override fun runSpec(spec: PapiBenchmarkAnalyzer.RunSpec): List<List<PapiBenchmarkAnalyzer.BenchmarkRunSample>> {
if (!spec.canBeSampled()) {
throw RuntimeException("Specification cannot be sampled")
}
val counter = counterSpec.getCounter(spec.counter)
Tracer.setCounters(counter!!.toInt())
val result = mutableListOf<List<PapiBenchmarkAnalyzer.BenchmarkRunSample>>()
val textResults = runIterations(spec)
val iterations = runIterations(spec)
textResults.map {
iterations.forEach {
// Analyze text for this iteration
val g = analyzeOutput(it)
val featureNumber = String.format("0x%x", counterSpec.getCounter(spec.counter))
g.predecessors(featureNumber).forEach {
val value = g.edgeValue(it, featureNumber)
result.add(value.get().toLong())
// Get features present in this iteration.
val counterSamples = spec.counters.map { counter ->
val featureNumber = String.format("0x%x", counterSpec.getCounter(counter))
val allocationSites = g.predecessors(featureNumber)
// There should be only one when we run the benchmarks
assert(allocationSites.size == 1)
val allocationSite = allocationSites.first()
val value = g.edgeValue(allocationSite, featureNumber)
PapiBenchmarkAnalyzer.BenchmarkRunSample(counter, value.get().toDouble())
}
result.add(counterSamples)
}
return result
}
fun getCyclesPerOpType(syntheticBenchmark: BCBenchmarkPackage<*>): List<Triple<Int, String, Long>> {
val runSpec = PapiBenchmarkAnalyzer.RunSpec(numRuns, "PAPI_TOT_CYC",
EventSet.create(Constants.PAPI_TOT_CYC), syntheticBenchmark)
fun getCyclesPerOpType(iterations : List<List<TraceRecord>>): List<Triple<Int, String, Double>> {
var result = mutableListOf<Triple<Int, String, Double>>()
val textResults = runIterations(runSpec)
val siteToCycles = mutableMapOf<Pair<OperationType, String>, Int>()
var runNumber = 0
var g = ValueGraphBuilder.undirected().build<String, Int>()
val featureNumber = String.format("0x%x", counterSpec.getCounter("PAPI_TOT_CYC"))
for (run in textResults) {
// We append to the graph
val runId = runNumber.toString()
for (line in run.lineSequence().filter { it.isNotEmpty() }) {
val splitted = line.split("\t")
val method = splitted[4]
val methodType = OperationTypeTable.getType(method)
val numberCycles = Integer.parseInt(splitted[2])
if (methodType != null) {
val featureLabel = OperationTypeTable.getOperationTypeFeatureName(methodType)
g.putEdgeValue(method, featureLabel, 0)
g.putEdgeValue("methodType", featureLabel, 0)
// We need to total number of cycles for each alloc site / operation type
for (iteration in iterations) {
for (record in iteration) {
if (featureNumber == record.featureName) {
val opType = OperationTypeTable.getType(record.method)
if (opType.isPresent) {
val key = Pair(opType.get(), record.allocationSite)
val cyclesForOpType = siteToCycles.getOrDefault(key, 0)
siteToCycles.put(key, cyclesForOpType + record.featureValue)
}
}
g.putEdgeValue(runId, method, numberCycles)
// To be able to get all the runs as neighbours quickly
g.putEdgeValue("runs", runId, 0)
}
runNumber++
}
var result = mutableListOf<Triple<Int, String, Long>>()
for (runId in g.successors("runs")) {
for (methodType in g.successors("methodType")) {
// We get the methods for that run and that method type
val methodsOfType = g.successors(runId).intersect(g.predecessors(methodType))
val totalCycles = methodsOfType.sumBy { g.edgeValueOrDefault(runId, it, 0)!! }
val triple = Triple(Integer.parseInt(runId), methodType, totalCycles.toLong())
result.add(triple)
var runId = 0
for (iteration in iterations) {
for (record in iteration) {
if (record.featureName == featureNumber) {
val opType = OperationTypeTable.getType(record.method)
if (opType.isPresent) {
val key = Pair(opType.get(), record.allocationSite)
val featureName = OperationTypeTable.getOperationTypeFeatureName(opType.get())!!
val value = record.featureValue.toDouble() / siteToCycles.get(key)!!
result.add(Triple(runId, featureName, value))
}
}
}
runId++
}
return result
......
......@@ -9,16 +9,16 @@ import se.lth.cs.CounterSpecification;
import se.lth.cs.PapiRunner;
import se.lth.cs.bcgen.*;
import se.lth.cs.SyntheticBenchmarkGeneration.*;
import se.lth.cs.papicounters.PapiBenchmarkAnalyzer;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.function.IntPredicate;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static se.lth.cs.UtilsKt.listOf;
public class PapiRunnerTest {
......@@ -29,8 +29,7 @@ public class PapiRunnerTest {
public void setup() {
File papiAvailableCounters = new File("../papi_avail");
Assertions.assertTrue(papiAvailableCounters.exists());
specification = CounterSpecification.Companion.fromFile(papiAvailableCounters);
runner = new PapiRunner(10, specification);
runner = new PapiRunner(10, new CounterSpecification(listOf("PAPI_TOT_CYC", "PAPI_BR_MSP")));
}
/**
* Tests that all performance counters can be put in an eventSet
......@@ -42,6 +41,7 @@ public class PapiRunnerTest {
public void TestPapiEventSet() throws PapiException {
Papi.init();
specification = CounterSpecification.Companion.fromFile(new File("../papi_avail"));
List<Integer> constants = specification.getCounterValues();
IntPredicate throwsExp =
......@@ -74,15 +74,22 @@ public class PapiRunnerTest {
3,
100
);
List<String> counters = listOf("PAPI_BR_MSP", "PAPI_L1_DCM", "PAPI_TOT_CYC");
List<PapiRunner.BenchmarkRunData> data = runner.runApplications(syntheticBenchmarks);
// We check all known Papi counters are in the map
Assertions.assertFalse(data.isEmpty());
for (PapiRunner.BenchmarkRunData counters : data) {
Assertions.assertTrue(
specification.getCurrentSpec().containsKey(counters.getCounter())
);
Assertions.assertTrue( counters.getValue() >= 0 );
for (PapiBenchmarkAnalyzer.BenchmarkRunData runData : data) {
Assertions.assertTrue(syntheticBenchmarks.contains(runData.getBenchmark()));
Assertions.assertTrue(runData.getIteration() < runner.getNumRuns());
for (PapiBenchmarkAnalyzer.BenchmarkRunSample s : runData.getSamples()) {
Assertions.assertTrue(counters.contains(s.getCounter()));
Assertions.assertTrue(s.getValue() > 0);
Assertions.assertTrue(s.getValue() < 10000000);
}
}
}
......
......@@ -24,6 +24,7 @@ class SyntheticBenchmarkFeaturePrinterTest {
PAPI_TOT_INS
""".trimIndent()
val specReader = StringReader(specString)
val counterSpec = CounterSpecification.fromReader(specReader)
val methodOutputFormat = "JAVA-STANDARD-FORMAT"
......
......@@ -4,6 +4,7 @@ import org.openjdk.jmh.infra.Blackhole
import papi.Constants
import papi.EventSet
import papi.Papi
import se.lth.cs.CounterSpecification
import se.lth.cs.CounterSpecification.Companion.fromFile
import se.lth.cs.bcgen.BCBenchmarkPackage
import se.lth.cs.papicounters.PapiBenchmarkAnalyzer
......@@ -117,7 +118,7 @@ class TracingCollectionRunnerTest {
fun setup() {
val papiAvailableCounters = File("../papi_avail")
Assertions.assertTrue(papiAvailableCounters.exists())
val specification = fromFile(papiAvailableCounters)
val specification = CounterSpecification(listOf("PAPI_TOT_CYC", "PAPI_TOT_INS", "PAPI_L1_DCM"))
runner = PapiTracerRunner(10, specification)
}
......@@ -126,14 +127,19 @@ class TracingCollectionRunnerTest {
val benches = listOf(
BCBenchmarkPackage.LIST(10, 10, 0, ArrayList<Int>())
)
val res = runner!!.runApplications(benches)
// We only tried one benchmark, but we should get several counters.
// One for each number
Assertions.assertEquals(runner!!.counterSpec.currentSpec.size * runner!!.numRuns,
res.size)