Commit 57df4ae3 authored by Noric Couderc's avatar Noric Couderc
Browse files

Refactored PapiRunner

Deleted some messy code, for a cleaner alternative. This might not work
though.
parent 46808e05
......@@ -81,83 +81,50 @@ open class PapiRunner(counters: CounterSpecification) {
}
/** Runs a set of programs (functions) without interleaving
* (Performance should get better if there is JIT compilation)
* @Returns A map from couples (counter, program-name) -> values over all runs
* TODO: Implement this using cartesianProduct from Guava instead
/**
* A function for running one benchmark, gathering all the hardware counters
* Present in the counter specification we have.
*/
open fun runApplications(numRuns: Int, applications: Sequence<SyntheticBenchmark<*>>):
List<MutableMap<String, List<Long>>> {
// We store a map from program names to map with counters and list of values
var list = mutableListOf<MutableMap<String, List<Long>>>()
// We initialize data with empty maps
for (app in applications) {
list.add(mutableMapOf())
}
// For each counter that is available
for (kvp in counterSpec.currentSpec) {
val counterName = kvp.key
println("Streamlined mode: '$counterName'")
val before = Instant.now()
// We record only one counter
val evset = EventSet.create(kvp.value)
// For each program...
var appIndex = 0
for (app in applications) {
if (list.get(appIndex).containsKey(counterName)) {
throw Exception(String.format("'%s' key already exists", counterName))
fun runApplication(numRuns: Int, app: SyntheticBenchmark<*>) : Map<String, List<Long>> {
// We will write the aggregators to nowhere
val writer = FileWriter("/dev/null")
// We get integers for all the counters.
val counters = counterSpec.currentSpec
var result = mutableMapOf<String, List<Long>>()
for (counter in counters.keys) {
val evset = EventSet.create(counters[counter]!!)
val samples = mutableListOf<Long>()
for (run in 0 until numRuns) {
// We do the measurements
app.reset(app.baseDataStructureSize)
var accumulator = 0
evset.start()
while (app.hasNext()) {
val result: Int? = app.invokeCurrentMethod() as? Int
accumulator += result ?: 1
app.tick()
}
list.get(appIndex)[counterName] = runApplication(numRuns, evset, app)
appIndex++
}
val after = Instant.now()
val totalDuration = Duration.between(before, after)
println("Done: ${totalDuration}")
}
return list.toList()
}
fun runApplications(numRuns: Int, syntheticBenchmarks : List<SyntheticBenchmark<*>>):
List<MutableMap<String, List<Long>>> {
return runApplications(numRuns, syntheticBenchmarks.asSequence())
}
private fun runApplication(numRuns: Int, evset: EventSet, app: SyntheticBenchmark<*>) : MutableList<Long> {
// We run it n times
val resultDataStructure = app.getDataStructure()
evset.stop()
val writer = FileWriter("/dev/null")
// We reset the benchmark
app.reset(0)
var values = mutableListOf<Long>()
for (run in 0 until numRuns) {
// We do the measurements
app.reset(app.baseDataStructureSize)
var accumulator = 0
evset.start()
while (app.hasNext()) {
val result : Int? = app.invokeCurrentMethod() as? Int
accumulator += result ?: 1
app.tick()
// Write the result somewhere to prevent dead code elimination
writer.write(accumulator)
writer.write(resultDataStructure.toString())
// We record the data
samples.addAll(evset.counters.toList())
}
val resultDataStructure = app.getDataStructure()
evset.stop()
app.reset(0)
writer.write(accumulator)
writer.write(resultDataStructure.toString())
// We record the data
values.addAll(evset.counters.toList())
result[counter] = ArrayList(samples)
samples.clear()
}
app.clearDataStructure()
writer.close()
return values
return result.toMap()
}
/**
......@@ -172,122 +139,7 @@ open class PapiRunner(counters: CounterSpecification) {
return medianLong(data)
}
/**
* A class for feature vectors with the label of the app (seed), the fastest datastructure for that app,
* and the performance counters for that app
*/
data class FeatureVector(val appLabel: String,
val dataStructure: String,
val bestDataStructure: String,
val counters: Map<String, Double>)
/**
* Runs a couple of generated applications and returns their feature vectors
*/
fun getFeatures(numRuns: Int, syntheticBenchmarks: List<SyntheticBenchmark<*>>):
List<FeatureVector> {
val trainingSet =
SyntheticBenchmarkRunner().runBenchmarks(syntheticBenchmarks).toList()
val distribution = trainingSet.groupBy { it.bestDataStructure }.mapValues { it.value.size }
println("Benchmark distribution : $distribution")
val apps = trainingSet.map { it.syntheticBenchmark }
val counters =
runApplications(numRuns, apps.asSequence()).map {
it.mapValues { medianLong(it.value) }
}
return trainingSet.zip(counters) { v, c ->
FeatureVector(
v.syntheticBenchmark.identifier,
v.dataStructure,
v.bestDataStructure,
c
)
}
}
/**
* Saves a list of feature vectors to a CSV file
* @param vectors The list of feature vectors to save
* @return the text of the file to be saved
*/
fun featuresToCSV(vectors: List<FeatureVector>): String {
if (vectors.isEmpty()) return ""
var header = mutableListOf(
"application",
"data_structure",
"best_data_structure")
val counters = vectors.map { it.counters.keys }
.fold(setOf()) { s: Set<String>, v -> s.union(v) }
header.addAll(counters)
val headerText = header.joinToString(",")
var values = mutableListOf<List<String>>()
for (v in vectors) {
var l = mutableListOf<String>()
l.add(v.appLabel)
l.add(v.dataStructure)
l.add(v.bestDataStructure)
for (c in counters) {
l.add(v.counters[c]?.toString() ?: "None")
}
values.add(l)
}
val valuesTexts = values.map {
it.joinToString(",")
}
val valuesText = valuesTexts.joinToString("\n")
return "$headerText\n$valuesText"
}
/**
* Transforms a sequence of JMH records to a list of feature vectors,
* by re-generating the application
* @param numRuns number of times each app is deemed to be ran
* @param jmhData the sequence of JMH records
* @return return a list of feature vectors
*/
fun processJMHData(numRuns: Int, jmhData: Sequence<JMHProcessor.JMHRecord>): List<FeatureVector> {
val applications = jmhData.map {
it.generateSyntheticBenchmark()!!
}
val appsBest = jmhData.map {
val app = it.generateSyntheticBenchmark()!!
val best = it.best
Pair(app, best)
}
// Map from application to list of counters
val results = runApplications(numRuns, applications)
.asSequence()
return appsBest.zip(results) { appAndBest, counters ->
val aggregates = counters.mapValues { medianLong(it.value) }
FeatureVector(appAndBest.first.seedString,
appAndBest.first.dataStructureName,
appAndBest.second,
aggregates)
}.toList()
fun runApplications(iterations: Int, syntheticBenchmarks: MutableList<SyntheticBenchmark<*>>): List<Map<String, List<Long>>> {
return syntheticBenchmarks.map { b -> runApplication(iterations, b) }
}
/**
* Transforms a list of JMH records to a list of feature vectors,
* by re-generating the application
* @param numRuns number of times each app is deemed to be ran
* @param jmhData the sequence of JMH records
* @return return a list of feature vectors
*/
fun processJMHData(numRuns: Int, jmhData: List<JMHProcessor.JMHRecord>): List<FeatureVector> {
return processJMHData(numRuns, jmhData.asSequence())
}}
}
......@@ -2,12 +2,24 @@ package se.lth.cs
import java.io.Writer
class SyntheticBenchmarkFeaturePrinter(out: Writer, features: List<MutableMap<String, List<Long>>>) : SyntheticBenchmarkDataPrinter(out) {
class SyntheticBenchmarkFeaturePrinter(out: Writer, papiRunner: PapiRunner) : SyntheticBenchmarkDataPrinter(out) {
private val papiRunner: PapiRunner = papiRunner
override fun printHeader() {
// The header contains:
// - all the PAPI Counters
// - software counters (names of methods)
// - names of data structures in the benchmark
TODO("not implemented") //To change body of created functions use File | Settings | File Templates.
}
override fun printBenchmarkRows(syntheticBenchmark: SyntheticBenchmark<*>) {
// Getting the collection name
val collection = syntheticBenchmark.dataStructureSimpleName
// Getting the software perf counters
val softwareCounters = syntheticBenchmark.methodHistogram()
TODO("not implemented") //To change body of created functions use File | Settings | File Templates.
}
}
\ No newline at end of file
......@@ -83,8 +83,7 @@ class PapiCommandLine : CliktCommand() {
SyntheticBenchmarkFeaturePrinter {
val counterSpec = CounterSpecification.fromFile(File("papi_avail"))
val r = PapiRunner(counterSpec)
val features = r.runApplications(numberRuns, benchmarks.asSequence())
return SyntheticBenchmarkFeaturePrinter(writer, features)
return SyntheticBenchmarkFeaturePrinter(writer, r)
}
}
......@@ -9,6 +9,7 @@ import se.lth.cs.SyntheticBenchmarkGeneration.ListSyntheticBenchmarkGenerator;
import java.io.File;
import java.io.StringReader;
import java.io.StringWriter;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
import java.util.function.IntPredicate;
......@@ -86,92 +87,52 @@ public class PapiRunnerTest {
}
@Test
public void TestEmptyBenchmark() throws PapiException {
runner.emptyBenchmark();
}
public void testPapiRunBenchmark() throws PapiException {
MapSyntheticBenchmark bench = new MapSyntheticBenchmark(0, 100, new HashMap());
@Test
public void TestPapiFeatureGathering() throws PapiException, InvocationTargetException, IllegalAccessException {
List<SyntheticBenchmark<?>> syntheticBenchmarks = new ListSyntheticBenchmarkGenerator().createApplications(
0,
3,
100
PapiRunner runner = new PapiRunner(
CounterSpecification.Companion.fromFile(new File("papi_avail"))
);
List<PapiRunner.FeatureVector> data = runner.getFeatures(10, syntheticBenchmarks);
Assert.assertFalse(data.isEmpty());
for (PapiRunner.FeatureVector v : data) {
Assert.assertFalse(v.getCounters().isEmpty());
Map<String, List<Long>> results = runner.runApplication(100, bench);
Assert.assertFalse(results.isEmpty());
for (String k : results.keySet()) {
Assert.assertFalse(results.get(k).isEmpty());
}
}
@Test
public void TestCSVExport() {
Assert.assertEquals("", runner.featuresToCSV(new ArrayList<>()));
List<PapiRunner.FeatureVector> data = new ArrayList();
data.add(new PapiRunner.FeatureVector("app1", "java.util.ArrayList",
"java.util.ArrayList",
new TreeMap<String, Double>() {{
put("COUNTER_1", 123.4);
put("COUNTER_2", 123.4);
}}
));
data.add(new PapiRunner.FeatureVector("app2", "java.util.Vector",
"java.util.ArrayList",
new TreeMap<String, Double>() {{
put("COUNTER_1", 1234.5);
put("COUNTER_3", 12345.0);
}}
));
String expectedHeader = "application,data_structure,best_data_structure,COUNTER_1,COUNTER_2,COUNTER_3";
String expectedData1 = "app1,java.util.ArrayList,java.util.ArrayList,123.4,123.4,None";
String expectedData2 = "app2,java.util.Vector,java.util.ArrayList,1234.5,None,12345.0";
Assert.assertEquals(
expectedHeader + "\n" + expectedData1 + "\n" + expectedData2,
runner.featuresToCSV(data));
}
/**
* Runs a test from JMH-processed data
*/
@Test
public void TestJMHRunner() {
JMHProcessor processor = new JMHProcessor();
String data = JMHProcessorTest.Companion.generateData();
List<JMHProcessor.JMHRecord> processed = processor.process(new StringReader(data));
List<PapiRunner.FeatureVector> result = runner.processJMHData(10, processed);
public void testRunManyBenchmarks() {
// We want to test our setup can handle many benchmarks without crashing.
PapiRunner runner = new PapiRunner(
CounterSpecification.Companion.fromFile(new File("papi_avail"))
);
List<SyntheticBenchmark<?>> benchmarks = new ArrayList<>();
for (int i = 0; i < 100; ++i) {
MapSyntheticBenchmark bench = new MapSyntheticBenchmark(i, 100, new HashMap<>());
benchmarks.add(bench);
}
Assert.assertFalse(result.isEmpty());
Assert.assertFalse(result.stream().allMatch(it -> it.getCounters().isEmpty()));
List<Map<String, List<Long>>> results = runner.runApplications(100, benchmarks);
for (Map<String, List<Long>> r : results) {
Assert.assertFalse(r.isEmpty());
for (String k : r.keySet()) {
Assert.assertFalse(r.get(k).isEmpty());
}
}
}
/**
* Runs a test for a bug where all values of a generated run where the same for all rows.
*/
@Test
public void TestRunApplicationsDifferent() {
JMHProcessor processor = new JMHProcessor();
String data = JMHProcessorTest.Companion.generateData();
List<JMHProcessor.JMHRecord> processed = processor.process(new StringReader(data));
List<PapiRunner.FeatureVector> result = runner.processJMHData(10, processed);
List<Double> column = result.stream().map((row) -> row.getCounters().get("PAPI_BR_NTK"))
.collect(Collectors.toList());
// Check the data is correct
Assert.assertTrue(column.stream().noneMatch(Objects::isNull));
Assert.assertFalse(column.stream().allMatch(x -> x.equals(column.get(0))));
public void TestEmptyBenchmark() throws PapiException {
runner.emptyBenchmark();
}
@Test
public void TestRunApplications() {
SetSyntheticBenchmark setsmall = new SetSyntheticBenchmark(0, 100, 0, new HashSet<>());
SetSyntheticBenchmark setbig = new SetSyntheticBenchmark(0, 100, 1000, new HashSet<>());
List<SetSyntheticBenchmark> apps = new ArrayList<>();
List<SyntheticBenchmark<?>> apps = new ArrayList<>();
apps.add(setsmall);
apps.add(setbig);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment