Commit 58d869ab authored by Christoph Reichenbach's avatar Christoph Reichenbach
Browse files

Initial bcgen trace processing code (needs testing...!)

parent 0a7db021
......@@ -106,14 +106,14 @@ dependencies {
jmh {
resultFormat="CSV"
// benchmarkParameters=["seed": (0..500).toList().collect { it.toString() } ]
benchmarkParameters=["seed": (0..500).toList().collect { it.toString() } ]
benchmarkParameters=["seed": (0..20).toList().collect { it.toString() } ]
duplicateClassesStrategy = 'warn'
iterations=5
// timeOnIteration="250ms"
timeOnIteration="50ms"
warmupIterations=1
warmupIterations=4
// warmup="250ms"
warmup="100ms"
warmup="50ms"
}
// A small JMH test
......
......@@ -31,7 +31,7 @@ public class BCBenchmarkInfo<S> {
this.op_names.add(op);
}
public List<? extends String>
public List<String>
getOpNames() {
return this.op_names;
}
......
......@@ -116,6 +116,12 @@ public abstract class BenchmarkGenerator<S> {
return new BCArg.Conversion<S>(ty, arg);
}
/**
* Fill the given container with the given data seed to the preconfigured size
*/
public abstract void
fill(S target, int data_seed);
public BCArg<S>
asObject(BCArg<S> arg) {
return asType(BCType.OBJECT, arg);
......@@ -268,6 +274,9 @@ public abstract class BenchmarkGenerator<S> {
S state, BCBenchmarkInfo<S> info) {
mpt.updateEligibility(); // hidden use of state
int method_id = this.method_selector.selectNextMethod(mpt);
if (method_id == MethodSelectionStrategy.SKIP) {
return; // skip method
}
String op_name = this.ops_keys_sorted.get(method_id);
BCBenchmarkOp<S> op = this.ops.get(op_name);
......@@ -328,6 +337,11 @@ public abstract class BenchmarkGenerator<S> {
}
}
public void
generateFromTrace(List<String> methods) {
this.setMethodSelectionStrategy(MethodSelectionStrategy.fromTrace(methods));
}
/**
* Helper class for method selection.
*
......@@ -338,6 +352,7 @@ public abstract class BenchmarkGenerator<S> {
private boolean[] eligibility;
private S state;
private Random random_gen;
private Map<String, Integer> method_indices = null;
public MethodProbabilityTable(S state, Random r) {
this.random_gen = r;
......@@ -369,6 +384,18 @@ public abstract class BenchmarkGenerator<S> {
return this.weights.length;
}
public Integer
findMethod(String method_name) {
if (this.method_indices == null) {
this.method_indices = new HashMap<>();
for (int i = 0; i < this.size(); ++i ) {
String opname = ops_keys_sorted.get(i);
this.method_indices.put(opname, i);
}
}
return this.method_indices.get(method_name);
}
/**
* Can be called in the current configuration
*/
......
......@@ -52,6 +52,7 @@ public class ListBenchmarkGenerator extends BenchmarkGenerator<List<Object>> {
/**
* Fills a list to the size expected by the generator
*/
@Override
public void
fill(List<Object> l, int data_seed) {
ListBenchmarkGenerator.fillCollection((Collection<Object>)l, this.initial_size, data_seed);
......
......@@ -46,6 +46,7 @@ public class MapBenchmarkGenerator extends BenchmarkGenerator<Map<Object, Object
/**
* Fills a list to the size expected by the generator
*/
@Override
public void
fill(Map<Object, Object> l, int seed) {
MapBenchmarkGenerator.fillMap(l, this.initial_size, seed);
......
package se.lth.cs.bcgen;
import java.util.List;
import java.util.Iterator;
/**
* Strategy for choosing the next method
*/
interface MethodSelectionStrategy {
public static final int SKIP = -1; // skip method
// may return SKIP
public int
selectNextMethod(BenchmarkGenerator<?>.MethodProbabilityTable table);
......@@ -28,4 +34,53 @@ interface MethodSelectionStrategy {
return selection;
}
};
/**
* Polya probability distribution
*/
public static Trace fromTrace(List<String> trace) {
return new Trace(trace);
};
public static class Trace implements MethodSelectionStrategy {
private List<String> trace;
private Iterator<String> trace_iterator = null;
private int misses = 0;
public
Trace(java.util.List<String> trace) {
this.trace = trace;
reset();
}
public int
getMisses() {
return this.misses;
}
public void
reset() {
this.misses = 0;
this.trace_iterator = trace.iterator();
}
@Override
public int selectNextMethod(BenchmarkGenerator<?>.MethodProbabilityTable table) {
if (!this.trace_iterator.hasNext()) {
throw new RuntimeException("Internal bug: ran out of methods during trace replay!");
}
final String method_name = this.trace_iterator.next();
final Integer method_index = table.findMethod(method_name);
if (method_index == null) {
throw new RuntimeException("Encountered unknown method: '" + method_name + "'");
}
if (!table.isEligible(method_index)) {
++this.misses;
return SKIP;
}
return method_index;
}
}
}
......@@ -48,6 +48,7 @@ public class SetBenchmarkGenerator extends BenchmarkGenerator<Set<Object>> {
/**
* Fills a list to the size expected by the generator
*/
@Override
public void
fill(Set<Object> l, int seed) {
SetBenchmarkGenerator.fillCollection((Collection<Object>)l, this.initial_size, seed);
......
......@@ -6,6 +6,7 @@ import org.apache.commons.csv.CSVPrinter
import org.apache.commons.csv.CSVRecord
import java.io.*
import java.lang.Exception
import se.lth.cs.bcgen.*
class JMHProcessor {
......@@ -18,21 +19,21 @@ class JMHProcessor {
return listOf(collection, seed.toString(), size.toString() , baseStructureSize.toString(), datastructure, best)
}
fun generateSyntheticBenchmark() : SyntheticBenchmark<*>? {
var syntheticBenchmark : SyntheticBenchmark<*>? = null
fun generateSyntheticBenchmark() : BCBenchmarkPackage<*>? {
var syntheticBenchmark : BCBenchmarkPackage<*>? = null
val dataStructure = getClassFromSimpleName(datastructure)
if (collection == "List") {
syntheticBenchmark = ListSyntheticBenchmark(seed, size, baseStructureSize, dataStructure as MutableList<Int>?)
syntheticBenchmark = BCBenchmarkPackage.LIST(seed, size, baseStructureSize, dataStructure as MutableList<Object>?)
return syntheticBenchmark
}
if (collection == "Map") {
syntheticBenchmark = MapSyntheticBenchmark(seed, size, baseStructureSize, dataStructure as MutableMap<Int, Int>?)
syntheticBenchmark = BCBenchmarkPackage.MAP(seed, size, baseStructureSize, dataStructure as MutableMap<Any, Any>?)
return syntheticBenchmark
}
if (collection == "Set") {
syntheticBenchmark = SetSyntheticBenchmark(seed, size, baseStructureSize, dataStructure as MutableSet<Int>?)
syntheticBenchmark = BCBenchmarkPackage.SET(seed, size, baseStructureSize, dataStructure as MutableSet<Object>?)
return syntheticBenchmark
}
return null
......
......@@ -6,6 +6,8 @@ import papi.PapiException
import java.io.FileWriter
import java.time.Duration
import java.time.Instant
import se.lth.cs.bcgen.*
import org.openjdk.jmh.infra.Blackhole
open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
......@@ -85,12 +87,12 @@ open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
data class RunSpec(val numRuns: Int,
val counter : String,
val eventSet: EventSet,
val syntheticBenchmark: SyntheticBenchmark<*>)
val syntheticBenchmark: BCBenchmarkPackage<*>)
/**
* Creates a list of specifications of what counter to get from what benchmarks
*/
fun createRunSpecs(syntheticBenchmarks: List<SyntheticBenchmark<*>>): List<RunSpec> {
fun createRunSpecs(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<RunSpec> {
val counters = counterSpec.currentSpec
// Let's say we want to minimize the counter change
......@@ -113,16 +115,12 @@ open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
val app = spec.syntheticBenchmark
val evset = spec.eventSet
// We do the measurements
app.reset(app.baseDataStructureSize)
app.reset(app.getDatastructureSize())
var accumulator = 0
evset.start()
while (app.hasNext()) {
val result: Int? = app.invokeCurrentMethod() as? Int
accumulator += result ?: 1
app.tick()
}
val resultDataStructure = app.getDataStructure()
app.runBenchmark()
evset.stop()
val resultDataStructure = app.getDatastructure()
// We reset the benchmark
app.reset(0)
......@@ -141,7 +139,7 @@ open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
* A function for running one benchmark, gathering all the hardware counters
* Present in the counter specification we have.
*/
fun runApplication(app: SyntheticBenchmark<*>) : Map<String, List<Long>> {
fun runApplication(app: BCBenchmarkPackage<*>) : Map<String, List<Long>> {
// We will write the aggregators to nowhere
val writer = FileWriter("/dev/null")
// We get integers for all the counters.
......@@ -154,16 +152,13 @@ open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
val samples = mutableListOf<Long>()
for (run in 0 until numRuns) {
// We do the measurements
app.reset(app.baseDataStructureSize)
app.reset(app.getDatastructureSize())
var accumulator = 0
evset.start()
while (app.hasNext()) {
val result: Int? = app.invokeCurrentMethod() as? Int
accumulator += result ?: 1
app.tick()
}
val resultDataStructure = app.getDataStructure()
app.runBenchmark()
evset.stop()
System.err.println("MISSED TRACE METHODS COUNT = " + app.getMissedMethods())
val resultDataStructure = app.getDatastructure()
// We reset the benchmark
app.reset(0)
......@@ -177,7 +172,7 @@ open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
result[counter] = ArrayList(samples)
samples.clear()
}
app.clearDataStructure()
app.clearAll();
writer.close()
......@@ -198,7 +193,7 @@ open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
fun runApplications(syntheticBenchmarks: List<SyntheticBenchmark<*>>): List<Triple<SyntheticBenchmark<*>, String, List<Long>>> {
fun runApplications(syntheticBenchmarks: List<BCBenchmarkPackage<*>>): List<Triple<BCBenchmarkPackage<*>, String, List<Long>>> {
// return syntheticBenchmarks.map { b -> runApplication(iterations, b) }
val specs = createRunSpecs(syntheticBenchmarks)
val spectToCounters = specs.map {
......@@ -213,14 +208,14 @@ open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
}
fun runApplicationsMedian(syntheticBenchmarks: List<SyntheticBenchmark<*>>) : List<Triple<SyntheticBenchmark<*>, String, Double>> {
fun runApplicationsMedian(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<Triple<BCBenchmarkPackage<*>, String, Double>> {
val samples = runApplications(syntheticBenchmarks)
return samples.map {
Triple(it.first, it.second, medianLong(it.third))
}
}
fun runApplicationsNormalized(syntheticBenchmarks: List<SyntheticBenchmark<*>>) : List<Triple<SyntheticBenchmark<*>, String, Double>> {
fun runApplicationsNormalized(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) : List<Triple<BCBenchmarkPackage<*>, String, Double>> {
val countersMedians = runApplicationsMedian(syntheticBenchmarks)
val instructionsForBenchmark = countersMedians.filter { it.second == "PAPI_TOT_INS" }
.map { Pair(it.first, it.third) }
......
......@@ -4,6 +4,7 @@ import org.apache.commons.csv.CSVFormat
import org.apache.commons.csv.CSVPrinter
import java.io.Writer
import java.lang.IllegalArgumentException
import se.lth.cs.bcgen.*
/**
* An abstract class for printing some data about synthetic benchmarks to CSV.
......@@ -27,7 +28,7 @@ abstract class SyntheticBenchmarkDataPrinter(out : Writer,
}
}
open fun printToCSV(syntheticBenchmarks: Sequence<SyntheticBenchmark<*>>) {
open fun printToCSV(syntheticBenchmarks: Sequence<BCBenchmarkPackage<*>>) {
printHeader();
for (syntheticBenchmark in syntheticBenchmarks) {
......@@ -38,8 +39,8 @@ abstract class SyntheticBenchmarkDataPrinter(out : Writer,
printer.close()
}
abstract fun printBenchmarkRows(syntheticBenchmark: SyntheticBenchmark<*>)
abstract fun printBenchmarkRows(syntheticBenchmark: BCBenchmarkPackage<*>)
abstract fun printHeader()
}
\ No newline at end of file
}
......@@ -2,6 +2,7 @@ package se.lth.cs
import java.io.Writer
import java.util.*
import se.lth.cs.bcgen.*
class SyntheticBenchmarkFeaturePrinter(out: Writer,
papiRunner: PapiRunner,
......@@ -13,7 +14,7 @@ class SyntheticBenchmarkFeaturePrinter(out: Writer,
private val normalizeFeatures = normalizeFeatures
private var benchmarkResults : Map<String, List<Triple<SyntheticBenchmark<*>, String, Double>>> = mapOf()
private var benchmarkResults : Map<String, List<Triple<BCBenchmarkPackage<*>, String, Double>>> = mapOf()
/**
* Prints the header for benchmark features
......@@ -27,7 +28,7 @@ class SyntheticBenchmarkFeaturePrinter(out: Writer,
* Prints rows for one benchmark
* It used to be in wide-form, but we are going to print it in long form, as it is easier to implement
*/
override fun printBenchmarkRows(syntheticBenchmark: SyntheticBenchmark<*>) {
override fun printBenchmarkRows(syntheticBenchmark: BCBenchmarkPackage<*>) {
val identifier = syntheticBenchmark.benchmarkIdentifier;
println("Extracting features for benchmark: $identifier")
......@@ -52,12 +53,12 @@ class SyntheticBenchmarkFeaturePrinter(out: Writer,
}
}
override fun printToCSV(syntheticBenchmarks : Sequence<SyntheticBenchmark<*>>) {
override fun printToCSV(syntheticBenchmarks : Sequence<BCBenchmarkPackage<*>>) {
runBenchmarks(syntheticBenchmarks.toList())
super.printToCSV(syntheticBenchmarks)
}
private fun runBenchmarks(syntheticBenchmarks: List<SyntheticBenchmark<*>>) {
private fun runBenchmarks(syntheticBenchmarks: List<BCBenchmarkPackage<*>>) {
benchmarkResults = when (normalizeFeatures) {
true -> papiRunner.runApplicationsNormalized(syntheticBenchmarks).groupBy {
it.first.benchmarkIdentifier
......
package se.lth.cs
import se.lth.cs.bcgen.*
import java.io.Writer
/**
......@@ -20,11 +21,11 @@ class SyntheticBenchmarkHistPrinter(out: Writer, methodOutputFormat: String)
)
}
override fun printBenchmarkRows(syntheticBenchmark: SyntheticBenchmark<*>) {
override fun printBenchmarkRows(syntheticBenchmark: BCBenchmarkPackage<*>) {
val hist = processHist(syntheticBenchmark)
val seed = syntheticBenchmark.seed
val baseStructureSize = syntheticBenchmark.baseStructureSize
val size = syntheticBenchmark.benchmarkSize
val baseStructureSize = syntheticBenchmark.getDatastructureSize()
val size = syntheticBenchmark.getTraceSize()
for (kvp in hist) {
printer.printRecord(
......@@ -39,9 +40,9 @@ class SyntheticBenchmarkHistPrinter(out: Writer, methodOutputFormat: String)
}
}
private fun processHist(syntheticBenchmark: SyntheticBenchmark<*>): List<Pair<String, Long>> {
private fun processHist(syntheticBenchmark: BCBenchmarkPackage<*>): List<Pair<String, Long>> {
return syntheticBenchmark.methodHistogram()
.toList()
.sortedBy<Pair<String, Long>, String> { it.first }
}
}
\ No newline at end of file
}
package se.lth.cs
import se.lth.cs.bcgen.*
import java.io.Writer
/**
......@@ -19,12 +20,12 @@ class SyntheticBenchmarkTracePrinter(out: Writer, methodOutputFormat: String)
"step"
)
}
override fun printBenchmarkRows(syntheticBenchmark: SyntheticBenchmark<*>) {
override fun printBenchmarkRows(syntheticBenchmark: BCBenchmarkPackage<*>) {
val id = syntheticBenchmark.benchmarkIdentifier
val trace = syntheticBenchmark.trace
val trace = syntheticBenchmark.getTrace()
val seed = syntheticBenchmark.seed
val baseStructureSize = syntheticBenchmark.baseStructureSize
val size = syntheticBenchmark.benchmarkSize
val baseStructureSize = syntheticBenchmark.getDatastructureSize()
val size = syntheticBenchmark.getTraceSize()
for (i in 0 until trace.size) {
val method = trace.get(i)
......@@ -34,9 +35,9 @@ class SyntheticBenchmarkTracePrinter(out: Writer, methodOutputFormat: String)
size,
baseStructureSize,
syntheticBenchmark.dataStructureSimpleName,
getMethodName(method.name),
getMethodName(method),
i
)
}
}
}
\ No newline at end of file
}
......@@ -5,19 +5,20 @@ import org.apache.commons.csv.CSVParser
import org.apache.commons.csv.CSVRecord
import org.apache.commons.lang3.tuple.ImmutablePair
import java.io.Reader
import se.lth.cs.bcgen.*
import java.util.*
import java.util.stream.Collectors
class TraceLoader {
@Throws(ClassNotFoundException::class, IllegalAccessException::class, InstantiationException::class)
fun readCsv(reader: Reader): Sequence<SyntheticBenchmark<*>> {
fun readCsv(reader: Reader): Sequence<BCBenchmarkPackage<*>> {
val parser = CSVParser(reader, CSVFormat.DEFAULT.withHeader())
// We group traces by : location (in the code) and target type.
val grouped =
parser.groupBy { r -> ImmutablePair(r["location"], r["target_type"]) }
val apps = ArrayList<SyntheticBenchmark<*>>()
val apps = ArrayList<BCBenchmarkPackage<*>>()
for (kvp in grouped.entries) {
// Sometimes, you cannot recreate the class.
// ignore it and report
......@@ -49,28 +50,28 @@ class TraceLoader {
traceKey : String,
interfaceCandidates: List<String>,
rows : List<CSVRecord>,
initClass: Class<*>): SyntheticBenchmark<out Any> {
initClass: Class<*>): BCBenchmarkPackage<out Any> {
val interfaceName = interfaceCandidates
.first()
.removePrefix("java.util.")
val methods = rows.stream()
.map { r -> r["method"].replace("\"", "") }
.map { r -> matchFunction(r) }
//.map { r -> matchFunction(r) }
.filter { m -> m != null }
.collect(Collectors.toList())
val app = when (interfaceName) {
"Set" -> SetSyntheticBenchmark(traceKey, methods, initClass.newInstance() as Set<Int>)
"Map" -> MapSyntheticBenchmark(traceKey, methods, initClass.newInstance() as Map<Int, Int>)
"List" -> ListSyntheticBenchmark(traceKey, methods, initClass.newInstance() as List<Int>)
val app : BCBenchmarkPackage<*> = when (interfaceName) {
"Set" -> BCBenchmarkPackage.SET(traceKey, methods, initClass.newInstance() as Set<Any>)
"Map" -> BCBenchmarkPackage.MAP(traceKey, methods, initClass.newInstance() as Map<Any, Any>)
"List" -> BCBenchmarkPackage.LIST(traceKey, methods, initClass.newInstance() as List<Any>)
else -> throw IllegalArgumentException(
String.format("The class '%s' is not supported by the current generator",
initClass.name)
)
}
System.out.println("Generated benchmark " + app.benchmarkIdentifier)
System.out.println("Generated benchmark " + app.getBenchmarkIdentifier())
return app
}
......
......@@ -6,6 +6,7 @@ import com.github.ajalt.clikt.parameters.options.flag
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.choice
import com.github.ajalt.clikt.parameters.types.int
import se.lth.cs.bcgen.*
import se.lth.cs.*
import java.io.File
import java.io.FileReader
......@@ -65,7 +66,7 @@ class PapiCommandLine : CliktCommand() {
val outputFile = File(outputFileName)
val reader = FileReader(inputFile)
val benchmarks = when (inputType) {
val benchmarks : Sequence<BCBenchmarkPackage<*>>? = when (inputType) {
"JMH" -> readJMHData(reader)
"TRACE" -> TraceLoader().readCsv(reader)
else -> null
......@@ -89,7 +90,7 @@ class PapiCommandLine : CliktCommand() {
}
private fun readJMHData(reader : Reader): Sequence<SyntheticBenchmark<*>> {
private fun readJMHData(reader : Reader): Sequence<BCBenchmarkPackage<*>> {
val jmhData = JMHProcessor().process(reader)
val jmhRows : List<JMHProcessor.JMHRecord> = if (numSamples == -1) {
jmhData
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment