Commit 7fad5c09 authored by Noric Couderc's avatar Noric Couderc
Browse files

Implemented benchmark generation from JMH record

Required some refactorings.

Created a type for interface type: InterfaceName
Created a type for method selection type : MethodSelectionType

Refactored benchmark generation for automatically loading markov
chains if needed, only the method selection type is needed to
create a benchmark generator.
parent 15921727
package se.lth.cs.jmh;
import se.lth.cs.SyntheticBenchmarkGeneration.markov.MarkovChain;
import se.lth.cs.bcgen.BenchmarkGenerator;
import se.lth.cs.bcgen.MarkovMethodSelectionStrategy;
import se.lth.cs.InterfaceName;
import se.lth.cs.bcgen.MethodSelectionStrategy;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
public abstract class ApplicationBenchmark {
public enum InterfaceName {
LIST,
MAP,
SET
}
public static Map<InterfaceName, MethodSelectionStrategy> methodSelectionStrategyForInterface =
new HashMap<>();
......@@ -35,48 +25,4 @@ public abstract class ApplicationBenchmark {
InterfaceName.SET, new File(dirPath.resolve("markov-chain-distinct-Set.json").toUri())
);
}
// TODO: Move this function to BenchmarkGenerator (since you need quite a lot of data from it)
public static
MethodSelectionStrategy getMethodSelectionStrategy(InterfaceName interfaceName, String mssId,
BenchmarkGenerator bcgen) {
MethodSelectionStrategy mss = null;
switch (mssId) {
case "UNIFORM": mss = MethodSelectionStrategy.UNIFORM;
break;
case "POLYA" : mss = MethodSelectionStrategy.POLYA;
break;
case "MARKOV" :
// Try to get it if it's already loaded,
// Load from file if it's not the case.
// The problem with this scheme is that the lazy loading of
// the markov chain prevent from taking the function to BenchmarkGenerator
// One option is to move the static maps above to the MethodSelectionStrategy class
// ANother option is to create a class to manage this loading of markov chains
// (Still needs the data from benchmark generator though...)
mss = methodSelectionStrategyForInterface.get(interfaceName);
if (mss == null) {
try {
MarkovChain<Integer> mkChain = bcgen.getMarkovChain(
new FileReader(markovChainFiles.get(interfaceName)));
mss = new MarkovMethodSelectionStrategy(mkChain);
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException();
}
methodSelectionStrategyForInterface.put(interfaceName, mss);
} else {
// We had already loaded it, we reset the markov chain
// In the markov method selection strategy
// so it's ready for a new benchmark
MarkovMethodSelectionStrategy mkss = (MarkovMethodSelectionStrategy) mss;
mkss.reset();
}
break;
default:
throw new RuntimeException("Invalid: " + mssId);
}
return mss;
}
}
......@@ -4,8 +4,10 @@ import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.infra.Blackhole;
import se.lth.cs.bcgen.BCBenchmark;
import se.lth.cs.bcgen.ListBenchmarkGenerator;
import se.lth.cs.bcgen.MethodSelectionStrategy;
import se.lth.cs.bcgen.MethodSelectionStrategyLoader;
import se.lth.cs.bcgen.MethodSelectionType;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
......@@ -35,7 +37,7 @@ public class ListApplicationBenchmark extends ApplicationBenchmark {
public Object currentSyntheticBenchmarkTarget;
@Setup(Level.Invocation)
public void doSetup() {
public void doSetup() throws IOException {
List<Object> datastructure;
switch (datastructureName) {
case "LinkedList":
......@@ -52,14 +54,13 @@ public class ListApplicationBenchmark extends ApplicationBenchmark {
}
ListBenchmarkGenerator benchmarkGenerator = new ListBenchmarkGenerator(baseStructureSize,
MethodSelectionStrategy.UNIFORM);
MethodSelectionType.POLYA);
// We set it after creating the benchmark because we need some information from the
// benchmarkGenerator to create the MethodSelectionStrategy
benchmarkGenerator.setMethodSelectionStrategy(
getMethodSelectionStrategy(
InterfaceName.LIST,
methodSelectionStrategyId,
benchmarkGenerator));
MethodSelectionStrategyLoader.getMethodSelectionStrategy(methodSelectionStrategyId,
benchmarkGenerator)
);
currentSyntheticBenchmark = benchmarkGenerator.genBenchmark(datastructureName + baseStructureSize, this.applicationSize, this.seed, this.seed);
benchmarkGenerator.fill(datastructure, this.seed);
currentSyntheticBenchmarkTarget = datastructure;
......
......@@ -4,8 +4,10 @@ import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.infra.Blackhole;
import se.lth.cs.bcgen.BCBenchmark;
import se.lth.cs.bcgen.MapBenchmarkGenerator;
import se.lth.cs.bcgen.MethodSelectionStrategy;
import se.lth.cs.bcgen.MethodSelectionStrategyLoader;
import se.lth.cs.bcgen.MethodSelectionType;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.Map;
......@@ -32,14 +34,13 @@ public class MapApplicationBenchmark extends ApplicationBenchmark {
public Object currentSyntheticBenchmarkTarget;
@Setup(Level.Invocation)
public void doSetup() throws ClassNotFoundException, IllegalAccessException, InstantiationException {
public void doSetup() throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException {
Map<Object, Object> datastructure = (Map<Object, Object>) Class.forName("java.util." + datastructureName).newInstance();
MapBenchmarkGenerator benchmarkGenerator = new MapBenchmarkGenerator(baseStructureSize,
MethodSelectionStrategy.POLYA);
MethodSelectionType.POLYA);
benchmarkGenerator.setMethodSelectionStrategy(
getMethodSelectionStrategy(InterfaceName.MAP,
methodSelectionStrategyId,
benchmarkGenerator)
MethodSelectionStrategyLoader.getMethodSelectionStrategy(
methodSelectionStrategyId, benchmarkGenerator)
);
currentSyntheticBenchmark = benchmarkGenerator.genBenchmark(datastructureName + baseStructureSize, this.applicationSize, this.seed, this.seed);
benchmarkGenerator.fill(datastructure, this.seed);
......
......@@ -3,9 +3,11 @@ package se.lth.cs.jmh;
import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.infra.Blackhole;
import se.lth.cs.bcgen.BCBenchmark;
import se.lth.cs.bcgen.MethodSelectionStrategy;
import se.lth.cs.bcgen.MethodSelectionStrategyLoader;
import se.lth.cs.bcgen.MethodSelectionType;
import se.lth.cs.bcgen.SetBenchmarkGenerator;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.Set;
......@@ -32,14 +34,14 @@ public class SetApplicationBenchmark extends ApplicationBenchmark {
public Object currentSyntheticBenchmarkTarget;
@Setup(Level.Invocation)
public void doSetup() throws ClassNotFoundException, IllegalAccessException, InstantiationException {
public void doSetup() throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException {
Set<Object> datastructure = (Set<Object>) Class.forName("java.util." + datastructureName).newInstance();
SetBenchmarkGenerator benchmarkGenerator = new SetBenchmarkGenerator(baseStructureSize,
MethodSelectionStrategy.POLYA);
MethodSelectionType.POLYA);
benchmarkGenerator.setMethodSelectionStrategy(
getMethodSelectionStrategy(InterfaceName.SET,
methodSelectionStrategyId,
benchmarkGenerator)
MethodSelectionStrategyLoader.getMethodSelectionStrategy(
methodSelectionStrategyId, benchmarkGenerator
)
);
currentSyntheticBenchmark = benchmarkGenerator.genBenchmark(datastructureName + baseStructureSize, this.applicationSize, this.seed, this.seed);
benchmarkGenerator.fill(datastructure, this.seed);
......
package se.lth.cs;
public enum InterfaceName {
LIST,
MAP,
SET
}
......@@ -52,7 +52,7 @@ public class BCBenchmarkPackage<T> {
? datastructure_size
: gen.getSize(datastructure);
this.generator = gen.makeGenerator(ds_size);
this.generator = gen.makeGenerator(ds_size, method_selection_type);
this.generator.overrideSimulationState(datastructure); // dangerous!
if (this.trace_method_selector != null) {
this.generator.setMethodSelectionStrategy(this.trace_method_selector);
......@@ -100,6 +100,7 @@ public class BCBenchmarkPackage<T> {
protected int seed;
protected int trace_size;
protected int datastructure_size;
protected MethodSelectionType method_selection_type;
protected BCGen<T> gen;
protected Blackhole blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
List<BCBenchStep> bench_steps = new ArrayList<BCBenchStep>();
......@@ -182,7 +183,9 @@ public class BCBenchmarkPackage<T> {
}
public BCBenchmarkPackage(int seed, int trace_size, int datastructure_size, T target,
MethodSelectionType mst,
BCGen<T> gen) {
this.method_selection_type = mst;
this.generation_type = GENTYPE_SYNTH;
this.datastructure = target;
this.seed = seed;
......@@ -199,6 +202,7 @@ public class BCBenchmarkPackage<T> {
public BCBenchmarkPackage(String trace_key, List<String> methods, T target,
BCGen<T> gen) {
this.method_selection_type = MethodSelectionType.TRACE;
this.generation_type = GENTYPE_TRACE + trace_key;
this.datastructure = target;
this.seed = 0;
......@@ -327,7 +331,8 @@ public class BCBenchmarkPackage<T> {
public interface BCGen<S> {
public void clear(S ds);
public BenchmarkGenerator<S> makeGenerator(int initial_datastructure_size);
public BenchmarkGenerator<S> makeGenerator(int initial_datastructure_size,
MethodSelectionType mst);
public int getSize(S ds);
/**
* Name of the short interface to be benchmarked interface (e.g., "List")
......@@ -338,11 +343,13 @@ public class BCBenchmarkPackage<T> {
// lists --------------------
private static final BCGen<List<Object>> BCGenLIST = new BCGen<List<Object>>() {
private static BCGen<List<Object>> BCGenLIST = new BCGen<List<Object>>() {
@Override
public BenchmarkGenerator<List<Object>>
makeGenerator(int ds_size) {
return new ListBenchmarkGenerator(ds_size, MethodSelectionStrategy.POLYA);
// TODO: Make the method selection strategy a parameter
makeGenerator(int ds_size, MethodSelectionType mst) {
return new ListBenchmarkGenerator(ds_size, mst);
}
@Override
......@@ -364,9 +371,16 @@ public class BCBenchmarkPackage<T> {
}
};
@Deprecated
public static BCBenchmarkPackage<List<Object>>
LIST(int seed, int trace_size, int datastructure_size, java.util.List<?> target) {
return LIST(seed, trace_size, datastructure_size, MethodSelectionType.POLYA, target);
}
public static BCBenchmarkPackage<List<Object>>
LIST(int seed, int trace_size, int datastructure_size, MethodSelectionType mst, java.util.List<?> target) {
return new BCBenchmarkPackage<List<Object>>(seed, trace_size, datastructure_size, (java.util.List<Object>)target,
mst,
BCGenLIST);
}
......@@ -380,8 +394,10 @@ public class BCBenchmarkPackage<T> {
private static final BCGen<Set<Object>> BCGenSET = new BCGen<Set<Object>>() {
@Override
public BenchmarkGenerator<Set<Object>>
makeGenerator(int ds_size) {
return new SetBenchmarkGenerator(ds_size, MethodSelectionStrategy.POLYA);
// TODO: Change method selection strategy
makeGenerator(int ds_size, MethodSelectionType mst
) {
return new SetBenchmarkGenerator(ds_size, mst);
}
@Override
......@@ -403,9 +419,16 @@ public class BCBenchmarkPackage<T> {
}
};
@Deprecated
public static BCBenchmarkPackage<Set<Object>>
SET(int seed, int trace_size, int datastructure_size, java.util.Set<?> target) {
return SET(seed, trace_size, datastructure_size, MethodSelectionType.POLYA, target);
}
public static BCBenchmarkPackage<Set<Object>>
SET(int seed, int trace_size, int datastructure_size, MethodSelectionType mst, java.util.Set<?> target) {
return new BCBenchmarkPackage<Set<Object>>(seed, trace_size, datastructure_size, (java.util.Set<Object>)target,
mst,
BCGenSET);
}
......@@ -419,8 +442,8 @@ public class BCBenchmarkPackage<T> {
private static final BCGen<Map<Object, Object>> BCGenMAP = new BCGen<Map<Object, Object>>() {
@Override
public BenchmarkGenerator<Map<Object, Object>>
makeGenerator(int ds_size) {
return new MapBenchmarkGenerator(ds_size, MethodSelectionStrategy.POLYA);
makeGenerator(int ds_size, MethodSelectionType mst) {
return new MapBenchmarkGenerator(ds_size, mst);
}
@Override
......@@ -442,9 +465,16 @@ public class BCBenchmarkPackage<T> {
}
};
@Deprecated
public static BCBenchmarkPackage<Map<Object, Object>>
MAP(int seed, int trace_size, int datastructure_size, java.util.Map<?,?> target) {
return MAP(seed, trace_size, datastructure_size, MethodSelectionType.POLYA, target);
}
public static BCBenchmarkPackage<Map<Object, Object>>
MAP(int seed, int trace_size, int datastructure_size, MethodSelectionType mst, java.util.Map<?,?> target) {
return new BCBenchmarkPackage<Map<Object, Object>>(seed, trace_size, datastructure_size, (java.util.Map<Object, Object>)target,
mst,
BCGenMAP);
}
......
......@@ -2,6 +2,7 @@ package se.lth.cs.bcgen;
import org.objectweb.asm.ClassWriter;
import org.objectweb.asm.MethodVisitor;
import se.lth.cs.InterfaceName;
import se.lth.cs.SyntheticBenchmarkGeneration.markov.MarkovChain;
import java.io.IOException;
......@@ -46,10 +47,19 @@ public abstract class BenchmarkGenerator<S> {
private Map<String, String> method_name_translator = new HashMap<>(); // "String toString()" -> "runToString"
private ArrayList<String> ops_keys_sorted = new ArrayList<>();
// Method selection is first a symbolic value and
// We initialize the method selector on demand
private MethodSelectionType method_selector_style;
private MethodSelectionStrategy method_selector;
private S state_override = null;
public BenchmarkGenerator(int initial_size, MethodSelectionStrategy mst) {
this.initial_size = initial_size;
this.method_selector = mst;
this.method_selector_style = mst.getSelectionType();
}
/**
* Use the specified datastructure (instead of a generated one) when simulating updates
*
......@@ -70,9 +80,10 @@ public abstract class BenchmarkGenerator<S> {
* collection indices heuristically.
*/
public
BenchmarkGenerator(int data_size, MethodSelectionStrategy s) {
BenchmarkGenerator(int data_size, MethodSelectionType mst) {
this.initial_size = data_size;
this.method_selector = s;
this.method_selector_style= mst;
this.method_selector = null; // Will be set when needed
}
public BenchmarkGenerator<S>
......@@ -287,6 +298,8 @@ public abstract class BenchmarkGenerator<S> {
gen(String pkgname, int length, int insn_seed, int data_seed) {
this.inst_random = new Random(insn_seed);
this.random = new Random(data_seed);
// We check the method selection strategy is set and if it isn't we create it
checkMethodSelectionStrategy();
final String target_typename_internal = this.getInterfaceNameInternal();
final String bcbench_classname_internal = BCBenchmark.NAME.replace('.', '/');
......@@ -678,6 +691,27 @@ public abstract class BenchmarkGenerator<S> {
}
/**
* Initializes the method selection strategy object based on the type
* of method selection strategy you want to use
* @param mst
*/
private void initializeMethodSelectionStrategy(MethodSelectionType mst) {
switch (mst) {
case UNIFORM: setMethodSelectionStrategy(MethodSelectionStrategyLoader.UNIFORM); break;
case POLYA: setMethodSelectionStrategy(MethodSelectionStrategyLoader.POLYA); break;
case MARKOV: setMethodSelectionStrategy(MethodSelectionStrategyLoader.MARKOV(this)); break;
// TODO: This is incorrect (I think we should just use the selection strategies to generate traces...
case TRACE: setMethodSelectionStrategy(MethodSelectionStrategy.fromTrace(new ArrayList<>()));
default: return;
}
}
private void checkMethodSelectionStrategy() {
if (this.method_selector == null) {
initializeMethodSelectionStrategy(this.method_selector_style);
}
}
public MarkovChain<Integer> getMarkovChain(Reader r) throws IOException {
MarkovChain<Integer> mkChain = MarkovChain.fromReader(r).map((label) -> {
......@@ -688,4 +722,6 @@ public abstract class BenchmarkGenerator<S> {
return mkChain;
}
public abstract InterfaceName getInterfaceType();
}
package se.lth.cs.bcgen;
import se.lth.cs.InterfaceName;
import java.util.*;
/**
......@@ -12,6 +14,10 @@ public class ListBenchmarkGenerator extends BenchmarkGenerator<List<Object>> {
public static final Collection<Object> C_3 = new ArrayList<>();
public static final Collection<Object> C_4 = new ArrayList<>();
public ListBenchmarkGenerator(int initial_size, MethodSelectionStrategy polya) {
super(initial_size, polya);
}
public static void fillCollection(Collection<Object> c, int size, int seed) {
Random random = new Random(seed + 1 + size);
for (int i = 0; i < size; i++) {
......@@ -41,8 +47,8 @@ public class ListBenchmarkGenerator extends BenchmarkGenerator<List<Object>> {
* The benchmark generator is only valid for the given size
*/
public
ListBenchmarkGenerator(int initial_size, MethodSelectionStrategy s) {
super(initial_size, s);
ListBenchmarkGenerator(int initial_size, MethodSelectionType mst) {
super(initial_size, mst);
}
/**
......@@ -54,6 +60,11 @@ public class ListBenchmarkGenerator extends BenchmarkGenerator<List<Object>> {
ListBenchmarkGenerator.fillCollection(l, this.initial_size, data_seed);
}
@Override
public InterfaceName getInterfaceType() {
return InterfaceName.LIST;
}
@Override
public List<Object>
initState(int seed) {
......
package se.lth.cs.bcgen;
import se.lth.cs.InterfaceName;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
......@@ -14,6 +16,10 @@ public class MapBenchmarkGenerator extends BenchmarkGenerator<Map<Object, Object
public static final Map<Object, Object> C_3 = new HashMap<>();
public static final Map<Object, Object> C_4 = new HashMap<>();
public MapBenchmarkGenerator(int initial_size, MethodSelectionStrategy mst) {
super(initial_size, mst);
}
public static void fillMap(Map<Object, Object> c, int size, int seed) {
Random random = new Random(size + 1 + seed);
int k = 0;
......@@ -39,8 +45,8 @@ public class MapBenchmarkGenerator extends BenchmarkGenerator<Map<Object, Object
* The seed is used for fill() only.
*/
public
MapBenchmarkGenerator(int initial_size, MethodSelectionStrategy s) {
super(initial_size, s);
MapBenchmarkGenerator(int initial_size, MethodSelectionType mst) {
super(initial_size, mst);
}
/**
......@@ -52,6 +58,11 @@ public class MapBenchmarkGenerator extends BenchmarkGenerator<Map<Object, Object
MapBenchmarkGenerator.fillMap(l, this.initial_size, seed);
}
@Override
public InterfaceName getInterfaceType() {
return InterfaceName.MAP;
}
@Override
public Map<Object, Object>
initState(int data_seed) {
......
......@@ -36,6 +36,11 @@ public class MarkovMethodSelectionStrategy extends MethodSelectionStrategy {
return sample;
}
@Override
public MethodSelectionType getSelectionType() {
return MethodSelectionType.MARKOV;
}
/**
* Resets the markov chain so that the next sample is the initial sample
*/
......
......@@ -14,10 +14,17 @@ public abstract class MethodSelectionStrategy {
abstract public int
selectNextMethod(BenchmarkGenerator<?>.MethodProbabilityTable table, Random random);
/**
* Returns the symbolic type of method selection strategy
* @return
*/
public abstract MethodSelectionType
getSelectionType();
/**
* Select one of the methods, using weighted sampling and filtering for eligibility
*/
private static int
protected static int
randomSelect(BenchmarkGenerator<?>.MethodProbabilityTable mpt, Random randomGen) {
int total_eligible_weights = 0;
for (int i = 0; i < mpt.size(); i++) {
......@@ -37,27 +44,7 @@ public abstract class MethodSelectionStrategy {
throw new RuntimeException("No eligible method to call!");
}
/**
* Uniform probability distribution
*/
public static MethodSelectionStrategy UNIFORM = new MethodSelectionStrategy() {
@Override
public int selectNextMethod(BenchmarkGenerator<?>.MethodProbabilityTable table, Random randomGen) {
return randomSelect(table, randomGen);
}
};
/**
* Polya probability distribution
*/
public static MethodSelectionStrategy POLYA = new MethodSelectionStrategy() {
@Override
public int selectNextMethod(BenchmarkGenerator<?>.MethodProbabilityTable table, Random randomGen) {
int selection = randomSelect(table, randomGen);
table.updateWeight(selection, 1);
return selection;
}
};
/**
* Polya probability distribution
......@@ -107,5 +94,10 @@ public abstract class MethodSelectionStrategy {
return method_index;
}
@Override
public MethodSelectionType getSelectionType() {
return MethodSelectionType.TRACE;
}
}
}
package se.lth.cs.bcgen;
import se.lth.cs.InterfaceName;
import se.lth.cs.SyntheticBenchmarkGeneration.markov.MarkovChain;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
public class MethodSelectionStrategyLoader {
public static Map<InterfaceName, File> markovChainFiles;
public static Path jbrainyPath = Path.of("SET ME");
static void loadMarkovChainFiles() {
assert(jbrainyPath.isAbsolute());
Path dirPath = jbrainyPath.resolve(Path.of("data", "markov-chains"));
markovChainFiles = Map.of(
InterfaceName.LIST, new File(dirPath.resolve("markov-chain-distinct-List.json").toUri()),
InterfaceName.MAP, new File(dirPath.resolve("markov-chain-distinct-Map.json").toUri()),
InterfaceName.SET, new File(dirPath.resolve("markov-chain-distinct-Set.json").toUri())
);
}
/**
* Uniform probability distribution
*/
public static MethodSelectionStrategy UNIFORM = new MethodSelectionStrategy() {
@Override
public int selectNextMethod(BenchmarkGenerator<?>.MethodProbabilityTable table, Random randomGen) {
return randomSelect(table, randomGen);