Commit 0de1f81c authored by Noric Couderc's avatar Noric Couderc
Browse files

Implemented normalization of PAPI features.

parent 2ba5612d
......@@ -4,16 +4,19 @@ import papi.EventSet
import papi.Papi
import papi.PapiException
import java.io.FileWriter
import java.time.Duration
import java.time.Instant
open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
open class PapiRunner(numRuns : Int, counters: CounterSpecification, normalizeFeatures : Boolean) {
constructor(numRuns: Int, specification: CounterSpecification) : this(numRuns, specification, false) {
}
init {
Papi.init()
}
val counterSpec = counters
val numRuns = numRuns
val normalizeFeatures = normalizeFeatures
/**
* Empty benchmark:
......@@ -197,7 +200,7 @@ open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
return medianLong(data)
}
fun runApplications(syntheticBenchmarks: List<SyntheticBenchmark<*>>): List<Map<String, List<Long>>> {
fun runApplications(syntheticBenchmarks: List<SyntheticBenchmark<*>>): List<Map<String, Double>> {
// return syntheticBenchmarks.map { b -> runApplication(iterations, b) }
val specs = createRunSpecs(syntheticBenchmarks)
val map = specs.map {
......@@ -206,12 +209,42 @@ open class PapiRunner(numRuns : Int, counters: CounterSpecification) {
}
// TODO: Return a Map<Pair<CounterString, Benchmark>, List<Long>> instead of this mess.
val map1 = map.groupBy { s -> s.first.syntheticBenchmark }
return map1.mapValues {
val mapsOfSamples = map1.mapValues {
it.value.map {
Pair(it.first.counter, it.second)
}.toMap()
}.map {
it.value
}
// If we are in normalized mode, the scores are normalized by the total number of
// instructions
return when (normalizeFeatures) {
true -> mapsOfSamples.map {
normalizeFeatures(medianFeatures(it))
}
false -> mapsOfSamples.map { medianFeatures(it) }
}
}
/**
* Compute the median of samples for each counter
* @param counters: A map from PAPI counter names to lists of samples
*/
fun medianFeatures(counters : Map<String, List<Long>>) : Map<String, Double> {
return counters.mapValues {
medianLong(it.value)
}
}
/**
* Normalize a map of counters by the PAPI_TOT_INS (Total number of instructions)
* @param counters: A map from PAPI counter names to their median value
*/
fun normalizeFeatures(counters : Map<String, Double>) : Map<String, Double> {
val total = counters["PAPI_TOT_INS"] ?: error("'PAPI_TOT_INS' not present in counters")
return counters.mapValues {
it.value / total
}
}
}
package se.lth.cs
import java.io.Writer
import java.util.HashMap
import java.util.*
class SyntheticBenchmarkFeaturePrinter(out: Writer, papiRunner: PapiRunner, methodPrintFormat : String)
class SyntheticBenchmarkFeaturePrinter(out: Writer,
papiRunner: PapiRunner,
methodPrintFormat: String)
: SyntheticBenchmarkDataPrinter(out, methodPrintFormat) {
private val papiRunner: PapiRunner = papiRunner
private var benchmarkToCounters = HashMap<SyntheticBenchmark<*>,
Map<String, List<Long>>>()
Map<String, Double>>()
/**
* Prints the header for benchmark features
......@@ -39,7 +41,7 @@ class SyntheticBenchmarkFeaturePrinter(out: Writer, papiRunner: PapiRunner, meth
val hardwareCounters = benchmarkToCounters[syntheticBenchmark]!!
for (kvp in hardwareCounters) {
printer.printRecord(identifier, kvp.key, medianLong(kvp.value))
printer.printRecord(identifier, kvp.key, kvp.value)
}
}
......
......@@ -2,11 +2,15 @@ package se.lth.cs.commandline
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.options.default
import com.github.ajalt.clikt.parameters.options.flag
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.choice
import com.github.ajalt.clikt.parameters.types.int
import se.lth.cs.*
import java.io.*
import java.io.File
import java.io.FileReader
import java.io.FileWriter
import java.io.Reader
import kotlin.system.exitProcess
class PapiCommandLine : CliktCommand() {
......@@ -41,10 +45,14 @@ class PapiCommandLine : CliktCommand() {
.choice("JAVA-STANDARD-FORMAT", "SYNTHETIC-RUN-FORMAT")
.default("JAVA-STANDARD-FORMAT")
val normalizeFeatures : Boolean by option("-nf", "--normalize-features",
help = "Activate or deactivate normalization of featurs")
.flag("normalize features", default = false)
fun checkFileExists(file: File) {
if (!file.exists()) {
print("'${file.name}' not found")
print("File '${file.name}' not found")
exitProcess(1)
}
}
......@@ -70,7 +78,8 @@ class PapiCommandLine : CliktCommand() {
"FEATURE-VECTORS" -> {
val runner = PapiRunner(
numberRuns,
CounterSpecification.fromFile(countersFile)
CounterSpecification.fromFile(countersFile),
normalizeFeatures = normalizeFeatures
)
SyntheticBenchmarkFeaturePrinter(writer, runner, methodOutputFormat)
}
......
......@@ -4,13 +4,18 @@ import org.junit.Test;
import papi.EventSet;
import papi.Papi;
import papi.PapiException;
import se.lth.cs.*;
import se.lth.cs.CounterSpecification;
import se.lth.cs.MapSyntheticBenchmark;
import se.lth.cs.PapiRunner;
import se.lth.cs.SyntheticBenchmark;
import se.lth.cs.SyntheticBenchmarkGeneration.ListSyntheticBenchmarkGenerator;
import java.io.File;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.IntPredicate;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
......@@ -68,11 +73,11 @@ public class PapiRunnerTest {
3,
100
);
List<Map<String, List<Long>>> data = runner.runApplications(syntheticBenchmarks);
List<Map<String, Double>> data = runner.runApplications(syntheticBenchmarks);
// We check all known Papi counters are in the map
Assert.assertFalse(data.isEmpty());
for (Map<String, List<Long>> counters : data) {
for (Map<String, Double> counters : data) {
Assert.assertEquals(
specification.getCurrentSpec().keySet(),
counters.keySet());
......@@ -101,49 +106,31 @@ public class PapiRunnerTest {
benchmarks.add(bench);
}
List<Map<String, List<Long>>> results = runner.runApplications(benchmarks);
for (Map<String, List<Long>> r : results) {
List<Map<String, Double>> results = runner.runApplications(benchmarks);
for (Map<String, Double> r : results) {
Assert.assertFalse(r.isEmpty());
for (String k : r.keySet()) {
Assert.assertFalse(r.get(k).isEmpty());
}
}
}
@Test
public void TestEmptyBenchmark() throws PapiException {
public void testEmptyBenchmark() throws PapiException {
runner.emptyBenchmark();
}
@Test
public void TestRunApplications() {
SetSyntheticBenchmark setsmall = new SetSyntheticBenchmark(0, 100, 0, new HashSet<>());
SetSyntheticBenchmark setbig = new SetSyntheticBenchmark(0, 100, 1000, new HashSet<>());
List<SyntheticBenchmark<?>> apps = new ArrayList<>();
apps.add(setsmall);
apps.add(setbig);
List<Map<String, List<Long>>> result = runner.runApplications(apps);
Assert.assertEquals(
result.get(0).keySet(),
result.get(1).keySet()
public void testNormalize() {
HashMap<String, Double> counters = new HashMap<>();
counters.put("PAPI_TOT_INS", 10.0);
counters.put("PAPI_TLB_IM", 5.0);
counters.put("PAPI_VEC_DP", 0.0);
Map<String, Double> normalized = runner.normalizeFeatures(counters);
Map<String, Double> expected = new HashMap<String, Double>(
);
List<Double> ratios = new ArrayList<>();
for (String key : result.get(0).keySet()) {
List<Long> smallAppValues = result.get(0).get(key);
List<Long> bigAppValues = result.get(1).get(key);
double smallAppMedian = UtilsKt.medianLong(smallAppValues);
double bigAppMedian = UtilsKt.medianLong(bigAppValues);
double ratio = smallAppMedian / bigAppMedian;
ratios.add(ratio);
}
// We compute that more than 75% of the ratios are below one,
// as a way to measure the statistical significance of the difference
// in size
Map<Boolean, List<Double>> distribution = ratios.stream().collect(Collectors.partitioningBy(x -> x <= 1.0));
System.out.println(distribution);
// TODO: Fix this
// Assert.assertTrue( distribution.get(true).size() >= 0.75 * ratios.size() );
expected.put("PAPI_TOT_INS", 1.0);
expected.put("PAPI_TLB_IM", 0.5);
expected.put("PAPI_VEC_DP", 0.0);
Assert.assertEquals(expected, normalized);
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment