Skip to content
Commits on Source (7)
Expand all Show whitespace changes
Inline Side-by-side
JBrainy model investigation.ipynb 0 → 100644
View file @ 724a22e7
%% Cell type:markdown id: tags:
# Model investigation
%% Cell type:markdown id: tags:
## Imports and stuff
%% Cell type:code id: tags:
``` python
%load_ext autoreload
```
%% Cell type:code id: tags:
``` python
%autoreload 2
```
%% Cell type:code id: tags:
``` python
from train_model import load_training_data, normalize_data
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
```
%% Cell type:markdown id: tags:
## Loading the data
%% Cell type:code id: tags:
``` python
data = load_training_data("data/jmh-results-9307f70f.csv",
"software-perf.csv",
"data/hardware-perf-data.csv")
```
%% Cell type:code id: tags:
``` python
data["data"].describe()
```
%% Output
Threads Samples Score Score Error (99.9%) \
count 39643.0 39643.0 39643.000000 39643.000000
mean 1.0 10.0 97446.013333 6353.113808
std 0.0 0.0 178141.235600 13825.193748
min 1.0 10.0 10.153360 0.070377
25% 1.0 10.0 1964.599295 73.185579
50% 1.0 10.0 12473.571612 535.074529
75% 1.0 10.0 96777.965251 4896.221083
max 1.0 10.0 709935.485128 224533.338017
Param: applicationSize Param: baseStructureSize Param: seed \
count 39643.000000 39643.000000 39643.000000
mean 376.808516 3639.709406 249.669046
std 448.570636 4489.781686 144.230804
min 10.000000 0.000000 0.000000
25% 10.000000 0.000000 125.000000
50% 100.000000 1000.000000 250.000000
75% 1000.000000 10000.000000 375.000000
max 1000.000000 10000.000000 499.000000
Lowest score Threads_best Samples_best ... PAPI_REF_CYC \
count 39643.000000 39643.0 39643.0 ... 3.964300e+04
mean 91092.899525 1.0 10.0 ... 3.422986e+05
std 168517.226906 0.0 0.0 ... 1.936199e+06
min 9.975083 1.0 10.0 ... 1.728000e+03
25% 1871.044644 1.0 10.0 ... 3.294000e+03
50% 11750.424375 1.0 10.0 ... 2.260800e+04
75% 85632.226297 1.0 10.0 ... 1.802070e+05
max 695996.818472 1.0 10.0 ... 5.683349e+07
PAPI_SP_OPS PAPI_SR_INS PAPI_STL_ICY PAPI_TLB_DM PAPI_TLB_IM \
count 39643.0 3.964300e+04 3.964300e+04 39643.000000 39643.000000
mean 0.0 8.176693e+04 1.392746e+04 12.370759 6.364364
std 0.0 4.709412e+05 7.647459e+04 17.167082 2.544259
min 0.0 3.170000e+02 8.500000e+01 0.000000 5.000000
25% 0.0 4.840000e+02 2.965000e+02 2.000000 5.000000
50% 0.0 3.817000e+03 1.434500e+03 6.000000 5.000000
75% 0.0 3.391000e+04 8.316000e+03 16.000000 6.000000
max 0.0 1.248077e+07 2.205866e+06 685.000000 27.000000
PAPI_TOT_CYC PAPI_TOT_INS PAPI_VEC_DP PAPI_VEC_SP
count 3.964300e+04 3.964300e+04 39643.0 39643.0
mean 3.625881e+05 1.024529e+06 0.0 0.0
std 2.045093e+06 6.593926e+06 0.0 0.0
min 1.825000e+03 2.297000e+03 0.0 0.0
25% 3.440750e+03 3.583000e+03 0.0 0.0
50% 2.376650e+04 3.187000e+04 0.0 0.0
75% 1.898682e+05 2.906225e+05 0.0 0.0
max 5.999046e+07 1.743255e+08 0.0 0.0
[8 rows x 100 columns]
%% Cell type:markdown id: tags:
## What are the most important features
Try to visualize the most important features here!
%% Cell type:code id: tags:
``` python
from pickle import load
# We load the model that we have trained and generated
model = load(open("jbrainy-classifier.pickle", 'r'))
```
%% Cell type:markdown id: tags:
Problem, the raw data that I have is not the same as the data the classifier was trained on. There have been several transformations!
1. The data has been normalized, with PAPI_TOT_INS for the HW performance counters, and with ApplicationSize for the SW performance counters
2. We added the data structures to the table (one hot encoding)
3. We used a poly transformer to get the polynomial features (good news, this one can give me the names of the features!)
4. The features have been "standard scaled": `(x - u) / s` where `u` is the average and `s` is the standard deviation
Several ideas: I can save the feature names with the classifier, or I can recover it.
Getting it directly seems like a better idea.
%% Cell type:code id: tags:
``` python
model.feature_names
```
%% Output
['1',
'PAPI_BR_CN',
'PAPI_BR_INS',
'PAPI_BR_MSP',
'PAPI_BR_NTK',
'PAPI_BR_PRC',
'PAPI_BR_TKN',
'PAPI_BR_UCN',
'PAPI_DP_OPS',
'PAPI_FDV_INS',
'PAPI_FP_INS',
'PAPI_FP_OPS',
'PAPI_L1_DCM',
'PAPI_L1_ICM',
'PAPI_L1_LDM',
'PAPI_L1_STM',
'PAPI_L1_TCM',
'PAPI_L2_DCA',
'PAPI_L2_DCH',
'PAPI_L2_DCM',
'PAPI_L2_DCR',
'PAPI_L2_DCW',
'PAPI_L2_ICA',
'PAPI_L2_ICH',
'PAPI_L2_ICM',
'PAPI_L2_ICR',
'PAPI_L2_STM',
'PAPI_L2_TCA',
'PAPI_L2_TCM',
'PAPI_L2_TCR',
'PAPI_L2_TCW',
'PAPI_L3_DCA',
'PAPI_L3_DCR',
'PAPI_L3_DCW',
'PAPI_L3_ICA',
'PAPI_L3_ICR',
'PAPI_L3_TCA',
'PAPI_L3_TCM',
'PAPI_L3_TCR',
'PAPI_L3_TCW',
'PAPI_LD_INS',
'PAPI_REF_CYC',
'PAPI_SP_OPS',
'PAPI_SR_INS',
'PAPI_STL_ICY',
'PAPI_TLB_DM',
'PAPI_TLB_IM',
'PAPI_TOT_CYC',
'PAPI_TOT_INS',
'PAPI_VEC_DP',
'PAPI_VEC_SP',
'add(Object)',
'add(int, Object)',
'addAll(Collection)',
'addAll(int, Collection)',
'clear()',
'contains(Object)',
'containsAll(Collection)',
'containsKey(Object)',
'containsValue(Object)',
'entrySet()',
'equals(Object)',
'get(Object)',
'hashCode()',
'indexOf(Object)',
'isEmpty()',
'iterator()',
'keySet()',
'lastIndexOf(Object)',
'listIterator()',
'listIterator(int)',
'put(Object, Object)',
'putAll(Map)',
'remove(Object)',
'remove(int)',
'removeAll(Collection)',
'retainAll(Collection)',
'set(int, Object)',
'size()',
'sort(Comparator)',
'subList(int, int)',
'toArray()',
'toArray(Object[])',
'values()',
'ArrayList',
'HashMap',
'HashSet',
'LinkedHashMap',
'LinkedHashSet',
'LinkedList',
'TreeMap',
'TreeSet',
'Vector',
'PAPI_BR_CN^2',
'PAPI_BR_CN PAPI_BR_INS',
'PAPI_BR_CN PAPI_BR_MSP',
'PAPI_BR_CN PAPI_BR_NTK',
'PAPI_BR_CN PAPI_BR_PRC',
'PAPI_BR_CN PAPI_BR_TKN',
'PAPI_BR_CN PAPI_BR_UCN',
'PAPI_BR_CN PAPI_DP_OPS',
'PAPI_BR_CN PAPI_FDV_INS',
'PAPI_BR_CN PAPI_FP_INS',
'PAPI_BR_CN PAPI_FP_OPS',
'PAPI_BR_CN PAPI_L1_DCM',
'PAPI_BR_CN PAPI_L1_ICM',
'PAPI_BR_CN PAPI_L1_LDM',
'PAPI_BR_CN PAPI_L1_STM',
'PAPI_BR_CN PAPI_L1_TCM',
'PAPI_BR_CN PAPI_L2_DCA',
'PAPI_BR_CN PAPI_L2_DCH',
'PAPI_BR_CN PAPI_L2_DCM',
'PAPI_BR_CN PAPI_L2_DCR',
'PAPI_BR_CN PAPI_L2_DCW',
'PAPI_BR_CN PAPI_L2_ICA',
'PAPI_BR_CN PAPI_L2_ICH',
'PAPI_BR_CN PAPI_L2_ICM',
'PAPI_BR_CN PAPI_L2_ICR',
'PAPI_BR_CN PAPI_L2_STM',
'PAPI_BR_CN PAPI_L2_TCA',
'PAPI_BR_CN PAPI_L2_TCM',
'PAPI_BR_CN PAPI_L2_TCR',
'PAPI_BR_CN PAPI_L2_TCW',
'PAPI_BR_CN PAPI_L3_DCA',
'PAPI_BR_CN PAPI_L3_DCR',
'PAPI_BR_CN PAPI_L3_DCW',
'PAPI_BR_CN PAPI_L3_ICA',
'PAPI_BR_CN PAPI_L3_ICR',
'PAPI_BR_CN PAPI_L3_TCA',
'PAPI_BR_CN PAPI_L3_TCM',
'PAPI_BR_CN PAPI_L3_TCR',
'PAPI_BR_CN PAPI_L3_TCW',
'PAPI_BR_CN PAPI_LD_INS',
'PAPI_BR_CN PAPI_REF_CYC',
'PAPI_BR_CN PAPI_SP_OPS',
'PAPI_BR_CN PAPI_SR_INS',
'PAPI_BR_CN PAPI_STL_ICY',
'PAPI_BR_CN PAPI_TLB_DM',
'PAPI_BR_CN PAPI_TLB_IM',
'PAPI_BR_CN PAPI_TOT_CYC',
'PAPI_BR_CN PAPI_TOT_INS',
'PAPI_BR_CN PAPI_VEC_DP',
'PAPI_BR_CN PAPI_VEC_SP',
'PAPI_BR_CN add(Object)',
'PAPI_BR_CN add(int, Object)',
'PAPI_BR_CN addAll(Collection)',
'PAPI_BR_CN addAll(int, Collection)',
'PAPI_BR_CN clear()',
'PAPI_BR_CN contains(Object)',
'PAPI_BR_CN containsAll(Collection)',
'PAPI_BR_CN containsKey(Object)',
'PAPI_BR_CN containsValue(Object)',
'PAPI_BR_CN entrySet()',
'PAPI_BR_CN equals(Object)',
'PAPI_BR_CN get(Object)',
'PAPI_BR_CN hashCode()',
'PAPI_BR_CN indexOf(Object)',
'PAPI_BR_CN isEmpty()',
'PAPI_BR_CN iterator()',
'PAPI_BR_CN keySet()',
'PAPI_BR_CN lastIndexOf(Object)',
'PAPI_BR_CN listIterator()',
'PAPI_BR_CN listIterator(int)',
'PAPI_BR_CN put(Object, Object)',
'PAPI_BR_CN putAll(Map)',
'PAPI_BR_CN remove(Object)',
'PAPI_BR_CN remove(int)',
'PAPI_BR_CN removeAll(Collection)',
'PAPI_BR_CN retainAll(Collection)',
'PAPI_BR_CN set(int, Object)',
'PAPI_BR_CN size()',
'PAPI_BR_CN sort(Comparator)',
'PAPI_BR_CN subList(int, int)',
'PAPI_BR_CN toArray()',
'PAPI_BR_CN toArray(Object[])',
'PAPI_BR_CN values()',
'PAPI_BR_CN ArrayList',
'PAPI_BR_CN HashMap',
'PAPI_BR_CN HashSet',
'PAPI_BR_CN LinkedHashMap',
'PAPI_BR_CN LinkedHashSet',
'PAPI_BR_CN LinkedList',
'PAPI_BR_CN TreeMap',
'PAPI_BR_CN TreeSet',
'PAPI_BR_CN Vector',
'PAPI_BR_INS^2',
'PAPI_BR_INS PAPI_BR_MSP',
'PAPI_BR_INS PAPI_BR_NTK',
'PAPI_BR_INS PAPI_BR_PRC',
'PAPI_BR_INS PAPI_BR_TKN',
'PAPI_BR_INS PAPI_BR_UCN',
'PAPI_BR_INS PAPI_DP_OPS',
'PAPI_BR_INS PAPI_FDV_INS',
'PAPI_BR_INS PAPI_FP_INS',
'PAPI_BR_INS PAPI_FP_OPS',
'PAPI_BR_INS PAPI_L1_DCM',
'PAPI_BR_INS PAPI_L1_ICM',
'PAPI_BR_INS PAPI_L1_LDM',
'PAPI_BR_INS PAPI_L1_STM',
'PAPI_BR_INS PAPI_L1_TCM',
'PAPI_BR_INS PAPI_L2_DCA',
'PAPI_BR_INS PAPI_L2_DCH',
'PAPI_BR_INS PAPI_L2_DCM',
'PAPI_BR_INS PAPI_L2_DCR',
'PAPI_BR_INS PAPI_L2_DCW',
'PAPI_BR_INS PAPI_L2_ICA',
'PAPI_BR_INS PAPI_L2_ICH',
'PAPI_BR_INS PAPI_L2_ICM',
'PAPI_BR_INS PAPI_L2_ICR',
'PAPI_BR_INS PAPI_L2_STM',
'PAPI_BR_INS PAPI_L2_TCA',
'PAPI_BR_INS PAPI_L2_TCM',
'PAPI_BR_INS PAPI_L2_TCR',
'PAPI_BR_INS PAPI_L2_TCW',
'PAPI_BR_INS PAPI_L3_DCA',
'PAPI_BR_INS PAPI_L3_DCR',
'PAPI_BR_INS PAPI_L3_DCW',
'PAPI_BR_INS PAPI_L3_ICA',
'PAPI_BR_INS PAPI_L3_ICR',
'PAPI_BR_INS PAPI_L3_TCA',
'PAPI_BR_INS PAPI_L3_TCM',
'PAPI_BR_INS PAPI_L3_TCR',
'PAPI_BR_INS PAPI_L3_TCW',
'PAPI_BR_INS PAPI_LD_INS',
'PAPI_BR_INS PAPI_REF_CYC',
'PAPI_BR_INS PAPI_SP_OPS',
'PAPI_BR_INS PAPI_SR_INS',
'PAPI_BR_INS PAPI_STL_ICY',
'PAPI_BR_INS PAPI_TLB_DM',
'PAPI_BR_INS PAPI_TLB_IM',
'PAPI_BR_INS PAPI_TOT_CYC',
'PAPI_BR_INS PAPI_TOT_INS',
'PAPI_BR_INS PAPI_VEC_DP',
'PAPI_BR_INS PAPI_VEC_SP',
'PAPI_BR_INS add(Object)',
'PAPI_BR_INS add(int, Object)',
'PAPI_BR_INS addAll(Collection)',
'PAPI_BR_INS addAll(int, Collection)',
'PAPI_BR_INS clear()',
'PAPI_BR_INS contains(Object)',
'PAPI_BR_INS containsAll(Collection)',
'PAPI_BR_INS containsKey(Object)',
'PAPI_BR_INS containsValue(Object)',
'PAPI_BR_INS entrySet()',
'PAPI_BR_INS equals(Object)',
'PAPI_BR_INS get(Object)',
'PAPI_BR_INS hashCode()',
'PAPI_BR_INS indexOf(Object)',
'PAPI_BR_INS isEmpty()',
'PAPI_BR_INS iterator()',
'PAPI_BR_INS keySet()',
'PAPI_BR_INS lastIndexOf(Object)',
'PAPI_BR_INS listIterator()',
'PAPI_BR_INS listIterator(int)',
'PAPI_BR_INS put(Object, Object)',
'PAPI_BR_INS putAll(Map)',
'PAPI_BR_INS remove(Object)',
'PAPI_BR_INS remove(int)',
'PAPI_BR_INS removeAll(Collection)',
'PAPI_BR_INS retainAll(Collection)',
'PAPI_BR_INS set(int, Object)',
'PAPI_BR_INS size()',
'PAPI_BR_INS sort(Comparator)',
'PAPI_BR_INS subList(int, int)',
'PAPI_BR_INS toArray()',
'PAPI_BR_INS toArray(Object[])',
'PAPI_BR_INS values()',
'PAPI_BR_INS ArrayList',
'PAPI_BR_INS HashMap',
'PAPI_BR_INS HashSet',
'PAPI_BR_INS LinkedHashMap',
'PAPI_BR_INS LinkedHashSet',
'PAPI_BR_INS LinkedList',
'PAPI_BR_INS TreeMap',
'PAPI_BR_INS TreeSet',
'PAPI_BR_INS Vector',
'PAPI_BR_MSP^2',
'PAPI_BR_MSP PAPI_BR_NTK',
'PAPI_BR_MSP PAPI_BR_PRC',
'PAPI_BR_MSP PAPI_BR_TKN',
'PAPI_BR_MSP PAPI_BR_UCN',
'PAPI_BR_MSP PAPI_DP_OPS',
'PAPI_BR_MSP PAPI_FDV_INS',
'PAPI_BR_MSP PAPI_FP_INS',
'PAPI_BR_MSP PAPI_FP_OPS',
'PAPI_BR_MSP PAPI_L1_DCM',
'PAPI_BR_MSP PAPI_L1_ICM',
'PAPI_BR_MSP PAPI_L1_LDM',
'PAPI_BR_MSP PAPI_L1_STM',
'PAPI_BR_MSP PAPI_L1_TCM',
'PAPI_BR_MSP PAPI_L2_DCA',
'PAPI_BR_MSP PAPI_L2_DCH',
'PAPI_BR_MSP PAPI_L2_DCM',
'PAPI_BR_MSP PAPI_L2_DCR',
'PAPI_BR_MSP PAPI_L2_DCW',
'PAPI_BR_MSP PAPI_L2_ICA',
'PAPI_BR_MSP PAPI_L2_ICH',
'PAPI_BR_MSP PAPI_L2_ICM',
'PAPI_BR_MSP PAPI_L2_ICR',
'PAPI_BR_MSP PAPI_L2_STM',
'PAPI_BR_MSP PAPI_L2_TCA',
'PAPI_BR_MSP PAPI_L2_TCM',
'PAPI_BR_MSP PAPI_L2_TCR',
'PAPI_BR_MSP PAPI_L2_TCW',
'PAPI_BR_MSP PAPI_L3_DCA',
'PAPI_BR_MSP PAPI_L3_DCR',
'PAPI_BR_MSP PAPI_L3_DCW',
'PAPI_BR_MSP PAPI_L3_ICA',
'PAPI_BR_MSP PAPI_L3_ICR',
'PAPI_BR_MSP PAPI_L3_TCA',
'PAPI_BR_MSP PAPI_L3_TCM',
'PAPI_BR_MSP PAPI_L3_TCR',
'PAPI_BR_MSP PAPI_L3_TCW',
'PAPI_BR_MSP PAPI_LD_INS',
'PAPI_BR_MSP PAPI_REF_CYC',
'PAPI_BR_MSP PAPI_SP_OPS',
'PAPI_BR_MSP PAPI_SR_INS',
'PAPI_BR_MSP PAPI_STL_ICY',
'PAPI_BR_MSP PAPI_TLB_DM',
'PAPI_BR_MSP PAPI_TLB_IM',
'PAPI_BR_MSP PAPI_TOT_CYC',
'PAPI_BR_MSP PAPI_TOT_INS',
'PAPI_BR_MSP PAPI_VEC_DP',
'PAPI_BR_MSP PAPI_VEC_SP',
'PAPI_BR_MSP add(Object)',
'PAPI_BR_MSP add(int, Object)',
'PAPI_BR_MSP addAll(Collection)',
'PAPI_BR_MSP addAll(int, Collection)',
'PAPI_BR_MSP clear()',
'PAPI_BR_MSP contains(Object)',
'PAPI_BR_MSP containsAll(Collection)',
'PAPI_BR_MSP containsKey(Object)',
'PAPI_BR_MSP containsValue(Object)',
'PAPI_BR_MSP entrySet()',
'PAPI_BR_MSP equals(Object)',
'PAPI_BR_MSP get(Object)',
'PAPI_BR_MSP hashCode()',
'PAPI_BR_MSP indexOf(Object)',
'PAPI_BR_MSP isEmpty()',
'PAPI_BR_MSP iterator()',
'PAPI_BR_MSP keySet()',
'PAPI_BR_MSP lastIndexOf(Object)',
'PAPI_BR_MSP listIterator()',
'PAPI_BR_MSP listIterator(int)',
'PAPI_BR_MSP put(Object, Object)',
'PAPI_BR_MSP putAll(Map)',
'PAPI_BR_MSP remove(Object)',
'PAPI_BR_MSP remove(int)',
'PAPI_BR_MSP removeAll(Collection)',
'PAPI_BR_MSP retainAll(Collection)',
'PAPI_BR_MSP set(int, Object)',
'PAPI_BR_MSP size()',
'PAPI_BR_MSP sort(Comparator)',
'PAPI_BR_MSP subList(int, int)',
'PAPI_BR_MSP toArray()',
'PAPI_BR_MSP toArray(Object[])',
'PAPI_BR_MSP values()',
'PAPI_BR_MSP ArrayList',
'PAPI_BR_MSP HashMap',
'PAPI_BR_MSP HashSet',
'PAPI_BR_MSP LinkedHashMap',
'PAPI_BR_MSP LinkedHashSet',
'PAPI_BR_MSP LinkedList',
'PAPI_BR_MSP TreeMap',
'PAPI_BR_MSP TreeSet',
'PAPI_BR_MSP Vector',
'PAPI_BR_NTK^2',
'PAPI_BR_NTK PAPI_BR_PRC',
'PAPI_BR_NTK PAPI_BR_TKN',
'PAPI_BR_NTK PAPI_BR_UCN',
'PAPI_BR_NTK PAPI_DP_OPS',
'PAPI_BR_NTK PAPI_FDV_INS',
'PAPI_BR_NTK PAPI_FP_INS',
'PAPI_BR_NTK PAPI_FP_OPS',
'PAPI_BR_NTK PAPI_L1_DCM',
'PAPI_BR_NTK PAPI_L1_ICM',
'PAPI_BR_NTK PAPI_L1_LDM',
'PAPI_BR_NTK PAPI_L1_STM',
'PAPI_BR_NTK PAPI_L1_TCM',
'PAPI_BR_NTK PAPI_L2_DCA',
'PAPI_BR_NTK PAPI_L2_DCH',
'PAPI_BR_NTK PAPI_L2_DCM',
'PAPI_BR_NTK PAPI_L2_DCR',
'PAPI_BR_NTK PAPI_L2_DCW',
'PAPI_BR_NTK PAPI_L2_ICA',
'PAPI_BR_NTK PAPI_L2_ICH',
'PAPI_BR_NTK PAPI_L2_ICM',
'PAPI_BR_NTK PAPI_L2_ICR',
'PAPI_BR_NTK PAPI_L2_STM',
'PAPI_BR_NTK PAPI_L2_TCA',
'PAPI_BR_NTK PAPI_L2_TCM',
'PAPI_BR_NTK PAPI_L2_TCR',
'PAPI_BR_NTK PAPI_L2_TCW',
'PAPI_BR_NTK PAPI_L3_DCA',
'PAPI_BR_NTK PAPI_L3_DCR',
'PAPI_BR_NTK PAPI_L3_DCW',
'PAPI_BR_NTK PAPI_L3_ICA',
'PAPI_BR_NTK PAPI_L3_ICR',
'PAPI_BR_NTK PAPI_L3_TCA',
'PAPI_BR_NTK PAPI_L3_TCM',
'PAPI_BR_NTK PAPI_L3_TCR',
'PAPI_BR_NTK PAPI_L3_TCW',
'PAPI_BR_NTK PAPI_LD_INS',
'PAPI_BR_NTK PAPI_REF_CYC',
'PAPI_BR_NTK PAPI_SP_OPS',
'PAPI_BR_NTK PAPI_SR_INS',
'PAPI_BR_NTK PAPI_STL_ICY',
'PAPI_BR_NTK PAPI_TLB_DM',
'PAPI_BR_NTK PAPI_TLB_IM',
'PAPI_BR_NTK PAPI_TOT_CYC',
'PAPI_BR_NTK PAPI_TOT_INS',
'PAPI_BR_NTK PAPI_VEC_DP',
'PAPI_BR_NTK PAPI_VEC_SP',
'PAPI_BR_NTK add(Object)',
'PAPI_BR_NTK add(int, Object)',
'PAPI_BR_NTK addAll(Collection)',
'PAPI_BR_NTK addAll(int, Collection)',
'PAPI_BR_NTK clear()',
'PAPI_BR_NTK contains(Object)',
'PAPI_BR_NTK containsAll(Collection)',
'PAPI_BR_NTK containsKey(Object)',
'PAPI_BR_NTK containsValue(Object)',
'PAPI_BR_NTK entrySet()',
'PAPI_BR_NTK equals(Object)',
'PAPI_BR_NTK get(Object)',
'PAPI_BR_NTK hashCode()',
'PAPI_BR_NTK indexOf(Object)',
'PAPI_BR_NTK isEmpty()',
'PAPI_BR_NTK iterator()',
'PAPI_BR_NTK keySet()',
'PAPI_BR_NTK lastIndexOf(Object)',
'PAPI_BR_NTK listIterator()',
'PAPI_BR_NTK listIterator(int)',
'PAPI_BR_NTK put(Object, Object)',
'PAPI_BR_NTK putAll(Map)',
'PAPI_BR_NTK remove(Object)',
'PAPI_BR_NTK remove(int)',
'PAPI_BR_NTK removeAll(Collection)',
'PAPI_BR_NTK retainAll(Collection)',
'PAPI_BR_NTK set(int, Object)',
'PAPI_BR_NTK size()',
'PAPI_BR_NTK sort(Comparator)',
'PAPI_BR_NTK subList(int, int)',
'PAPI_BR_NTK toArray()',
'PAPI_BR_NTK toArray(Object[])',
'PAPI_BR_NTK values()',
'PAPI_BR_NTK ArrayList',
'PAPI_BR_NTK HashMap',
'PAPI_BR_NTK HashSet',
'PAPI_BR_NTK LinkedHashMap',
'PAPI_BR_NTK LinkedHashSet',
'PAPI_BR_NTK LinkedList',
'PAPI_BR_NTK TreeMap',
'PAPI_BR_NTK TreeSet',
'PAPI_BR_NTK Vector',
'PAPI_BR_PRC^2',
'PAPI_BR_PRC PAPI_BR_TKN',
'PAPI_BR_PRC PAPI_BR_UCN',
'PAPI_BR_PRC PAPI_DP_OPS',
'PAPI_BR_PRC PAPI_FDV_INS',
'PAPI_BR_PRC PAPI_FP_INS',
'PAPI_BR_PRC PAPI_FP_OPS',
'PAPI_BR_PRC PAPI_L1_DCM',
'PAPI_BR_PRC PAPI_L1_ICM',
'PAPI_BR_PRC PAPI_L1_LDM',
'PAPI_BR_PRC PAPI_L1_STM',
'PAPI_BR_PRC PAPI_L1_TCM',
'PAPI_BR_PRC PAPI_L2_DCA',
'PAPI_BR_PRC PAPI_L2_DCH',
'PAPI_BR_PRC PAPI_L2_DCM',
'PAPI_BR_PRC PAPI_L2_DCR',
'PAPI_BR_PRC PAPI_L2_DCW',
'PAPI_BR_PRC PAPI_L2_ICA',
'PAPI_BR_PRC PAPI_L2_ICH',
'PAPI_BR_PRC PAPI_L2_ICM',
'PAPI_BR_PRC PAPI_L2_ICR',
'PAPI_BR_PRC PAPI_L2_STM',
'PAPI_BR_PRC PAPI_L2_TCA',
'PAPI_BR_PRC PAPI_L2_TCM',
'PAPI_BR_PRC PAPI_L2_TCR',
'PAPI_BR_PRC PAPI_L2_TCW',
'PAPI_BR_PRC PAPI_L3_DCA',
'PAPI_BR_PRC PAPI_L3_DCR',
'PAPI_BR_PRC PAPI_L3_DCW',
'PAPI_BR_PRC PAPI_L3_ICA',
'PAPI_BR_PRC PAPI_L3_ICR',
'PAPI_BR_PRC PAPI_L3_TCA',
'PAPI_BR_PRC PAPI_L3_TCM',
'PAPI_BR_PRC PAPI_L3_TCR',
'PAPI_BR_PRC PAPI_L3_TCW',
'PAPI_BR_PRC PAPI_LD_INS',
'PAPI_BR_PRC PAPI_REF_CYC',
'PAPI_BR_PRC PAPI_SP_OPS',
'PAPI_BR_PRC PAPI_SR_INS',
'PAPI_BR_PRC PAPI_STL_ICY',
'PAPI_BR_PRC PAPI_TLB_DM',
'PAPI_BR_PRC PAPI_TLB_IM',
'PAPI_BR_PRC PAPI_TOT_CYC',
'PAPI_BR_PRC PAPI_TOT_INS',
'PAPI_BR_PRC PAPI_VEC_DP',
'PAPI_BR_PRC PAPI_VEC_SP',
'PAPI_BR_PRC add(Object)',
'PAPI_BR_PRC add(int, Object)',
'PAPI_BR_PRC addAll(Collection)',
'PAPI_BR_PRC addAll(int, Collection)',
'PAPI_BR_PRC clear()',
'PAPI_BR_PRC contains(Object)',
'PAPI_BR_PRC containsAll(Collection)',
'PAPI_BR_PRC containsKey(Object)',
'PAPI_BR_PRC containsValue(Object)',
'PAPI_BR_PRC entrySet()',
'PAPI_BR_PRC equals(Object)',
'PAPI_BR_PRC get(Object)',
'PAPI_BR_PRC hashCode()',
'PAPI_BR_PRC indexOf(Object)',
'PAPI_BR_PRC isEmpty()',
'PAPI_BR_PRC iterator()',
'PAPI_BR_PRC keySet()',
'PAPI_BR_PRC lastIndexOf(Object)',
'PAPI_BR_PRC listIterator()',
'PAPI_BR_PRC listIterator(int)',
'PAPI_BR_PRC put(Object, Object)',
'PAPI_BR_PRC putAll(Map)',
'PAPI_BR_PRC remove(Object)',
'PAPI_BR_PRC remove(int)',
'PAPI_BR_PRC removeAll(Collection)',
'PAPI_BR_PRC retainAll(Collection)',
'PAPI_BR_PRC set(int, Object)',
'PAPI_BR_PRC size()',
'PAPI_BR_PRC sort(Comparator)',
'PAPI_BR_PRC subList(int, int)',
'PAPI_BR_PRC toArray()',
'PAPI_BR_PRC toArray(Object[])',
'PAPI_BR_PRC values()',
'PAPI_BR_PRC ArrayList',
'PAPI_BR_PRC HashMap',
'PAPI_BR_PRC HashSet',
'PAPI_BR_PRC LinkedHashMap',
'PAPI_BR_PRC LinkedHashSet',
'PAPI_BR_PRC LinkedList',
'PAPI_BR_PRC TreeMap',
'PAPI_BR_PRC TreeSet',
'PAPI_BR_PRC Vector',
'PAPI_BR_TKN^2',
'PAPI_BR_TKN PAPI_BR_UCN',
'PAPI_BR_TKN PAPI_DP_OPS',
'PAPI_BR_TKN PAPI_FDV_INS',
'PAPI_BR_TKN PAPI_FP_INS',
'PAPI_BR_TKN PAPI_FP_OPS',
'PAPI_BR_TKN PAPI_L1_DCM',
'PAPI_BR_TKN PAPI_L1_ICM',
'PAPI_BR_TKN PAPI_L1_LDM',
'PAPI_BR_TKN PAPI_L1_STM',
'PAPI_BR_TKN PAPI_L1_TCM',
'PAPI_BR_TKN PAPI_L2_DCA',
'PAPI_BR_TKN PAPI_L2_DCH',
'PAPI_BR_TKN PAPI_L2_DCM',
'PAPI_BR_TKN PAPI_L2_DCR',
'PAPI_BR_TKN PAPI_L2_DCW',
'PAPI_BR_TKN PAPI_L2_ICA',
'PAPI_BR_TKN PAPI_L2_ICH',
'PAPI_BR_TKN PAPI_L2_ICM',
'PAPI_BR_TKN PAPI_L2_ICR',
'PAPI_BR_TKN PAPI_L2_STM',
'PAPI_BR_TKN PAPI_L2_TCA',
'PAPI_BR_TKN PAPI_L2_TCM',
'PAPI_BR_TKN PAPI_L2_TCR',
'PAPI_BR_TKN PAPI_L2_TCW',
'PAPI_BR_TKN PAPI_L3_DCA',
'PAPI_BR_TKN PAPI_L3_DCR',
'PAPI_BR_TKN PAPI_L3_DCW',
'PAPI_BR_TKN PAPI_L3_ICA',
'PAPI_BR_TKN PAPI_L3_ICR',
'PAPI_BR_TKN PAPI_L3_TCA',
'PAPI_BR_TKN PAPI_L3_TCM',
'PAPI_BR_TKN PAPI_L3_TCR',
'PAPI_BR_TKN PAPI_L3_TCW',
'PAPI_BR_TKN PAPI_LD_INS',
'PAPI_BR_TKN PAPI_REF_CYC',
'PAPI_BR_TKN PAPI_SP_OPS',
'PAPI_BR_TKN PAPI_SR_INS',
'PAPI_BR_TKN PAPI_STL_ICY',
'PAPI_BR_TKN PAPI_TLB_DM',
'PAPI_BR_TKN PAPI_TLB_IM',
'PAPI_BR_TKN PAPI_TOT_CYC',
'PAPI_BR_TKN PAPI_TOT_INS',
'PAPI_BR_TKN PAPI_VEC_DP',
'PAPI_BR_TKN PAPI_VEC_SP',
'PAPI_BR_TKN add(Object)',
'PAPI_BR_TKN add(int, Object)',
'PAPI_BR_TKN addAll(Collection)',
'PAPI_BR_TKN addAll(int, Collection)',
'PAPI_BR_TKN clear()',
'PAPI_BR_TKN contains(Object)',
'PAPI_BR_TKN containsAll(Collection)',
'PAPI_BR_TKN containsKey(Object)',
'PAPI_BR_TKN containsValue(Object)',
'PAPI_BR_TKN entrySet()',
'PAPI_BR_TKN equals(Object)',
'PAPI_BR_TKN get(Object)',
'PAPI_BR_TKN hashCode()',
'PAPI_BR_TKN indexOf(Object)',
'PAPI_BR_TKN isEmpty()',
'PAPI_BR_TKN iterator()',
'PAPI_BR_TKN keySet()',
'PAPI_BR_TKN lastIndexOf(Object)',
'PAPI_BR_TKN listIterator()',
'PAPI_BR_TKN listIterator(int)',
'PAPI_BR_TKN put(Object, Object)',
'PAPI_BR_TKN putAll(Map)',
'PAPI_BR_TKN remove(Object)',
'PAPI_BR_TKN remove(int)',
'PAPI_BR_TKN removeAll(Collection)',
'PAPI_BR_TKN retainAll(Collection)',
'PAPI_BR_TKN set(int, Object)',
'PAPI_BR_TKN size()',
'PAPI_BR_TKN sort(Comparator)',
'PAPI_BR_TKN subList(int, int)',
'PAPI_BR_TKN toArray()',
'PAPI_BR_TKN toArray(Object[])',
'PAPI_BR_TKN values()',
'PAPI_BR_TKN ArrayList',
'PAPI_BR_TKN HashMap',
'PAPI_BR_TKN HashSet',
'PAPI_BR_TKN LinkedHashMap',
'PAPI_BR_TKN LinkedHashSet',
'PAPI_BR_TKN LinkedList',
'PAPI_BR_TKN TreeMap',
'PAPI_BR_TKN TreeSet',
'PAPI_BR_TKN Vector',
'PAPI_BR_UCN^2',
'PAPI_BR_UCN PAPI_DP_OPS',
'PAPI_BR_UCN PAPI_FDV_INS',
'PAPI_BR_UCN PAPI_FP_INS',
'PAPI_BR_UCN PAPI_FP_OPS',
'PAPI_BR_UCN PAPI_L1_DCM',
'PAPI_BR_UCN PAPI_L1_ICM',
'PAPI_BR_UCN PAPI_L1_LDM',
'PAPI_BR_UCN PAPI_L1_STM',
'PAPI_BR_UCN PAPI_L1_TCM',
'PAPI_BR_UCN PAPI_L2_DCA',
'PAPI_BR_UCN PAPI_L2_DCH',
'PAPI_BR_UCN PAPI_L2_DCM',
'PAPI_BR_UCN PAPI_L2_DCR',
'PAPI_BR_UCN PAPI_L2_DCW',
'PAPI_BR_UCN PAPI_L2_ICA',
'PAPI_BR_UCN PAPI_L2_ICH',
'PAPI_BR_UCN PAPI_L2_ICM',
'PAPI_BR_UCN PAPI_L2_ICR',
'PAPI_BR_UCN PAPI_L2_STM',
'PAPI_BR_UCN PAPI_L2_TCA',
'PAPI_BR_UCN PAPI_L2_TCM',
'PAPI_BR_UCN PAPI_L2_TCR',
'PAPI_BR_UCN PAPI_L2_TCW',
'PAPI_BR_UCN PAPI_L3_DCA',
'PAPI_BR_UCN PAPI_L3_DCR',
'PAPI_BR_UCN PAPI_L3_DCW',
'PAPI_BR_UCN PAPI_L3_ICA',
'PAPI_BR_UCN PAPI_L3_ICR',
'PAPI_BR_UCN PAPI_L3_TCA',
'PAPI_BR_UCN PAPI_L3_TCM',
'PAPI_BR_UCN PAPI_L3_TCR',
'PAPI_BR_UCN PAPI_L3_TCW',
'PAPI_BR_UCN PAPI_LD_INS',
'PAPI_BR_UCN PAPI_REF_CYC',
'PAPI_BR_UCN PAPI_SP_OPS',
'PAPI_BR_UCN PAPI_SR_INS',
'PAPI_BR_UCN PAPI_STL_ICY',
'PAPI_BR_UCN PAPI_TLB_DM',
'PAPI_BR_UCN PAPI_TLB_IM',
'PAPI_BR_UCN PAPI_TOT_CYC',
'PAPI_BR_UCN PAPI_TOT_INS',
'PAPI_BR_UCN PAPI_VEC_DP',
'PAPI_BR_UCN PAPI_VEC_SP',
'PAPI_BR_UCN add(Object)',
'PAPI_BR_UCN add(int, Object)',
'PAPI_BR_UCN addAll(Collection)',
'PAPI_BR_UCN addAll(int, Collection)',
'PAPI_BR_UCN clear()',
'PAPI_BR_UCN contains(Object)',
'PAPI_BR_UCN containsAll(Collection)',
'PAPI_BR_UCN containsKey(Object)',
'PAPI_BR_UCN containsValue(Object)',
'PAPI_BR_UCN entrySet()',
'PAPI_BR_UCN equals(Object)',
'PAPI_BR_UCN get(Object)',
'PAPI_BR_UCN hashCode()',
'PAPI_BR_UCN indexOf(Object)',
'PAPI_BR_UCN isEmpty()',
'PAPI_BR_UCN iterator()',
'PAPI_BR_UCN keySet()',
'PAPI_BR_UCN lastIndexOf(Object)',
'PAPI_BR_UCN listIterator()',
'PAPI_BR_UCN listIterator(int)',
'PAPI_BR_UCN put(Object, Object)',
'PAPI_BR_UCN putAll(Map)',
'PAPI_BR_UCN remove(Object)',
'PAPI_BR_UCN remove(int)',
'PAPI_BR_UCN removeAll(Collection)',
'PAPI_BR_UCN retainAll(Collection)',
'PAPI_BR_UCN set(int, Object)',
'PAPI_BR_UCN size()',
'PAPI_BR_UCN sort(Comparator)',
'PAPI_BR_UCN subList(int, int)',
'PAPI_BR_UCN toArray()',
'PAPI_BR_UCN toArray(Object[])',
'PAPI_BR_UCN values()',
'PAPI_BR_UCN ArrayList',
'PAPI_BR_UCN HashMap',
'PAPI_BR_UCN HashSet',
'PAPI_BR_UCN LinkedHashMap',
'PAPI_BR_UCN LinkedHashSet',
'PAPI_BR_UCN LinkedList',
'PAPI_BR_UCN TreeMap',
'PAPI_BR_UCN TreeSet',
'PAPI_BR_UCN Vector',
'PAPI_DP_OPS^2',
'PAPI_DP_OPS PAPI_FDV_INS',
'PAPI_DP_OPS PAPI_FP_INS',
'PAPI_DP_OPS PAPI_FP_OPS',
'PAPI_DP_OPS PAPI_L1_DCM',
'PAPI_DP_OPS PAPI_L1_ICM',
'PAPI_DP_OPS PAPI_L1_LDM',
'PAPI_DP_OPS PAPI_L1_STM',
'PAPI_DP_OPS PAPI_L1_TCM',
'PAPI_DP_OPS PAPI_L2_DCA',
'PAPI_DP_OPS PAPI_L2_DCH',
'PAPI_DP_OPS PAPI_L2_DCM',
'PAPI_DP_OPS PAPI_L2_DCR',
'PAPI_DP_OPS PAPI_L2_DCW',
'PAPI_DP_OPS PAPI_L2_ICA',
'PAPI_DP_OPS PAPI_L2_ICH',
'PAPI_DP_OPS PAPI_L2_ICM',
'PAPI_DP_OPS PAPI_L2_ICR',
'PAPI_DP_OPS PAPI_L2_STM',
'PAPI_DP_OPS PAPI_L2_TCA',
'PAPI_DP_OPS PAPI_L2_TCM',
'PAPI_DP_OPS PAPI_L2_TCR',
'PAPI_DP_OPS PAPI_L2_TCW',
'PAPI_DP_OPS PAPI_L3_DCA',
'PAPI_DP_OPS PAPI_L3_DCR',
'PAPI_DP_OPS PAPI_L3_DCW',
'PAPI_DP_OPS PAPI_L3_ICA',
'PAPI_DP_OPS PAPI_L3_ICR',
'PAPI_DP_OPS PAPI_L3_TCA',
'PAPI_DP_OPS PAPI_L3_TCM',
'PAPI_DP_OPS PAPI_L3_TCR',
'PAPI_DP_OPS PAPI_L3_TCW',
'PAPI_DP_OPS PAPI_LD_INS',
'PAPI_DP_OPS PAPI_REF_CYC',
'PAPI_DP_OPS PAPI_SP_OPS',
'PAPI_DP_OPS PAPI_SR_INS',
'PAPI_DP_OPS PAPI_STL_ICY',
'PAPI_DP_OPS PAPI_TLB_DM',
'PAPI_DP_OPS PAPI_TLB_IM',
'PAPI_DP_OPS PAPI_TOT_CYC',
'PAPI_DP_OPS PAPI_TOT_INS',
'PAPI_DP_OPS PAPI_VEC_DP',
'PAPI_DP_OPS PAPI_VEC_SP',
'PAPI_DP_OPS add(Object)',
'PAPI_DP_OPS add(int, Object)',
'PAPI_DP_OPS addAll(Collection)',
'PAPI_DP_OPS addAll(int, Collection)',
'PAPI_DP_OPS clear()',
'PAPI_DP_OPS contains(Object)',
'PAPI_DP_OPS containsAll(Collection)',
'PAPI_DP_OPS containsKey(Object)',
'PAPI_DP_OPS containsValue(Object)',
'PAPI_DP_OPS entrySet()',
'PAPI_DP_OPS equals(Object)',
'PAPI_DP_OPS get(Object)',
'PAPI_DP_OPS hashCode()',
'PAPI_DP_OPS indexOf(Object)',
'PAPI_DP_OPS isEmpty()',
'PAPI_DP_OPS iterator()',
'PAPI_DP_OPS keySet()',
'PAPI_DP_OPS lastIndexOf(Object)',
'PAPI_DP_OPS listIterator()',
'PAPI_DP_OPS listIterator(int)',
'PAPI_DP_OPS put(Object, Object)',
'PAPI_DP_OPS putAll(Map)',
'PAPI_DP_OPS remove(Object)',
'PAPI_DP_OPS remove(int)',
'PAPI_DP_OPS removeAll(Collection)',
'PAPI_DP_OPS retainAll(Collection)',
'PAPI_DP_OPS set(int, Object)',
'PAPI_DP_OPS size()',
'PAPI_DP_OPS sort(Comparator)',
'PAPI_DP_OPS subList(int, int)',
'PAPI_DP_OPS toArray()',
'PAPI_DP_OPS toArray(Object[])',
'PAPI_DP_OPS values()',
'PAPI_DP_OPS ArrayList',
'PAPI_DP_OPS HashMap',
'PAPI_DP_OPS HashSet',
'PAPI_DP_OPS LinkedHashMap',
'PAPI_DP_OPS LinkedHashSet',
'PAPI_DP_OPS LinkedList',
'PAPI_DP_OPS TreeMap',
'PAPI_DP_OPS TreeSet',
'PAPI_DP_OPS Vector',
'PAPI_FDV_INS^2',
'PAPI_FDV_INS PAPI_FP_INS',
'PAPI_FDV_INS PAPI_FP_OPS',
'PAPI_FDV_INS PAPI_L1_DCM',
'PAPI_FDV_INS PAPI_L1_ICM',
'PAPI_FDV_INS PAPI_L1_LDM',
'PAPI_FDV_INS PAPI_L1_STM',
'PAPI_FDV_INS PAPI_L1_TCM',
'PAPI_FDV_INS PAPI_L2_DCA',
'PAPI_FDV_INS PAPI_L2_DCH',
'PAPI_FDV_INS PAPI_L2_DCM',
'PAPI_FDV_INS PAPI_L2_DCR',
'PAPI_FDV_INS PAPI_L2_DCW',
'PAPI_FDV_INS PAPI_L2_ICA',
'PAPI_FDV_INS PAPI_L2_ICH',
'PAPI_FDV_INS PAPI_L2_ICM',
'PAPI_FDV_INS PAPI_L2_ICR',
'PAPI_FDV_INS PAPI_L2_STM',
'PAPI_FDV_INS PAPI_L2_TCA',
'PAPI_FDV_INS PAPI_L2_TCM',
'PAPI_FDV_INS PAPI_L2_TCR',
'PAPI_FDV_INS PAPI_L2_TCW',
'PAPI_FDV_INS PAPI_L3_DCA',
'PAPI_FDV_INS PAPI_L3_DCR',
'PAPI_FDV_INS PAPI_L3_DCW',
'PAPI_FDV_INS PAPI_L3_ICA',
'PAPI_FDV_INS PAPI_L3_ICR',
'PAPI_FDV_INS PAPI_L3_TCA',
'PAPI_FDV_INS PAPI_L3_TCM',
'PAPI_FDV_INS PAPI_L3_TCR',
'PAPI_FDV_INS PAPI_L3_TCW',
'PAPI_FDV_INS PAPI_LD_INS',
'PAPI_FDV_INS PAPI_REF_CYC',
'PAPI_FDV_INS PAPI_SP_OPS',
'PAPI_FDV_INS PAPI_SR_INS',
'PAPI_FDV_INS PAPI_STL_ICY',
'PAPI_FDV_INS PAPI_TLB_DM',
'PAPI_FDV_INS PAPI_TLB_IM',
'PAPI_FDV_INS PAPI_TOT_CYC',
'PAPI_FDV_INS PAPI_TOT_INS',
'PAPI_FDV_INS PAPI_VEC_DP',
'PAPI_FDV_INS PAPI_VEC_SP',
'PAPI_FDV_INS add(Object)',
'PAPI_FDV_INS add(int, Object)',
'PAPI_FDV_INS addAll(Collection)',
'PAPI_FDV_INS addAll(int, Collection)',
'PAPI_FDV_INS clear()',
'PAPI_FDV_INS contains(Object)',
'PAPI_FDV_INS containsAll(Collection)',
'PAPI_FDV_INS containsKey(Object)',
'PAPI_FDV_INS containsValue(Object)',
'PAPI_FDV_INS entrySet()',
'PAPI_FDV_INS equals(Object)',
'PAPI_FDV_INS get(Object)',
'PAPI_FDV_INS hashCode()',
'PAPI_FDV_INS indexOf(Object)',
'PAPI_FDV_INS isEmpty()',
'PAPI_FDV_INS iterator()',
'PAPI_FDV_INS keySet()',
'PAPI_FDV_INS lastIndexOf(Object)',
'PAPI_FDV_INS listIterator()',
'PAPI_FDV_INS listIterator(int)',
'PAPI_FDV_INS put(Object, Object)',
'PAPI_FDV_INS putAll(Map)',
'PAPI_FDV_INS remove(Object)',
'PAPI_FDV_INS remove(int)',
'PAPI_FDV_INS removeAll(Collection)',
'PAPI_FDV_INS retainAll(Collection)',
'PAPI_FDV_INS set(int, Object)',
'PAPI_FDV_INS size()',
'PAPI_FDV_INS sort(Comparator)',
'PAPI_FDV_INS subList(int, int)',
'PAPI_FDV_INS toArray()',
'PAPI_FDV_INS toArray(Object[])',
'PAPI_FDV_INS values()',
'PAPI_FDV_INS ArrayList',
'PAPI_FDV_INS HashMap',
'PAPI_FDV_INS HashSet',
'PAPI_FDV_INS LinkedHashMap',
'PAPI_FDV_INS LinkedHashSet',
'PAPI_FDV_INS LinkedList',
'PAPI_FDV_INS TreeMap',
'PAPI_FDV_INS TreeSet',
'PAPI_FDV_INS Vector',
'PAPI_FP_INS^2',
'PAPI_FP_INS PAPI_FP_OPS',
'PAPI_FP_INS PAPI_L1_DCM',
'PAPI_FP_INS PAPI_L1_ICM',
'PAPI_FP_INS PAPI_L1_LDM',
'PAPI_FP_INS PAPI_L1_STM',
'PAPI_FP_INS PAPI_L1_TCM',
'PAPI_FP_INS PAPI_L2_DCA',
'PAPI_FP_INS PAPI_L2_DCH',
'PAPI_FP_INS PAPI_L2_DCM',
'PAPI_FP_INS PAPI_L2_DCR',
'PAPI_FP_INS PAPI_L2_DCW',
'PAPI_FP_INS PAPI_L2_ICA',
'PAPI_FP_INS PAPI_L2_ICH',
'PAPI_FP_INS PAPI_L2_ICM',
'PAPI_FP_INS PAPI_L2_ICR',
'PAPI_FP_INS PAPI_L2_STM',
'PAPI_FP_INS PAPI_L2_TCA',
'PAPI_FP_INS PAPI_L2_TCM',
'PAPI_FP_INS PAPI_L2_TCR',
'PAPI_FP_INS PAPI_L2_TCW',
'PAPI_FP_INS PAPI_L3_DCA',
'PAPI_FP_INS PAPI_L3_DCR',
'PAPI_FP_INS PAPI_L3_DCW',
'PAPI_FP_INS PAPI_L3_ICA',
'PAPI_FP_INS PAPI_L3_ICR',
'PAPI_FP_INS PAPI_L3_TCA',
'PAPI_FP_INS PAPI_L3_TCM',
'PAPI_FP_INS PAPI_L3_TCR',
'PAPI_FP_INS PAPI_L3_TCW',
'PAPI_FP_INS PAPI_LD_INS',
'PAPI_FP_INS PAPI_REF_CYC',
'PAPI_FP_INS PAPI_SP_OPS',
'PAPI_FP_INS PAPI_SR_INS',
'PAPI_FP_INS PAPI_STL_ICY',
'PAPI_FP_INS PAPI_TLB_DM',
'PAPI_FP_INS PAPI_TLB_IM',
'PAPI_FP_INS PAPI_TOT_CYC',
'PAPI_FP_INS PAPI_TOT_INS',
'PAPI_FP_INS PAPI_VEC_DP',
'PAPI_FP_INS PAPI_VEC_SP',
'PAPI_FP_INS add(Object)',
'PAPI_FP_INS add(int, Object)',
'PAPI_FP_INS addAll(Collection)',
'PAPI_FP_INS addAll(int, Collection)',
'PAPI_FP_INS clear()',
'PAPI_FP_INS contains(Object)',
'PAPI_FP_INS containsAll(Collection)',
'PAPI_FP_INS containsKey(Object)',
'PAPI_FP_INS containsValue(Object)',
'PAPI_FP_INS entrySet()',
'PAPI_FP_INS equals(Object)',
'PAPI_FP_INS get(Object)',
'PAPI_FP_INS hashCode()',
'PAPI_FP_INS indexOf(Object)',
'PAPI_FP_INS isEmpty()',
'PAPI_FP_INS iterator()',
'PAPI_FP_INS keySet()',
'PAPI_FP_INS lastIndexOf(Object)',
'PAPI_FP_INS listIterator()',
'PAPI_FP_INS listIterator(int)',
'PAPI_FP_INS put(Object, Object)',
'PAPI_FP_INS putAll(Map)',
'PAPI_FP_INS remove(Object)',
'PAPI_FP_INS remove(int)',
'PAPI_FP_INS removeAll(Collection)',
'PAPI_FP_INS retainAll(Collection)',
'PAPI_FP_INS set(int, Object)',
'PAPI_FP_INS size()',
'PAPI_FP_INS sort(Comparator)',
'PAPI_FP_INS subList(int, int)',
'PAPI_FP_INS toArray()',
'PAPI_FP_INS toArray(Object[])',
'PAPI_FP_INS values()',
'PAPI_FP_INS ArrayList',
'PAPI_FP_INS HashMap',
'PAPI_FP_INS HashSet',
'PAPI_FP_INS LinkedHashMap',
'PAPI_FP_INS LinkedHashSet',
'PAPI_FP_INS LinkedList',
'PAPI_FP_INS TreeMap',
'PAPI_FP_INS TreeSet',
'PAPI_FP_INS Vector',
'PAPI_FP_OPS^2',
'PAPI_FP_OPS PAPI_L1_DCM',
'PAPI_FP_OPS PAPI_L1_ICM',
'PAPI_FP_OPS PAPI_L1_LDM',
'PAPI_FP_OPS PAPI_L1_STM',
'PAPI_FP_OPS PAPI_L1_TCM',
'PAPI_FP_OPS PAPI_L2_DCA',
'PAPI_FP_OPS PAPI_L2_DCH',
'PAPI_FP_OPS PAPI_L2_DCM',
'PAPI_FP_OPS PAPI_L2_DCR',
'PAPI_FP_OPS PAPI_L2_DCW',
'PAPI_FP_OPS PAPI_L2_ICA',
'PAPI_FP_OPS PAPI_L2_ICH',
'PAPI_FP_OPS PAPI_L2_ICM',
'PAPI_FP_OPS PAPI_L2_ICR',
'PAPI_FP_OPS PAPI_L2_STM',
'PAPI_FP_OPS PAPI_L2_TCA',
'PAPI_FP_OPS PAPI_L2_TCM',
'PAPI_FP_OPS PAPI_L2_TCR',
'PAPI_FP_OPS PAPI_L2_TCW',
'PAPI_FP_OPS PAPI_L3_DCA',
'PAPI_FP_OPS PAPI_L3_DCR',
'PAPI_FP_OPS PAPI_L3_DCW',
'PAPI_FP_OPS PAPI_L3_ICA',
'PAPI_FP_OPS PAPI_L3_ICR',
'PAPI_FP_OPS PAPI_L3_TCA',
'PAPI_FP_OPS PAPI_L3_TCM',
'PAPI_FP_OPS PAPI_L3_TCR',
'PAPI_FP_OPS PAPI_L3_TCW',
'PAPI_FP_OPS PAPI_LD_INS',
'PAPI_FP_OPS PAPI_REF_CYC',
'PAPI_FP_OPS PAPI_SP_OPS',
...]
%% Cell type:code id: tags:
``` python
# Prepare the data:
# We need a data frame mapping the name of features to their importance.
# Then maybe we need to only display the five / ten most important features.
feature_importances = pd.DataFrame()
feature_importances["name"] = model.feature_names
feature_importances["importance"] = model.feature_importances_
data_sorted = feature_importances.sort_values(by="importance", ascending=False).head(10)
sns.barplot(data=data_sorted, x="importance", y="name")
```
%% Output
<matplotlib.axes._subplots.AxesSubplot at 0x7ffa14924910>
%% Cell type:markdown id: tags:
The above plot is not that easy to read. `PAPI_REF_CYC * remove(int)`? What am I supposed to do with that?
Maybe it would be better to avoid using polynomial features if we want an explainable model.
`PAPI_STL_ICY` refers to the number of cycles with no instruction issued. Maybe this is actually a good indicator of poor performance. It is hard to see how this value might be used.
%% Cell type:markdown id: tags:
## How does the model compare with the "dumb classifier"
The "dumb classifier" always predicts that the data-structure that wins most often in the experiments is the right one. How often does my model disagree with it?!
%% Cell type:code id: tags:
``` python
# Step one, how do we find the "dumb classifier"?
# The dumb classifier is a function from the type of interface to the most common winning implementation of that interface.
# So we need to group rows by the type of the implemented interface, and then we can select the most common winning one.
grouped_interface = data["data"].groupby("Benchmark")
# I need to map the number of times the data structure shows up in best data structure for each group.
dumb_classification_function = {}
for label, group in grouped_interface:
dumb_classification_function[label] = group["Param: datastructureName_best"].value_counts().argmax()
print(dumb_classification_function)
```
%% Output
{'se.lth.cs.jmh.MapApplicationBenchmark.MapApplicationBenchmark': 'TreeMap', 'se.lth.cs.jmh.SetApplicationBenchmark.SetApplicationBenchmark': 'LinkedHashSet', 'se.lth.cs.jmh.ListApplicationBenchmark.ListApplicationBenchmark': 'ArrayList'}
/home/noric/Dev/jbrainy/env/lib/python2.7/site-packages/ipykernel_launcher.py:8: FutureWarning:
The current behaviour of 'Series.argmax' is deprecated, use 'idxmax'
instead.
The behavior of 'argmax' will be corrected to return the positional
maximum in the future. For now, use 'series.values.argmax' or
'np.argmax(np.array(values))' to get the position of the maximum
row.
%% Cell type:code id: tags:
``` python
# Now that we know how to find the "dumb classification", we can try to look for how often it disagrees
# with the random forest.
# We need to make a column with the result of the "classification".
# We make a function for the classifier
classify = lambda row: dumb_classification_function[row["Benchmark"]]
classified = data["data"].apply(classify, axis=1)
comparison = (data["data"]["Param: datastructureName_best"] == classified).value_counts()
# We plot them
sns.barplot(x=comparison.index, y=comparison)
```
%% Output
<matplotlib.axes._subplots.AxesSubplot at 0x7ffa0d97f950>
%% Cell type:markdown id: tags:
We can see that the "dumb classifier" agrees with the "smarter" one more often than not. Overall, it would be interesting to try to see if the classification effort is really worth it.
For instance, we may want to know which one of the dumb classifier and the random forest are the most accurate (it is pretty likely that it is the random forest). But it might be interesting which ones wins in practice.
%% Cell type:markdown id: tags:
## How does the model compare with using Neural networks for the same thing?
Since the Brainy guys use a neural network, how come I don't use one?
SPOILER: I hope to show that in fact, you do not need a neural network!
%% Cell type:code id: tags:
``` python
# TODO.
```
Learning2-skimmed.ipynb
View file @ 724a22e7
This diff is collapsed.
train_model.py
View file @ 724a22e7
......@@ -7,6 +7,8 @@ import pandas as pd
import numpy
import pickle
import os.path # Checking existence of paths
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import matthews_corrcoef
......@@ -103,15 +105,20 @@ def load_jmh_data(filename):
worst_case_times = jmh_data["Score"] - jmh_data["Score Error (99.9%)"]
jmh_data["Lowest score"] = worst_case_times
# We filter out the jmh rows where the lowest score is equal to zero
# This is potentially why some values were dropped in the resulting table,
# even if their data structure type was not a WeakHashMap or an
# IdentityHashMap
jmh_data_filtered = jmh_data[jmh_data["Lowest score"] > 0]
# Grouping the applications, to compare similar ones.
selected_jmh_columns = ["Param: seed",
"Param: applicationSize",
"Param: baseStructureSize",
"Param: baseStructureSize"
"Benchmark"]
# Best data structures
jmh_best_structures = jmh_data_filtered.sort_values(by="Lowest score",
ascending=False).drop_duplicates(selected_jmh_columns)
jmh_best_structures = jmh_data_filtered\
.sort_values(by="Lowest score",
ascending=False)\
.drop_duplicates(selected_jmh_columns)
jmh_best_structures = jmh_best_structures.reset_index(drop=True)
# # Best data structures, computing the improvement
......@@ -136,7 +143,7 @@ def compute_sample_weights(jmh_data):
The weight is computer by computing the ratio of improvement,
and adding it's log to 1/N, where N is the number of samples
"""
ratios = jmh_data["Lowest score_best"] / jmh_with_best["Lowest score"]
ratios = jmh_data["Lowest score_best"] / jmh_data["Lowest score"]
sample_weights = (1 / ratios.shape[0]) + numpy.log(ratios)
return sample_weights
......@@ -255,23 +262,35 @@ def normalize_data(data, selected_columns, normalizing_column):
#%%
if __name__ == "__main__":
# The script takes a number of files as input
# Loads the JMH data, SW & HW performance counters
# Merges them to get a well built data file
# And train the classifier.
#%%
print("Loading training data...")
jmh_with_best = load_jmh_data("data/jmh-results-34f6dc24.csv")
def load_training_data(jmh_results_filename,
software_counters_filename,
hardware_counters_filename):
""" A function that loads the training data for training JBrainy
input: - The name of the file containing the JMH benchmark data
- The name of the software counters file
- The name of the hardware counters filename
returns: A report containing the data
and some metadata, like the selected SW columns and HW columns
and the names of the data structures we will be dealing with
"""
# Precondition: All the provided files must exist.
for file in [jmh_results_filename,
software_counters_filename,
hardware_counters_filename]:
if os.path.isfile(file):
# OK.
continue
else:
raise "The path '{0}' does not exist".format(file)
# Ok here we go
jmh_with_best = load_jmh_data(jmh_results_filename)
jmh_with_best["Sample weight"] = compute_sample_weights(jmh_with_best)
software_data = load_software_counters("data/software-perf.csv")
software_data = load_software_counters(software_counters_filename)
software_with_jmh = merge_jmh_software(jmh_with_best, software_data)
software_selected_columns = [c for c in software_with_jmh.columns if "run" in c]
software_selected_columns = software_data.columns
software_features = software_with_jmh.get(software_selected_columns)
data_structure_names=software_with_jmh["Param: datastructureName_best"].sort_values().unique()
papi_data = load_hardware_counters("data/hardware-perf-50.csv")
papi_data = load_hardware_counters(hardware_counters_filename)
papi_data['size'] = jmh_with_best['Param: applicationSize']
hardware_selected_columns = [c for c in papi_data.columns if "PAPI" in c]
hardware_features = papi_data.get(hardware_selected_columns)
......@@ -280,10 +299,31 @@ if __name__ == "__main__":
# apps where they were the best choice and apps where they were used.
software_plus_hardware = software_with_jmh.join(hardware_features)
sw_hw_cleaned = remove_weakmap_hashmap(software_plus_hardware)
return { "data" : sw_hw_cleaned,
"data_structure_names" : software_plus_hardware["Param: datastructureName_best"].sort_values().unique(),
"software_selected_columns": software_selected_columns,
"hardware_selected_columns": hardware_selected_columns }
if __name__ == "__main__":
# The script takes a number of files as input
# Loads the JMH data, SW & HW performance counters
# Merges them to get a well built data file
# And train the classifier.
#%%
print("Loading training data...")
training_data = load_training_data("data/jmh-results-9307f70f.csv",
"software-perf.csv",
"data/hardware-perf-data.csv")
print("Training data loaded")
#%%
sw_hw_cleaned = training_data["data"]
hardware_selected_columns = training_data["hardware_selected_columns"]
software_selected_columns = training_data["software_selected_columns"]
print("Normalizing data")
hw_normalized_data = normalize_data(sw_hw_cleaned,
hardware_selected_columns,
......@@ -309,7 +349,8 @@ if __name__ == "__main__":
axis=1)
poly_transformer = PolynomialFeatures(degree=2)
features_extended_poly = poly_transformer.fit_transform(features_extended)
features_extended_poly = poly_transformer.fit_transform(features_extended,
features_extended.columns.values)
features_extended_norm = StandardScaler().fit_transform(features_extended_poly)
labels = sw_hw_cleaned["Param: datastructureName_best"]
......@@ -332,6 +373,11 @@ if __name__ == "__main__":
classifier = RandomForestClassifier()
classifier.fit(X_train, y_train, w_train)
# We just add it to the class
classifier.feature_names = poly_transformer \
.get_feature_names(features_extended.columns.values)
print("Accuracy: {0}".format(classifier.score(X_test, y_test, w_test)))
print("Classifier trained")
......