Commit fa290b1c authored by Konstantinos Chatzilygeroudis's avatar Konstantinos Chatzilygeroudis
Browse files

Fixed json and added meaningful variants

parent 310ea01b
......@@ -146,6 +146,8 @@ def compile_regression_benchmarks(bld, json_file):
optimizer = [optimizer]
if len(optimizer_params) > 0:
optimizer_params = [optimizer_params]
if not isinstance(optimizer_params, types.ListType):
optimizer_params = [optimizer_params]
kernel_find = kernel_type.rfind('::')
mean_find = mean_type.rfind('::')
......
......@@ -139,8 +139,13 @@ def plot_ax(ax, data, points, labely, disp_legend=True, disp_xaxis=False):
labels = []
replicates = 0
kk = 0
# sort the variants
sorted_keys = sorted(points.keys(), reverse=True)
# for each variant
for var in points.keys():
for vv in range(len(sorted_keys)):
var = sorted_keys[vv]
labels.append(var)
var_p = points[var]
var_data = data[var]
......@@ -232,13 +237,15 @@ def plot_all():
rst_file.write("- Learning time: lower is better\n")
rst_file.write("- Querying time (for 1 by 1 query points): lower is better\n\n")
rst_file.write("- In each replicate, all the variants (see below for the variants available) are using exactly the same data\n\n")
rst_file.write("- The data are uniformly sampled and some noise is added (according to the variance of the data).\n\n")
rst_file.write("Naming convention\n")
rst_file.write("------------------\n\n")
rst_file.write("- GP-SE-Full: Limbo with Squared Exponential kernel where the signal noise, signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation\n")
rst_file.write("- GP-SE: Same as above but the signal noise is not optimized and set to a default value: 0.01\n")
rst_file.write("- GPy: GPy with Squared Exponential kernel where the signal noise, signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation\n\n")
rst_file.write("- GP-SE-Full-Rprop: Limbo with Squared Exponential kernel where the signal noise, signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation with the Rprop optimizer (default for limbo)\n")
rst_file.write("- GP-SE-Full-SLSQP: Limbo with Squared Exponential kernel where the signal noise, signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation with the SLSQP optimizer (provided by NLOpt)\n")
rst_file.write("- GP-SE-Rprop: Limbo with Squared Exponential kernel where the signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation with the Rprop optimizer (default for limbo) and where the signal noise is not optimized but set to a default value: 0.01\n")
rst_file.write("- GPy: GPy with Squared Exponential kernel where the signal noise, signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation (with the L-BFGS-B optimizer --- `check scipy.optimize.fmin_l_bfgs_b= <https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fmin_l_bfgs_b.html>`_)\n\n")
print('loading data...')
points,times_learn,times_query,mses = load_data()
......
[{ "name" : "regression_benchmark",
"functions" : ["GramacyLee", "Step"],
"dimensions" : [[1], [1] ],
"functions" : ["Rastrigin", "GramacyLee", "Step", "RobotArm", "OTLCircuit", "PistonSimulation", "PlanarInverseDynamicsI", "PlanarInverseDynamicsII"],
"dimensions" : [[1,2,4,8], [1], [1], [8], [6], [7], [6], [6]],
"points" : [50, 100, 200, 400, 600],
"randomness": ["uniform"],
"noise" : "true",
"models" : [{"name" : "GP-SE-Full", "type" : "GP", "kernel" : {"type": "SquaredExpARD", "optimize_noise" : "true", "params" : {"sigma_sq" : ["double", 1.0]}}, "mean" : {"type": "NullFunction", "has_defaults" : "false"}, "hp_opt" : {"type" : "KernelLFOpt", "optimizer" : "Rprop", "params" : {"eps_stop" : ["double", 1e-6]}}},
{"name" : "GP-SE", "type" : "GP", "kernel" : {"type": "SquaredExpARD", "optimize_noise" : "false", "params" : {"sigma_sq" : ["double", 1.0]}}, "mean" : {"type": "NullFunction", "has_defaults" : "false"}, "hp_opt" : {"type" : "KernelLFOpt", "optimizer" : "Rprop", "params" : {"eps_stop" : ["double", 1e-6]}}}]
"models" : [{"name" : "GP-SE-Full-Rprop", "type" : "GP", "kernel" : {"type": "SquaredExpARD", "optimize_noise" : "true", "params" : {"sigma_sq" : ["double", 1.0]}}, "mean" : {"type": "NullFunction", "has_defaults" : "false"}, "hp_opt" : {"type" : "KernelLFOpt", "optimizer" : "Rprop", "params" : {"eps_stop" : ["double", 1e-4]}}},
{"name" : "GP-SE-Full-SLSQP", "type" : "GP", "kernel" : {"type": "SquaredExpARD", "optimize_noise" : "true", "params" : {"sigma_sq" : ["double", 1.0]}}, "mean" : {"type": "NullFunction", "has_defaults" : "false"}, "hp_opt" : {"type" : "KernelLFOpt", "optimizer" : ["NLOptGrad", "nlopt::LD_SLSQP"], "params" : {"fun_tolerance" : ["double", 1e-4], "xrel_tolerance" : ["double", 1e-4]}}},
{"name" : "GP-SE-Rprop", "type" : "GP", "kernel" : {"type": "SquaredExpARD", "optimize_noise" : "false", "params" : {"sigma_sq" : ["double", 1.0]}}, "mean" : {"type": "NullFunction", "has_defaults" : "false"}, "hp_opt" : {"type" : "KernelLFOpt", "optimizer" : "Rprop", "params" : {"eps_stop" : ["double", 1e-4]}}}]
}]
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment