Fixed json and added meaningful variants

waf_tools/benchmarks.py

@@ -146,6 +146,8 @@ def compile_regression_benchmarks(bld, json_file):
                 optimizer = [optimizer]
                 if len(optimizer_params) > 0:
                     optimizer_params = [optimizer_params]
+            if not isinstance(optimizer_params, types.ListType):
+                optimizer_params = [optimizer_params]
 
             kernel_find = kernel_type.rfind('::')
             mean_find = mean_type.rfind('::')

waf_tools/plot_regression_benchmarks.py

@@ -139,8 +139,13 @@ def plot_ax(ax, data, points, labely, disp_legend=True, disp_xaxis=False):
     labels = []
     replicates = 0
     kk = 0
+
+    # sort the variants
+    sorted_keys = sorted(points.keys(), reverse=True)
+
     # for each variant
-    for var in points.keys():
+    for vv in range(len(sorted_keys)):
+        var = sorted_keys[vv]
         labels.append(var)
         var_p = points[var]
         var_data = data[var]
@@ -232,13 +237,15 @@ def plot_all():
     rst_file.write("- Learning time: lower is better\n")
     rst_file.write("- Querying time (for 1 by 1 query points): lower is better\n\n")
     rst_file.write("- In each replicate, all the variants (see below for the variants available) are using exactly the same data\n\n")
+    rst_file.write("- The data are uniformly sampled and some noise is added (according to the variance of the data).\n\n")
 
     rst_file.write("Naming convention\n")
     rst_file.write("------------------\n\n")
 
-    rst_file.write("- GP-SE-Full: Limbo with Squared Exponential kernel where the signal noise, signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation\n")
-    rst_file.write("- GP-SE: Same as above but the signal noise is not optimized and set to a default value: 0.01\n")
-    rst_file.write("- GPy: GPy with Squared Exponential kernel where the signal noise, signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation\n\n")
+    rst_file.write("- GP-SE-Full-Rprop: Limbo with Squared Exponential kernel where the signal noise, signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation with the Rprop optimizer (default for limbo)\n")
+    rst_file.write("- GP-SE-Full-SLSQP: Limbo with Squared Exponential kernel where the signal noise, signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation with the SLSQP optimizer (provided by NLOpt)\n")
+    rst_file.write("- GP-SE-Rprop: Limbo with Squared Exponential kernel where the signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation with the Rprop optimizer (default for limbo) and where the signal noise is not optimized but set to a default value: 0.01\n")
+    rst_file.write("- GPy: GPy with Squared Exponential kernel where the signal noise, signal variance and kernel lengthscales are optimized via Maximum Likelihood Estimation (with the L-BFGS-B optimizer --- `check scipy.optimize.fmin_l_bfgs_b= <https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fmin_l_bfgs_b.html>`_)\n\n")
 
     print('loading data...')
     points,times_learn,times_query,mses = load_data()

waf_tools/regression_benchmarks.json

 [{ "name" : "regression_benchmark",
-  "functions" : ["GramacyLee", "Step"],
-  "dimensions" : [[1], [1] ],
+  "functions" : ["Rastrigin", "GramacyLee", "Step", "RobotArm", "OTLCircuit", "PistonSimulation", "PlanarInverseDynamicsI", "PlanarInverseDynamicsII"],
+  "dimensions" : [[1,2,4,8], [1], [1], [8], [6], [7], [6], [6]],
   "points" : [50, 100, 200, 400, 600],
   "randomness": ["uniform"],
   "noise" : "true",
-  "models" : [{"name" : "GP-SE-Full", "type" : "GP", "kernel" : {"type": "SquaredExpARD", "optimize_noise" : "true", "params" : {"sigma_sq" : ["double", 1.0]}}, "mean" : {"type": "NullFunction", "has_defaults" : "false"}, "hp_opt" : {"type" : "KernelLFOpt", "optimizer" : "Rprop", "params" : {"eps_stop" : ["double", 1e-6]}}},
-        {"name" : "GP-SE", "type" : "GP", "kernel" : {"type": "SquaredExpARD", "optimize_noise" : "false", "params" : {"sigma_sq" : ["double", 1.0]}}, "mean" : {"type": "NullFunction", "has_defaults" : "false"}, "hp_opt" : {"type" : "KernelLFOpt", "optimizer" : "Rprop", "params" : {"eps_stop" : ["double", 1e-6]}}}]
+  "models" : [{"name" : "GP-SE-Full-Rprop", "type" : "GP", "kernel" : {"type": "SquaredExpARD", "optimize_noise" : "true", "params" : {"sigma_sq" : ["double", 1.0]}}, "mean" : {"type": "NullFunction", "has_defaults" : "false"}, "hp_opt" : {"type" : "KernelLFOpt", "optimizer" : "Rprop", "params" : {"eps_stop" : ["double", 1e-4]}}},
+  {"name" : "GP-SE-Full-SLSQP", "type" : "GP", "kernel" : {"type": "SquaredExpARD", "optimize_noise" : "true", "params" : {"sigma_sq" : ["double", 1.0]}}, "mean" : {"type": "NullFunction", "has_defaults" : "false"}, "hp_opt" : {"type" : "KernelLFOpt", "optimizer" : ["NLOptGrad", "nlopt::LD_SLSQP"], "params" : {"fun_tolerance" : ["double", 1e-4], "xrel_tolerance" : ["double", 1e-4]}}},
+        {"name" : "GP-SE-Rprop", "type" : "GP", "kernel" : {"type": "SquaredExpARD", "optimize_noise" : "false", "params" : {"sigma_sq" : ["double", 1.0]}}, "mean" : {"type": "NullFunction", "has_defaults" : "false"}, "hp_opt" : {"type" : "KernelLFOpt", "optimizer" : "Rprop", "params" : {"eps_stop" : ["double", 1e-4]}}}]
 }]