Commit 9a4915e3 authored by Jean-Baptiste Mouret's avatar Jean-Baptiste Mouret
Browse files

generate rst file

parent c55ff653
...@@ -40,7 +40,7 @@ Contents: ...@@ -40,7 +40,7 @@ Contents:
guides/index guides/index
api api
defaults defaults
benchmarks bo_benchmarks
faq faq
......
...@@ -46,6 +46,7 @@ ...@@ -46,6 +46,7 @@
#| knowledge of the CeCILL-C license and that you accept its terms. #| knowledge of the CeCILL-C license and that you accept its terms.
#|# plot the results of the Bayesian Optimization benchmarks #|# plot the results of the Bayesian Optimization benchmarks
from glob import glob from glob import glob
import os
from collections import defaultdict from collections import defaultdict
from datetime import datetime from datetime import datetime
import platform import platform
...@@ -103,7 +104,7 @@ def load_data(): ...@@ -103,7 +104,7 @@ def load_data():
for f in files: for f in files:
fs = f.split("/") fs = f.split("/")
func, var, lib = fs[-1], fs[-2], fs[-3] func, var, lib = fs[-1], fs[-2], fs[-3]
print(func, var, lib) #print(func, var, lib)
data[func][lib][var] = np.loadtxt(f) data[func][lib][var] = np.loadtxt(f)
return data return data
...@@ -175,34 +176,56 @@ def plot(func_name, data, rst_file): ...@@ -175,34 +176,56 @@ def plot(func_name, data, rst_file):
bp = ax.boxplot(da_time, 0, 'rs', 0) bp = ax.boxplot(da_time, 0, 'rs', 0)
custom_boxes(ax, bp) custom_boxes(ax, bp)
ax.set_yticklabels([]) ax.set_yticklabels([])
ax.set_title("Wall clock time") ax.set_title("Wall clock time (s)")
name = func_name.split('.')[0] name = func_name.split('.')[0]
fig.savefig("benchmark_results/fig_benchmarks" + name + ".png") fig.savefig("benchmark_results/fig_benchmarks/" + name + ".png")
rst_file.write(name + "\n") rst_file.write(name + "\n")
rst_file.write("-----------------\n\n") rst_file.write("-----------------\n\n")
rst_file.write(str(len(da_ac)) + " replicates \n\n") rst_file.write(str(len(da_acc[0])) + " replicates \n\n")
rst_file.write(".. figure:: fig_benchmarks/" + name + ".png\n\n") rst_file.write(".. figure:: fig_benchmarks/" + name + ".png\n\n")
def plot_all(): def plot_all():
if not plot_ok: if not plot_ok:
print_log('YELLOW', "No plot") print_log('YELLOW', "No plot")
return return
fig_dir = os.getcwd() + '/benchmark_results/fig_benchmarks/'
try: try:
os.makedirs('benchmark_results/fig_benchmarks') os.makedirs(fig_dir)
print("created :" + fig_dir)
except: except:
print('WARNING: directory \'%s\' could not be created! (it probably exists already)' % res_dir) print('WARNING: directory \'%s\' could not be created! (it probably exists already)' % fig_dir)
rst_file = open("benchmark_results/bo_benchmarks.rst", "w") rst_file = open("benchmark_results/bo_benchmarks.rst", "w")
rst_file.write("Bayesian optimization benchmarks\n") rst_file.write("Bayesian optimization benchmarks\n")
rst_file.write("===============================\n\n") rst_file.write("===============================\n\n")
date = "{:%B %d, %Y}".format(datetime.now()) date = "{:%B %d, %Y}".format(datetime.datetime.now())
node = platform.node() node = platform.node()
rst_file.write("/" + date + " / -- " + node + " (" + str(multiprocessing.cpu_count()) + " cores)\n\n") rst_file.write("*" + date + "* -- " + node + " (" + str(multiprocessing.cpu_count()) + " cores)\n\n")
rst_file.write("- We compare to BayesOpt (https://github.com/rmcantin/bayesopt) \n")
rst_file.write("- Accuracy: lower is better (difference with the optimum)\n")
rst_file.write("- Wall time: lower is better\n\n")
rst_file.write("- In each replicate, 10 random samples + 190 function evaluations\n")
rst_file.write("- see `src/benchmarks/limbo/bench.cpp` and `src/benchmarks/bayesopt/bench.cpp`\n\n")
rst_file.write("Naming convention\n")
rst_file.write("------------------\n\n")
rst_file.write("- limbo_def: default Limbo parameters\n\n")
rst_file.write("- opt_cmaes: use CMA-ES (from libcmaes) to optimize the acquisition function\n")
rst_file.write("- opt_direct: use DIRECT (from NLopt) to optimize the acquisition function\n")
rst_file.write("- acq_ucb: use UCB for the acquisition function\n")
rst_file.write("- acq_ei: use EI for the acquisition function\n")
rst_file.write("- hp_opt: use hyper-parameter optimization\n")
rst_file.write("- bayesopt_def: same parameters as default parameters in BayesOpt\n")
print('loading data...') print('loading data...')
data = load_data() data = load_data()
print('data loaded') print('data loaded')
for k in data.keys(): for k in sorted(data.keys()):
print('plotting for ' + k + '...') print('plotting for ' + k + '...')
plot(k, data, rst_file) plot(k, data, rst_file)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment