plot_bo_benchmarks.py 8.36 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
#!/usr/bin/env python
# encoding: utf-8
#| Copyright Inria May 2015
#| This project has received funding from the European Research Council (ERC) under
#| the European Union's Horizon 2020 research and innovation programme (grant
#| agreement No 637972) - see http://www.resibots.eu
#|
#| Contributor(s):
#|   - Jean-Baptiste Mouret (jean-baptiste.mouret@inria.fr)
#|   - Antoine Cully (antoinecully@gmail.com)
#|   - Konstantinos Chatzilygeroudis (konstantinos.chatzilygeroudis@inria.fr)
#|   - Federico Allocati (fede.allocati@gmail.com)
#|   - Vaios Papaspyros (b.papaspyros@gmail.com)
#|   - Roberto Rama (bertoski@gmail.com)
#|
#| This software is a computer library whose purpose is to optimize continuous,
#| black-box functions. It mainly implements Gaussian processes and Bayesian
#| optimization.
#| Main repository: http://github.com/resibots/limbo
#| Documentation: http://www.resibots.eu/limbo
#|
#| This software is governed by the CeCILL-C license under French law and
#| abiding by the rules of distribution of free software.  You can  use,
#| modify and/ or redistribute the software under the terms of the CeCILL-C
#| license as circulated by CEA, CNRS and INRIA at the following URL
#| "http://www.cecill.info".
#|
#| As a counterpart to the access to the source code and  rights to copy,
#| modify and redistribute granted by the license, users are provided only
#| with a limited warranty  and the software's author,  the holder of the
#| economic rights,  and the successive licensors  have only  limited
#| liability.
#|
#| In this respect, the user's attention is drawn to the risks associated
#| with loading,  using,  modifying and/or developing or reproducing the
#| software by the user in light of its specific status of free software,
#| that may mean  that it is complicated to manipulate,  and  that  also
#| therefore means  that it is reserved for developers  and  experienced
#| professionals having in-depth computer knowledge. Users are therefore
#| encouraged to load and test the software's suitability as regards their
#| requirements in conditions enabling the security of their systems and/or
#| data to be ensured and,  more generally, to use and operate it in the
#| same conditions as regards security.
#|
#| The fact that you are presently reading this means that you have had
#| knowledge of the CeCILL-C license and that you accept its terms.
47
#|
48
from glob import glob
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
49
import os
50
from collections import defaultdict
51
52
53
from datetime import datetime
import platform
import multiprocessing
54

55
56
57
58
59
60
61
62
try:
    from waflib import Logs
    def print_log(c, s): Logs.pprint(c, s)
except: # not in waf
    def print_log(c, s): print(s)

try:
    import numpy as np
63
64
65
66
67
68
69
    numpy_found = True
except:
    Logs.pprint('YELLOW', 'WARNING: numpy not found')

try:
    import matplotlib
    matplotlib.use('Agg') # for headless generation    
70
    from pylab import *
71
72
73
74
75
    pylab_found = True
except:
    Logs.pprint('YELLOW', 'WARNING: pylab/matplotlib not found')

try:
76
77
78
    import brewer2mpl
    bmap = brewer2mpl.get_map('Set2', 'qualitative', 8)
    colors = bmap.mpl_colors
79
    brewer2mpl_found = True;
80
except:
81
82
83
84
85
86
    Logs.pprint('YELLOW', 'WARNING: brewer2mpl (colors) not found')


if numpy_found and pylab_found and brewer2mpl_found:
    plot_ok = True
else:
87
88
    plot_ok = False
    Logs.pprint('YELLOW', 'WARNING: numpy/matplotlib not found: no plot of the BO benchmark results')
89
90

def load_data():
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
91
    files = glob("benchmark_results/*/*/*.dat")
92
93
94
95
    data = defaultdict(lambda : defaultdict(dict))
    for f in files:
        fs = f.split("/")
        func, var, lib = fs[-1], fs[-2], fs[-3]
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
96
        #print(func, var, lib)
97
98
99
100
101
102
103
104
105
        data[func][lib][var] = np.loadtxt(f)
    return data

def custom_ax(ax):
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
106
    ax.set_axisbelow(True)
107
108
    ax.grid(axis='x', color="0.9", linestyle='-')

Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
109
def custom_boxes(ax, bp):
110
    for i in range(0, len(bp['boxes'])):
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
111
112
113
114
115
116
117
118
119
120
121
        box = bp['boxes'][i]
        box.set_linewidth(0)
        boxX = []
        boxY = []
        for j in range(5):
            boxX.append(box.get_xdata()[j])
            boxY.append(box.get_ydata()[j])
            boxCoords = zip(boxX,boxY)
            boxPolygon = Polygon(boxCoords, facecolor = colors[i % len(colors)], linewidth=0)
            ax.add_patch(boxPolygon)

122
    for i in range(0, len(bp['boxes'])):
123
        c_i = colors[i % len(colors)]
124
125
        bp['boxes'][i].set_color(c_i)
        # we have two whiskers!
126
127
128
129
130
131
132
133
        bp['whiskers'][i * 2].set_color(c_i)
        bp['whiskers'][i * 2 + 1].set_color(c_i)
        bp['whiskers'][i * 2].set_linewidth(2)
        bp['whiskers'][i * 2 + 1].set_linewidth(2)
        # ... and one set of fliers
        bp['fliers'][i].set(markerfacecolor=c_i,
                            marker='o', alpha=0.75, markersize=6,
                            markeredgecolor='none')
134
135
136
137
138
        bp['medians'][i].set_color('black')
        bp['medians'][i].set_linewidth(2)
        # and 4 caps to remove
        for c in bp['caps']:
            c.set_linewidth(0)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
139

140
# plot a single function
141
def plot(func_name, data, rst_file):
142
143
144
145
146
147
148
149
150
151
152
153
154
    # set the figure size
    params = {
        'axes.labelsize' : 8,
        'text.fontsize' : 8,
        'axes.titlesize': 10,
        'legend.fontsize' : 10,
        'xtick.labelsize': 5,
        'ytick.labelsize' : 10,
        'figure.figsize' : [9, 2.5]
    }
    rcParams.update(params)
    
    # plot
155
156
157
158
159
160
161
162
163
164
165
166
167
    d = data[func_name]
    da_acc = []
    da_time = []
    labels = [] 
    for k in d.iterkeys():
        for k2 in d[k].iterkeys():
            da_acc.append(d[k][k2][:, 0])
            da_time.append(d[k][k2][:, 1] / 1000.0)
            labels.append(k + "/" + k2)
    fig = figure()
    fig.subplots_adjust(left=0.3)
    ax = fig.add_subplot(121)
    custom_ax(ax)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
168
169
    bp = ax.boxplot(da_acc, 0, 'rs', 0)
    custom_boxes(ax, bp)
170
171
172
173
    ax.set_yticklabels(labels)
    ax.set_title("Accuracy")
    ax = fig.add_subplot(122)
    custom_ax(ax)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
174
175
    bp = ax.boxplot(da_time, 0, 'rs', 0)
    custom_boxes(ax, bp)
176
    ax.set_yticklabels([])
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
177
    ax.set_title("Wall clock time (s)")
178

179
    name = func_name.split('.')[0]
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
180
    fig.savefig("benchmark_results/fig_benchmarks/" + name + ".png")    
181
182
    rst_file.write(name + "\n")
    rst_file.write("-----------------\n\n")
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
183
    rst_file.write(str(len(da_acc[0])) + " replicates \n\n")
184
    rst_file.write(".. figure:: fig_benchmarks/" + name + ".png\n\n")
185

Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
186
def plot_all():
187
188
189
    if not plot_ok:
        print_log('YELLOW', "No plot")
        return
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
190
    fig_dir = os.getcwd() + '/benchmark_results/fig_benchmarks/'
191
    try:
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
192
193
        os.makedirs(fig_dir)
        print("created :" + fig_dir)
194
    except:
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
195
        print('WARNING: directory \'%s\' could not be created! (it probably exists already)' % fig_dir)
196
197
198
199

    rst_file = open("benchmark_results/bo_benchmarks.rst", "w")
    rst_file.write("Bayesian optimization benchmarks\n")
    rst_file.write("===============================\n\n")
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
200
    date = "{:%B %d, %Y}".format(datetime.datetime.now())
201
    node = platform.node()
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
    rst_file.write("*" + date + "* -- " + node + " (" + str(multiprocessing.cpu_count()) + " cores)\n\n")

    rst_file.write("- We compare to BayesOpt (https://github.com/rmcantin/bayesopt) \n")
    rst_file.write("- Accuracy: lower is better (difference with the optimum)\n")
    rst_file.write("- Wall time: lower is better\n\n")
    rst_file.write("- In each replicate, 10 random samples + 190 function evaluations\n")
    rst_file.write("- see `src/benchmarks/limbo/bench.cpp` and `src/benchmarks/bayesopt/bench.cpp`\n\n")

    rst_file.write("Naming convention\n")
    rst_file.write("------------------\n\n")

    rst_file.write("- limbo_def: default Limbo parameters\n\n")
    rst_file.write("- opt_cmaes: use CMA-ES (from libcmaes) to optimize the acquisition function\n")
    rst_file.write("- opt_direct: use DIRECT (from NLopt) to optimize the acquisition function\n")
    rst_file.write("- acq_ucb: use UCB for the acquisition function\n")
    rst_file.write("- acq_ei: use EI for the acquisition function\n")
    rst_file.write("- hp_opt: use hyper-parameter optimization\n")
    rst_file.write("- bayesopt_def: same parameters as default parameters in BayesOpt\n")
    
    
    
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
223
    print('loading data...')
224
    data = load_data()
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
225
    print('data loaded')
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
226
    for k in sorted(data.keys()):
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
227
        print('plotting for ' + k + '...')
228
        plot(k, data, rst_file)
229

Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
230
if __name__ == "__main__":
231
    plot_all()