plot_bo_benchmarks.py 8.57 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
#!/usr/bin/env python
# encoding: utf-8
#| Copyright Inria May 2015
#| This project has received funding from the European Research Council (ERC) under
#| the European Union's Horizon 2020 research and innovation programme (grant
#| agreement No 637972) - see http://www.resibots.eu
#|
#| Contributor(s):
#|   - Jean-Baptiste Mouret (jean-baptiste.mouret@inria.fr)
#|   - Antoine Cully (antoinecully@gmail.com)
#|   - Konstantinos Chatzilygeroudis (konstantinos.chatzilygeroudis@inria.fr)
#|   - Federico Allocati (fede.allocati@gmail.com)
#|   - Vaios Papaspyros (b.papaspyros@gmail.com)
#|   - Roberto Rama (bertoski@gmail.com)
#|
#| This software is a computer library whose purpose is to optimize continuous,
#| black-box functions. It mainly implements Gaussian processes and Bayesian
#| optimization.
#| Main repository: http://github.com/resibots/limbo
#| Documentation: http://www.resibots.eu/limbo
#|
#| This software is governed by the CeCILL-C license under French law and
#| abiding by the rules of distribution of free software.  You can  use,
#| modify and/ or redistribute the software under the terms of the CeCILL-C
#| license as circulated by CEA, CNRS and INRIA at the following URL
#| "http://www.cecill.info".
#|
#| As a counterpart to the access to the source code and  rights to copy,
#| modify and redistribute granted by the license, users are provided only
#| with a limited warranty  and the software's author,  the holder of the
#| economic rights,  and the successive licensors  have only  limited
#| liability.
#|
#| In this respect, the user's attention is drawn to the risks associated
#| with loading,  using,  modifying and/or developing or reproducing the
#| software by the user in light of its specific status of free software,
#| that may mean  that it is complicated to manipulate,  and  that  also
#| therefore means  that it is reserved for developers  and  experienced
#| professionals having in-depth computer knowledge. Users are therefore
#| encouraged to load and test the software's suitability as regards their
#| requirements in conditions enabling the security of their systems and/or
#| data to be ensured and,  more generally, to use and operate it in the
#| same conditions as regards security.
#|
#| The fact that you are presently reading this means that you have had
#| knowledge of the CeCILL-C license and that you accept its terms.
47
#|
48
from glob import glob
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
49
import os
50
from collections import defaultdict
51
52
53
from datetime import datetime
import platform
import multiprocessing
54

55
56
57
58
59
60
61
62
try:
    from waflib import Logs
    def print_log(c, s): Logs.pprint(c, s)
except: # not in waf
    def print_log(c, s): print(s)

try:
    import numpy as np
63
64
65
66
67
68
69
    numpy_found = True
except:
    Logs.pprint('YELLOW', 'WARNING: numpy not found')

try:
    import matplotlib
    matplotlib.use('Agg') # for headless generation    
70
    from pylab import *
71
72
73
74
75
    pylab_found = True
except:
    Logs.pprint('YELLOW', 'WARNING: pylab/matplotlib not found')

try:
76
77
78
    import brewer2mpl
    bmap = brewer2mpl.get_map('Set2', 'qualitative', 8)
    colors = bmap.mpl_colors
79
    brewer2mpl_found = True;
80
except:
81
82
83
84
85
86
    Logs.pprint('YELLOW', 'WARNING: brewer2mpl (colors) not found')


if numpy_found and pylab_found and brewer2mpl_found:
    plot_ok = True
else:
87
88
    plot_ok = False
    Logs.pprint('YELLOW', 'WARNING: numpy/matplotlib not found: no plot of the BO benchmark results')
89
90

def load_data():
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
91
    files = glob("benchmark_results/*/*/*.dat")
92
93
94
95
    data = defaultdict(lambda : defaultdict(dict))
    for f in files:
        fs = f.split("/")
        func, var, lib = fs[-1], fs[-2], fs[-3]
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
96
        #print(func, var, lib)
97
98
99
100
101
102
103
104
105
        data[func][lib][var] = np.loadtxt(f)
    return data

def custom_ax(ax):
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
106
    ax.set_axisbelow(True)
107
108
    ax.grid(axis='x', color="0.9", linestyle='-')

Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
109
def custom_boxes(ax, bp):
110
    for i in range(0, len(bp['boxes'])):
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
111
112
113
114
115
116
117
118
119
120
121
        box = bp['boxes'][i]
        box.set_linewidth(0)
        boxX = []
        boxY = []
        for j in range(5):
            boxX.append(box.get_xdata()[j])
            boxY.append(box.get_ydata()[j])
            boxCoords = zip(boxX,boxY)
            boxPolygon = Polygon(boxCoords, facecolor = colors[i % len(colors)], linewidth=0)
            ax.add_patch(boxPolygon)

122
    for i in range(0, len(bp['boxes'])):
123
        c_i = colors[i % len(colors)]
124
125
        bp['boxes'][i].set_color(c_i)
        # we have two whiskers!
126
127
128
129
130
131
132
133
        bp['whiskers'][i * 2].set_color(c_i)
        bp['whiskers'][i * 2 + 1].set_color(c_i)
        bp['whiskers'][i * 2].set_linewidth(2)
        bp['whiskers'][i * 2 + 1].set_linewidth(2)
        # ... and one set of fliers
        bp['fliers'][i].set(markerfacecolor=c_i,
                            marker='o', alpha=0.75, markersize=6,
                            markeredgecolor='none')
134
135
136
137
138
        bp['medians'][i].set_color('black')
        bp['medians'][i].set_linewidth(2)
        # and 4 caps to remove
        for c in bp['caps']:
            c.set_linewidth(0)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
139

140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
def clean_labels(k1, k2):
    m = {'opt_cmaes': 'Acqu. opt.=CMA-ES, ',
         'opt_direct' :'Acqu. opt.=DIRECT, ',
         'bayesopt_hp_opt':'HP Opt.=yes',
         '_hpopt' : 'HP Opt.=yes, ',
         'acq_ucb': 'Acqu. fun=UCB, ',
         'acq_ei':'Acqu. fun=EI',
         'limbo_def': 'Limbo defaults, ',
         'bayesopt_default':'BayesOpt defaults, ',
         'bench_bayesopt_def':'BayesOpt defaults, ',
         'bench_': '',
         
    }
    m2 = { 'limbo': 'Limbo',
           'bayesopt' : 'BayesOpt'}
    k1 = m2[k1]
    for i in m.keys():
        k2 = k2.replace(i, m[i])
    if k2[-2:len(k2)] == ', ':
        k2 = k2[0:-2]
    return k1, k2
    
162
# plot a single function
163
def plot(func_name, data, rst_file):
164
165
166
167
168
169
    # set the figure size
    params = {
        'axes.labelsize' : 8,
        'text.fontsize' : 8,
        'axes.titlesize': 10,
        'legend.fontsize' : 10,
170
        'xtick.labelsize': 7,
171
        'ytick.labelsize' : 10,
172
        'figure.figsize' : [11, 2.5]
173
174
175
176
    }
    rcParams.update(params)
    
    # plot
177
178
179
    d = data[func_name]
    da_acc = []
    da_time = []
180
181
182
183
184
185
186
187
188
189
190
191
192
193
    labels = []
    def sort_fun(x, y):
        if 'def' in x and 'def' in y and len(x) < len(y):
            return 1
        if 'def' in x and 'def' in y and len(x) > len(y):
            return -1
        if 'def' in x and 'def' not in y:
            return 1
        if 'def' in y and 'def' not in x:
            return -1
        return x < y
        
    for k in sorted(d.iterkeys()):
        for k2 in sorted(d[k].iterkeys(), sort_fun):
194
195
            da_acc.append(d[k][k2][:, 0])
            da_time.append(d[k][k2][:, 1] / 1000.0)
196
197
            x, y = clean_labels(k, k2)
            labels.append(x + " (" + y + ")")
198
    fig = figure()
199
    fig.subplots_adjust(left=0.3, right=0.95)
200
201
    ax = fig.add_subplot(121)
    custom_ax(ax)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
202
203
    bp = ax.boxplot(da_acc, 0, 'rs', 0)
    custom_boxes(ax, bp)
204
    ax.set_yticklabels(labels)
205
    ax.set_title("Accuracy (difference with optimum cost)")
206
207
    ax = fig.add_subplot(122)
    custom_ax(ax)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
208
209
    bp = ax.boxplot(da_time, 0, 'rs', 0)
    custom_boxes(ax, bp)
210
    ax.set_yticklabels([])
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
211
    ax.set_title("Wall clock time (s)")
212

213
    name = func_name.split('.')[0]
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
214
    fig.savefig("benchmark_results/fig_benchmarks/" + name + ".png")    
215
216
    rst_file.write(name + "\n")
    rst_file.write("-----------------\n\n")
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
217
    rst_file.write(str(len(da_acc[0])) + " replicates \n\n")
218
    rst_file.write(".. figure:: fig_benchmarks/" + name + ".png\n\n")
219

220
221
222
223
224
225

# dst file is already open
def include(src_file, dst_file):
    for i in open(src_file):
        dst_file.write(i)
    
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
226
def plot_all():
227
228
229
    if not plot_ok:
        print_log('YELLOW', "No plot")
        return
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
230
    fig_dir = os.getcwd() + '/benchmark_results/fig_benchmarks/'
231
    try:
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
232
233
        os.makedirs(fig_dir)
        print("created :" + fig_dir)
234
    except:
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
235
        print('WARNING: directory \'%s\' could not be created! (it probably exists already)' % fig_dir)
236
237
238
239

    rst_file = open("benchmark_results/bo_benchmarks.rst", "w")
    rst_file.write("Bayesian optimization benchmarks\n")
    rst_file.write("===============================\n\n")
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
240
    date = "{:%B %d, %Y}".format(datetime.datetime.now())
241
    node = platform.node()
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
242
243
    rst_file.write("*" + date + "* -- " + node + " (" + str(multiprocessing.cpu_count()) + " cores)\n\n")

244
    include("docs/benchmark_res_bo.inc", rst_file)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
245
246
    
    
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
247
    print('loading data...')
248
    data = load_data()
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
249
    print('data loaded')
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
250
    for k in sorted(data.keys()):
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
251
        print('plotting for ' + k + '...')
252
        plot(k, data, rst_file)
253

Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
254
if __name__ == "__main__":
255
    plot_all()