plot_bo_benchmarks.py 6.46 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
#!/usr/bin/env python
# encoding: utf-8
#| Copyright Inria May 2015
#| This project has received funding from the European Research Council (ERC) under
#| the European Union's Horizon 2020 research and innovation programme (grant
#| agreement No 637972) - see http://www.resibots.eu
#|
#| Contributor(s):
#|   - Jean-Baptiste Mouret (jean-baptiste.mouret@inria.fr)
#|   - Antoine Cully (antoinecully@gmail.com)
#|   - Konstantinos Chatzilygeroudis (konstantinos.chatzilygeroudis@inria.fr)
#|   - Federico Allocati (fede.allocati@gmail.com)
#|   - Vaios Papaspyros (b.papaspyros@gmail.com)
#|   - Roberto Rama (bertoski@gmail.com)
#|
#| This software is a computer library whose purpose is to optimize continuous,
#| black-box functions. It mainly implements Gaussian processes and Bayesian
#| optimization.
#| Main repository: http://github.com/resibots/limbo
#| Documentation: http://www.resibots.eu/limbo
#|
#| This software is governed by the CeCILL-C license under French law and
#| abiding by the rules of distribution of free software.  You can  use,
#| modify and/ or redistribute the software under the terms of the CeCILL-C
#| license as circulated by CEA, CNRS and INRIA at the following URL
#| "http://www.cecill.info".
#|
#| As a counterpart to the access to the source code and  rights to copy,
#| modify and redistribute granted by the license, users are provided only
#| with a limited warranty  and the software's author,  the holder of the
#| economic rights,  and the successive licensors  have only  limited
#| liability.
#|
#| In this respect, the user's attention is drawn to the risks associated
#| with loading,  using,  modifying and/or developing or reproducing the
#| software by the user in light of its specific status of free software,
#| that may mean  that it is complicated to manipulate,  and  that  also
#| therefore means  that it is reserved for developers  and  experienced
#| professionals having in-depth computer knowledge. Users are therefore
#| encouraged to load and test the software's suitability as regards their
#| requirements in conditions enabling the security of their systems and/or
#| data to be ensured and,  more generally, to use and operate it in the
#| same conditions as regards security.
#|
#| The fact that you are presently reading this means that you have had
#| knowledge of the CeCILL-C license and that you accept its terms.
#|# plot the results of the Bayesian Optimization benchmarks
48
49
50
from glob import glob
from collections import defaultdict

51
52
53
54
55
56
57
58
try:
    from waflib import Logs
    def print_log(c, s): Logs.pprint(c, s)
except: # not in waf
    def print_log(c, s): print(s)

try:
    import numpy as np
59
60
61
62
63
64
65
    numpy_found = True
except:
    Logs.pprint('YELLOW', 'WARNING: numpy not found')

try:
    import matplotlib
    matplotlib.use('Agg') # for headless generation    
66
    from pylab import *
67
68
69
70
71
    pylab_found = True
except:
    Logs.pprint('YELLOW', 'WARNING: pylab/matplotlib not found')

try:
72
73
74
    import brewer2mpl
    bmap = brewer2mpl.get_map('Set2', 'qualitative', 8)
    colors = bmap.mpl_colors
75
    brewer2mpl_found = True;
76
except:
77
78
79
80
81
82
    Logs.pprint('YELLOW', 'WARNING: brewer2mpl (colors) not found')


if numpy_found and pylab_found and brewer2mpl_found:
    plot_ok = True
else:
83
84
    plot_ok = False
    Logs.pprint('YELLOW', 'WARNING: numpy/matplotlib not found: no plot of the BO benchmark results')
85
86
87
88
89
90
91
92
93
94
95
96
97

params = {
    'axes.labelsize' : 8,
    'text.fontsize' : 8,
    'axes.titlesize': 10,
    'legend.fontsize' : 10,
    'xtick.labelsize': 5,
    'ytick.labelsize' : 10,
    'figure.figsize' : [9, 2.5]
}
rcParams.update(params)

def load_data():
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
98
    files = glob("benchmark_results/*/*/*.dat")
99
100
101
102
    data = defaultdict(lambda : defaultdict(dict))
    for f in files:
        fs = f.split("/")
        func, var, lib = fs[-1], fs[-2], fs[-3]
103
        print(func, var, lib)
104
105
106
107
108
109
110
111
112
        data[func][lib][var] = np.loadtxt(f)
    return data

def custom_ax(ax):
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
113
    ax.set_axisbelow(True)
114
115
    ax.grid(axis='x', color="0.9", linestyle='-')

Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
116
117
118
119
120
121
122
123
124
125
126
127
128
def custom_boxes(ax, bp):
    for i in range(len(bp['boxes'])):
        box = bp['boxes'][i]
        box.set_linewidth(0)
        boxX = []
        boxY = []
        for j in range(5):
            boxX.append(box.get_xdata()[j])
            boxY.append(box.get_ydata()[j])
            boxCoords = zip(boxX,boxY)
            boxPolygon = Polygon(boxCoords, facecolor = colors[i % len(colors)], linewidth=0)
            ax.add_patch(boxPolygon)

129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
    for i in range(0, len(bp['boxes'])):
        c_i = colors[i%len(colors)]
        bp['boxes'][i].set_color(c_i)
        # we have two whiskers!
        bp['whiskers'][i*2].set_color(c_i)
        bp['whiskers'][i*2 + 1].set_color(c_i)
        bp['whiskers'][i*2].set_linewidth(2)
        bp['whiskers'][i*2 + 1].set_linewidth(2)
        # top and bottom fliers
        bp['fliers'][i*2].set(markerfacecolor=c_i,
                        marker='o', alpha=0.75, markersize=6,
                        markeredgecolor='none')
        bp['fliers'][i * 2 + 1].set(markerfacecolor=c_i,
                        marker='o', alpha=0.75, markersize=6,
                        markeredgecolor='none')
        bp['medians'][i].set_color('black')
        bp['medians'][i].set_linewidth(2)
        # and 4 caps to remove
        for c in bp['caps']:
            c.set_linewidth(0)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
149
150


151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
# plot a single function
def plot(func_name, data):
    d = data[func_name]
    da_acc = []
    da_time = []
    labels = [] 
    for k in d.iterkeys():
        for k2 in d[k].iterkeys():
            da_acc.append(d[k][k2][:, 0])
            da_time.append(d[k][k2][:, 1] / 1000.0)
            labels.append(k + "/" + k2)
    fig = figure()
    fig.subplots_adjust(left=0.3)
    ax = fig.add_subplot(121)
    custom_ax(ax)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
166
167
    bp = ax.boxplot(da_acc, 0, 'rs', 0)
    custom_boxes(ax, bp)
168
169
170
171
    ax.set_yticklabels(labels)
    ax.set_title("Accuracy")
    ax = fig.add_subplot(122)
    custom_ax(ax)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
172
173
    bp = ax.boxplot(da_time, 0, 'rs', 0)
    custom_boxes(ax, bp)
174
175
176
    ax.set_yticklabels([])
    ax.set_title("Wall clock time")

Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
177
    fig.savefig("benchmark_results/" + func_name.split('.')[0] + ".png")    
178
179


Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
180
def plot_all():
181
182
183
    if not plot_ok:
        print_log('YELLOW', "No plot")
        return
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
184
    print('loading data...')
185
    data = load_data()
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
186
    print('data loaded')
187
    for k in data.keys():
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
188
        print('plotting for ' + k + '...')
189
190
        plot(k, data)

Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
191
if __name__ == "__main__":
192
    plot_all()