plot_bo_benchmarks.py 6.04 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
#!/usr/bin/env python
# encoding: utf-8
#| Copyright Inria May 2015
#| This project has received funding from the European Research Council (ERC) under
#| the European Union's Horizon 2020 research and innovation programme (grant
#| agreement No 637972) - see http://www.resibots.eu
#|
#| Contributor(s):
#|   - Jean-Baptiste Mouret (jean-baptiste.mouret@inria.fr)
#|   - Antoine Cully (antoinecully@gmail.com)
#|   - Konstantinos Chatzilygeroudis (konstantinos.chatzilygeroudis@inria.fr)
#|   - Federico Allocati (fede.allocati@gmail.com)
#|   - Vaios Papaspyros (b.papaspyros@gmail.com)
#|   - Roberto Rama (bertoski@gmail.com)
#|
#| This software is a computer library whose purpose is to optimize continuous,
#| black-box functions. It mainly implements Gaussian processes and Bayesian
#| optimization.
#| Main repository: http://github.com/resibots/limbo
#| Documentation: http://www.resibots.eu/limbo
#|
#| This software is governed by the CeCILL-C license under French law and
#| abiding by the rules of distribution of free software.  You can  use,
#| modify and/ or redistribute the software under the terms of the CeCILL-C
#| license as circulated by CEA, CNRS and INRIA at the following URL
#| "http://www.cecill.info".
#|
#| As a counterpart to the access to the source code and  rights to copy,
#| modify and redistribute granted by the license, users are provided only
#| with a limited warranty  and the software's author,  the holder of the
#| economic rights,  and the successive licensors  have only  limited
#| liability.
#|
#| In this respect, the user's attention is drawn to the risks associated
#| with loading,  using,  modifying and/or developing or reproducing the
#| software by the user in light of its specific status of free software,
#| that may mean  that it is complicated to manipulate,  and  that  also
#| therefore means  that it is reserved for developers  and  experienced
#| professionals having in-depth computer knowledge. Users are therefore
#| encouraged to load and test the software's suitability as regards their
#| requirements in conditions enabling the security of their systems and/or
#| data to be ensured and,  more generally, to use and operate it in the
#| same conditions as regards security.
#|
#| The fact that you are presently reading this means that you have had
#| knowledge of the CeCILL-C license and that you accept its terms.
#|# plot the results of the Bayesian Optimization benchmarks
48
49
50
from glob import glob
from collections import defaultdict

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
try:
    from waflib import Logs
    def print_log(c, s): Logs.pprint(c, s)
except: # not in waf
    def print_log(c, s): print(s)

try:
    import numpy as np
    from pylab import *
    import brewer2mpl
    bmap = brewer2mpl.get_map('Set2', 'qualitative', 8)
    colors = bmap.mpl_colors
    plot_ok = True
except:
    plot_ok = False
    Logs.pprint('YELLOW', 'WARNING: numpy/matplotlib not found: no plot of the BO benchmark results')
67
68
69
70
71
72
73
74
75
76
77
78
79

params = {
    'axes.labelsize' : 8,
    'text.fontsize' : 8,
    'axes.titlesize': 10,
    'legend.fontsize' : 10,
    'xtick.labelsize': 5,
    'ytick.labelsize' : 10,
    'figure.figsize' : [9, 2.5]
}
rcParams.update(params)

def load_data():
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
80
    files = glob("benchmark_results/*/*/*.dat")
81
82
83
84
    data = defaultdict(lambda : defaultdict(dict))
    for f in files:
        fs = f.split("/")
        func, var, lib = fs[-1], fs[-2], fs[-3]
85
        print(func, var, lib)
86
87
88
89
90
91
92
93
94
        data[func][lib][var] = np.loadtxt(f)
    return data

def custom_ax(ax):
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
95
    ax.set_axisbelow(True)
96
97
    ax.grid(axis='x', color="0.9", linestyle='-')

Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
98
99
100
101
102
103
104
105
106
107
108
109
110
def custom_boxes(ax, bp):
    for i in range(len(bp['boxes'])):
        box = bp['boxes'][i]
        box.set_linewidth(0)
        boxX = []
        boxY = []
        for j in range(5):
            boxX.append(box.get_xdata()[j])
            boxY.append(box.get_ydata()[j])
            boxCoords = zip(boxX,boxY)
            boxPolygon = Polygon(boxCoords, facecolor = colors[i % len(colors)], linewidth=0)
            ax.add_patch(boxPolygon)

111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
    for i in range(0, len(bp['boxes'])):
        c_i = colors[i%len(colors)]
        bp['boxes'][i].set_color(c_i)
        # we have two whiskers!
        bp['whiskers'][i*2].set_color(c_i)
        bp['whiskers'][i*2 + 1].set_color(c_i)
        bp['whiskers'][i*2].set_linewidth(2)
        bp['whiskers'][i*2 + 1].set_linewidth(2)
        # top and bottom fliers
        bp['fliers'][i*2].set(markerfacecolor=c_i,
                        marker='o', alpha=0.75, markersize=6,
                        markeredgecolor='none')
        bp['fliers'][i * 2 + 1].set(markerfacecolor=c_i,
                        marker='o', alpha=0.75, markersize=6,
                        markeredgecolor='none')
        bp['medians'][i].set_color('black')
        bp['medians'][i].set_linewidth(2)
        # and 4 caps to remove
        for c in bp['caps']:
            c.set_linewidth(0)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
131
132


133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
# plot a single function
def plot(func_name, data):
    d = data[func_name]
    da_acc = []
    da_time = []
    labels = [] 
    for k in d.iterkeys():
        for k2 in d[k].iterkeys():
            da_acc.append(d[k][k2][:, 0])
            da_time.append(d[k][k2][:, 1] / 1000.0)
            labels.append(k + "/" + k2)
    fig = figure()
    fig.subplots_adjust(left=0.3)
    ax = fig.add_subplot(121)
    custom_ax(ax)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
148
149
    bp = ax.boxplot(da_acc, 0, 'rs', 0)
    custom_boxes(ax, bp)
150
151
152
153
    ax.set_yticklabels(labels)
    ax.set_title("Accuracy")
    ax = fig.add_subplot(122)
    custom_ax(ax)
Jean-Baptiste Mouret's avatar
Jean-Baptiste Mouret committed
154
155
    bp = ax.boxplot(da_time, 0, 'rs', 0)
    custom_boxes(ax, bp)
156
157
158
    ax.set_yticklabels([])
    ax.set_title("Wall clock time")

Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
159
    fig.savefig("benchmark_results/" + func_name.split('.')[0] + ".png")    
160
161


Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
162
def plot_all():
163
164
165
    if not plot_ok:
        print_log('YELLOW', "No plot")
        return
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
166
    print('loading data...')
167
    data = load_data()
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
168
    print('data loaded')
169
    for k in data.keys():
Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
170
        print('plotting for ' + k + '...')
171
172
        plot(k, data)

Jean-Baptiste Mouret's avatar
misc    
Jean-Baptiste Mouret committed
173
174
if __name__ == "__main__":
    plot_all()