Commit b6a2e4b5 authored by Konstantinos Chatzilygeroudis's avatar Konstantinos Chatzilygeroudis
Browse files

Merge pull request #93 from resibots/local_jobs

Ability to submit local jobs
parents e605fd93 54f7d33b
import os
import stat
import subprocess
import time
import threading
from waflib.Tools import waf_unit_test
json_ok = True
......@@ -113,6 +115,78 @@ def _sub_script(tpl, conf_file):
fnames += [(fname, directory)]
return fnames
def _sub_script_local(conf_file):
if 'LD_LIBRARY_PATH' in os.environ:
ld_lib_path = os.environ['LD_LIBRARY_PATH']
else:
ld_lib_path = "''"
print 'LD_LIBRARY_PATH=' + ld_lib_path
# parse conf
list_exps = simplejson.load(open(conf_file))
fnames = []
for conf in list_exps:
exps = conf['exps']
nb_runs = conf['nb_runs']
res_dir = conf['res_dir']
bin_dir = conf['bin_dir']
wall_time = conf['wall_time']
use_mpi = "false"
try:
use_mpi = conf['use_mpi']
except:
use_mpi = "false"
try:
nb_cores = conf['nb_cores']
except:
nb_cores = 1
try:
args = conf['args']
except:
args = ''
email = conf['email']
if (use_mpi == "true"):
ppn = '1'
mpirun = 'mpirun'
else:
# nb_cores = 1;
ppn = "8"
mpirun = ''
#fnames = []
for i in range(0, nb_runs):
for e in exps:
directory = res_dir + "/" + e + "/exp_" + str(i)
try:
os.makedirs(directory)
except:
print "WARNING, dir:" + directory + " not be created"
subprocess.call('cp ' + bin_dir + '/' + e + ' ' + '"' + directory + '"', shell=True)
fname = e
fnames += [(fname, directory)]
return fnames,args
def run_local_one(directory, s):
std_out = open(directory+"/stdout.txt", "w")
std_err = open(directory+"/stderr.txt", "w")
retcode = subprocess.call(s, shell=True, env=None, stdout=std_out, stderr=std_err)
def run_local(conf_file, serial = True):
fnames,arguments = _sub_script_local(conf_file)
threads = []
for (fname, directory) in fnames:
s = "cd " + '"' + directory + '"' + " && " + "./" + fname + ' ' + arguments
print "Executing " + s
if not serial:
t = threading.Thread(target=run_local_one, args=(directory,s,))
threads.append(t)
else:
run_local_one(directory,s)
if not serial:
for i in range(len(threads)):
threads[i].start()
for i in range(len(threads)):
threads[i].join()
def qsub(conf_file):
tpl = """#!/bin/sh
......
......@@ -33,6 +33,8 @@ def options(opt):
opt.add_option('--exp', type='string', help='exp(s) to build, separate by comma', dest='exp')
opt.add_option('--qsub', type='string', help='config file (json) to submit to torque', dest='qsub')
opt.add_option('--oar', type='string', help='config file (json) to submit to oar', dest='oar')
opt.add_option('--local', type='string', help='config file (json) to run local', dest='local')
opt.add_option('--local_serial', type='string', help='config file (json) to run local', dest='local_serial')
opt.add_option('--experimental', action='store_true', help='specify to compile the experimental examples', dest='experimental')
# tests
opt.add_option('--tests', action='store_true', help='compile tests or not', dest='tests')
......@@ -123,6 +125,10 @@ def shutdown(ctx):
limbo.qsub(ctx.options.qsub)
if ctx.options.oar:
limbo.oar(ctx.options.oar)
if ctx.options.local:
limbo.run_local(ctx.options.local, False)
if ctx.options.local_serial:
limbo.run_local(ctx.options.local_serial)
class BuildExtensiveTestsContext(BuildContext):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment