Merge pull request #93 from resibots/local_jobs

Ability to submit local jobs

waf_tools/limbo.py

 import os
 import stat
 import subprocess
+import time
+import threading
 from waflib.Tools import waf_unit_test
 
 json_ok = True
@@ -113,6 +115,78 @@ def _sub_script(tpl, conf_file):
                 fnames += [(fname, directory)]
     return fnames
 
+def _sub_script_local(conf_file):
+    if 'LD_LIBRARY_PATH' in os.environ:
+        ld_lib_path = os.environ['LD_LIBRARY_PATH']
+    else:
+        ld_lib_path = "''"
+    print 'LD_LIBRARY_PATH=' + ld_lib_path
+     # parse conf
+    list_exps = simplejson.load(open(conf_file))
+    fnames = []
+    for conf in list_exps:
+        exps = conf['exps']
+        nb_runs = conf['nb_runs']
+        res_dir = conf['res_dir']
+        bin_dir = conf['bin_dir']
+        wall_time = conf['wall_time']
+        use_mpi = "false"
+        try:
+            use_mpi = conf['use_mpi']
+        except:
+            use_mpi = "false"
+        try:
+            nb_cores = conf['nb_cores']
+        except:
+            nb_cores = 1
+        try:
+            args = conf['args']
+        except:
+            args = ''
+        email = conf['email']
+        if (use_mpi == "true"):
+            ppn = '1'
+            mpirun = 'mpirun'
+        else:
+     #      nb_cores = 1;
+            ppn = "8"
+            mpirun = ''
+
+        #fnames = []
+        for i in range(0, nb_runs):
+            for e in exps:
+                directory = res_dir + "/" + e + "/exp_" + str(i)
+                try:
+                    os.makedirs(directory)
+                except:
+                    print "WARNING, dir:" + directory + " not be created"
+                subprocess.call('cp ' + bin_dir + '/' + e + ' ' + '"' + directory + '"', shell=True)
+                fname = e
+                fnames += [(fname, directory)]
+    return fnames,args
+
+def run_local_one(directory, s):
+    std_out = open(directory+"/stdout.txt", "w")
+    std_err = open(directory+"/stderr.txt", "w")
+    retcode = subprocess.call(s, shell=True, env=None, stdout=std_out, stderr=std_err)
+
+def run_local(conf_file, serial = True):
+    fnames,arguments = _sub_script_local(conf_file)
+    threads = []
+    for (fname, directory) in fnames:
+        s = "cd " + '"' + directory + '"' + " && " + "./" + fname + ' ' + arguments
+        print "Executing " + s
+        if not serial:
+            t = threading.Thread(target=run_local_one, args=(directory,s,))
+            threads.append(t)
+        else:
+            run_local_one(directory,s)
+
+    if not serial:
+        for i in range(len(threads)):
+            threads[i].start()
+        for i in range(len(threads)):
+            threads[i].join()
 
 def qsub(conf_file):
     tpl = """#!/bin/sh

wscript

@@ -33,6 +33,8 @@ def options(opt):
         opt.add_option('--exp', type='string', help='exp(s) to build, separate by comma', dest='exp')
         opt.add_option('--qsub', type='string', help='config file (json) to submit to torque', dest='qsub')
         opt.add_option('--oar', type='string', help='config file (json) to submit to oar', dest='oar')
+        opt.add_option('--local', type='string', help='config file (json) to run local', dest='local')
+        opt.add_option('--local_serial', type='string', help='config file (json) to run local', dest='local_serial')
         opt.add_option('--experimental', action='store_true', help='specify to compile the experimental examples', dest='experimental')
         # tests
         opt.add_option('--tests', action='store_true', help='compile tests or not', dest='tests')
@@ -123,6 +125,10 @@ def shutdown(ctx):
         limbo.qsub(ctx.options.qsub)
     if ctx.options.oar:
         limbo.oar(ctx.options.oar)
+    if ctx.options.local:
+        limbo.run_local(ctx.options.local, False)
+    if ctx.options.local_serial:
+        limbo.run_local(ctx.options.local_serial)
 
 
 class BuildExtensiveTestsContext(BuildContext):