#!/usr/bin/env python # -*- coding: utf-8; -*- # # Script "comp.py": # - compilation de Metafor # - lancement automatique de la batterie from parametricJob import * # -- repository classes -------------------------------------------------------- class Repo(object): def __init__(self, name, url): self.name = name self.url = url class GitRepo(Repo): def __init__(self, name, url): super(GitRepo, self).__init__(name, url) def co_cmd(self): cmd = "git clone --quiet %s %s" % (self.url, self.name) return cmd class SVNRepo(Repo): def __init__(self, name, url): super(SVNRepo, self).__init__(name, url) def co_cmd(self): cmd = "svn co --quiet %s %s" % (self.url, self.name) return cmd # -- CompJob class ------------------------------------------------------------- class CompJob(ParametricJob): """ Manage a compilation + battery.py """ def __init__(self, _jobId=''): # init base class self.jobId = _jobId cfgfile = "comp%s.cfg" % self.jobId ParametricJob.__init__(self, cfgfile) # list of required repositories self.repos = [] self.repos.append(SVNRepo('oo_meta', 'svn+ssh://blueberry.ltas.ulg.ac.be/home/metafor/SVN/oo_meta/trunk')) self.repos.append(SVNRepo('oo_nda', 'svn+ssh://blueberry.ltas.ulg.ac.be/home/metafor/SVN/oo_nda/trunk')) self.repos.append(GitRepo('linuxbin', 'https://github.com/ulgltas/linuxbin.git')) self.repos.append(GitRepo('parasolid', 'blueberry.ltas.ulg.ac.be:/home/metafor/GIT/parasolid.git')) def setDefaultPars(self): if len(self.pars)!=0: return TextPRM(self.pars, 'MAIL_ADDR', 'e-mail address (reports)', os.getenv('USER')) TextPRM(self.pars, 'SMTP_SERV', 'SMTP email server', 'smtp.ulg.ac.be') TextPRM(self.pars, 'ARC_NAME', 'archive name', '~/dev.zip') try: machineName = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] # lpx : marche pas dans des vbox except: machineName = socket.gethostname().split('.')[0] # marche dans ma vbox TextPRM(self.pars, 'CMAKELIST', 'build options', "%s.cmake" %machineName) YesNoPRM(self.pars, 'DEBUG_MODE', 'debug mode', False) TextPRM(self.pars, 'NICE_VALUE', 'nice value', "0") TextPRM(self.pars, 'NB_TASKS', 'nb of task launched in parallel', "1") TextPRM(self.pars, 'NB_THREADS', 'nb of threads by task', "1") MultiPRM(self.pars, 'RUNMETHOD', 'Run Method', ["interactive", "at", "batch"], "batch") TextPRM(self.pars, 'AT_TIME' , 'Delay for at launch (no syntax check, use with care)', "now") MultiPRM(self.pars, 'UNZIP', 'source', ["zip", "checkout", "present"], "zip") YesNoPRM(self.pars, 'COMPILE', 'compile', True) MultiPRM(self.pars, 'BATTERY', 'battery', [True, False, "continue"], True) PRMAction(self.actions, 'a', self.pars['MAIL_ADDR']) PRMAction(self.actions, 'b', self.pars['ARC_NAME']) PRMAction(self.actions, 'c', self.pars['CMAKELIST']) PRMAction(self.actions, 'd', self.pars['DEBUG_MODE']) PRMAction(self.actions, 'h', self.pars['NICE_VALUE']) PRMAction(self.actions, 'j', self.pars['NB_TASKS']) PRMAction(self.actions, 'k', self.pars['NB_THREADS']) PRMAction(self.actions, 'm', self.pars['RUNMETHOD']) # AT paramters PRMAction(self.actions, 'n', self.pars['AT_TIME']) # Actions NoAction(self.actions) PRMAction(self.actions, '1', self.pars['UNZIP']) PRMAction(self.actions, '2', self.pars['COMPILE']) PRMAction(self.actions, '3', self.pars['BATTERY']) NoAction (self.actions) GoAction (self.actions, 'G') SaveAction(self.actions, 'S') QuitAction(self.actions, 'Q') def configActions(self): self.pars['ARC_NAME'].enable(self.pars['UNZIP'].val=="zip") self.pars['NB_TASKS'].enable(self.pars['COMPILE'].val==True or self.pars['BATTERY'].val!=False) self.pars['NB_THREADS'].enable(self.pars['COMPILE'].val==True or self.pars['BATTERY'].val!=False) self.pars['CMAKELIST'].enable(self.pars['COMPILE'].val==True) self.pars['DEBUG_MODE'].enable(self.pars['COMPILE'].val==True) self.pars['NICE_VALUE'].enable(self.pars['BATTERY'].val!=False and self.pars['RUNMETHOD'].val!='sge') # Batch self.pars['AT_TIME'].enable(self.pars['RUNMETHOD'].val=='at') def touchFiles(self): for repo in self.repos: print "touching %s" % repo.name for path, dirs, files in os.walk(repo.name): for file in files: os.utime(os.path.join(path,file), None) # touch file def checkOut(self): for repo in self.repos: if not os.path.isdir(repo.name): print 'checking out "%s" from %s...' % (repo.name, repo.url) cmd = repo.co_cmd() os.system(cmd) def doClean(self): dirs = [ repo.name for repo in self.repos ] dirs.append('oo_metaB') for dir in dirs: if os.path.isdir(dir): print "removing old %s directory" % dir shutil.rmtree(dir) def doUnzip(self): print "unzipping files..." file = self.pars['ARC_NAME'].val if not os.path.isfile(os.path.expanduser(file)): self.error("archive %s is not here!" % file) ext = os.path.splitext(file)[1] if ext==".zip": # unzip the source and try to convert text files cmd = 'unzip -a %s -x "*/.svn/*" >/dev/null' % file sysOutput = os.system(cmd) if (sysOutput != 0): self.error("unable to unzip archive %s !" % file) # no conversion for ".svn" database cmd = 'unzip %s "*/.svn/*" >/dev/null' % file sysOutput = os.system(cmd) if (sysOutput != 0): self.error("unable to unzip archive %s !" % file) # convert some text files (if zip "text compression" was disabled) dos2unix([ repo.name for repo in self.repos ], '*.txt;*.CPE;*.CRE;*.stp;*.l;*.y;*.dat') elif ext==".tgz" or ext==".tar.gz": tar = tarfile.open(os.path.expanduser(file),'r:gz') for tarinfo in tar: tar.extract(tarinfo) tar.close() else: self.error("archive %s of unknown extension!" % file) def compileMETAFOR(self): # release or debug names/flags exe='bin/Metafor' dflag='' if self.pars['DEBUG_MODE'].val: dflag='-D CMAKE_BUILD_TYPE=Debug' # first check if not os.path.isdir('oo_meta'): self.error('oo_meta not here!') # create bin dir if os.path.isdir('oo_metaB'): print 'removing old %s directory' % 'oo_metaB' shutil.rmtree('oo_metaB') os.mkdir('oo_metaB') os.chdir('oo_metaB') # configure print "configuring oo_meta" cmfile = '../oo_meta/CMake/%s' % self.pars['CMAKELIST'].val #print cmfile if not os.path.isfile(cmfile): msg = '%s not found!' % cmfile print msg self.mailmsg(msg) cmd = 'cmake -C %s %s ../oo_meta >autocf.log 2>&1' % (cmfile, dflag) #print cmd os.system(cmd) # compile ncpu = int(self.pars['NB_TASKS'].val) * int(self.pars['NB_THREADS'].val) print 'compiling %s using %s cpu(s) (have a coffee)' % (exe, ncpu) os.system('make -j %d >compile.log 2>&1' % (ncpu)) # check exe if os.path.isfile(exe) and os.access(exe, os.X_OK): msg='compilation of %s OK' % exe print msg self.mailmsg(msg, 'compile.log') else: msg='compilation of %s FAILED' % exe self.error(msg, 'compile.log') os.chdir('..') def compile(self): self.compileMETAFOR() def cleanBattery(self): os.chdir('oo_metaB/bin') print "cleaning old results" os.system("python battery.py clean >/dev/null 2>&1") os.chdir('../..') def startBat(self): now = datetime.datetime.now() print "starting battery at %s (come back tomorrow)" % now.ctime() os.chdir('oo_metaB/bin') cmd="nice -%s python battery.py -j %s -k %s >battery.log 2>&1" % (self.pars['NICE_VALUE'].val, self.pars['NB_TASKS'].val, self.pars['NB_THREADS'].val) p = subprocess.Popen(cmd, shell=True) p.wait() # finish script now = datetime.datetime.now() print "battery completed at %s" % now.ctime() self.mailmsg("battery complete", file='battery.log') os.chdir('../..') def checkResults(self): # pars indep os.chdir('oo_metaB/bin') print "diff'ing results" cmd="python battery.py diff" print "checkResults: cmd = %s" % cmd os.system(cmd) file='verif/%s-diffs.html' % machineid() print "verif file name = %s" % file #self.mailhtml(file, "html report") self.mailHtmlAsAttachement(file, "html report") print "file %s sent as attachement ..." % file os.chdir('../..') def getJobName(self): return os.path.basename(os.getcwd())+".battery" def run(self): # kill script that kills running tree self.killScript(self.jobId, os.getpgrp()) if self.pars['UNZIP'].val=="checkout": self.doClean() self.checkOut() elif self.pars['UNZIP'].val=="zip": self.doClean() self.doUnzip() self.checkOut() # only missing folders self.touchFiles() if self.pars['COMPILE'].val: self.compile() if self.pars['BATTERY'].val==True: self.cleanBattery() if not self.pars['BATTERY'].val==False: self.startBat() self.checkResults() if os.path.isfile("kill%s.py" % self.jobId): os.remove("kill%s.py" % self.jobId) if (self.pars['RUNMETHOD'].val == 'at' or self.pars['RUNMETHOD'].val == 'batch'): if os.path.isfile("atrm%s.py" % self.jobId): os.remove("atrm%s.py" % self.jobId) if os.path.isfile(self.cfgfile): os.remove(self.cfgfile) print "done." # -- main ---------------------------------------------------------------------- if __name__ == "__main__": try: import signal signal.signal(signal.SIGBREAK, sigbreak); except: pass from optparse import OptionParser parser = OptionParser() parser.add_option("-d", "--directory", dest="rundir", metavar="DIR", help="specify run directory (batch mode)") parser.add_option("-x", "--nogui", action="store_false", dest="usegui",default=True, help="disable menu") parser.add_option("-i", "--jobId", dest="jobId", type="str", default='', help="job id") (options, args) = parser.parse_args() #print "options = ", options #print "args = ", args if len(args)!=0: parser.error("too many arguments") if options.rundir: os.chdir(options.rundir) #print "options.jobId = %s"% options.jobId job = CompJob(options.jobId) if options.usegui: job.menu() else: job.run()