diff --git a/.gitignore b/.gitignore index 0ab1fdf64783d47ee74f0a18ea352325a69a8d5a..c7c2b9bf2c7f416df4a65532d801a13ca46eb07a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ *~ .vscode/ *.cfg +*.bak diff --git a/cleanLocalHdd.py b/cleanLocalHdd.py index d345543efeccebb119ec97111e2aed4402c37a0a..dfe60c6f2464c07de829a4c41167772b304c377a 100755 --- a/cleanLocalHdd.py +++ b/cleanLocalHdd.py @@ -4,11 +4,14 @@ # Script cleanLocalHdd.py : listing de l'utilisation et nettoyage des disques locaux sur le cluster # +from __future__ import print_function +from builtins import input +from builtins import range import os, sys, subprocess def listByUser(user, nodesList): - print "listing local disk use for user %s"%user - print "\t on nodes : %s"%nodesList + print("listing local disk use for user %s"%user) + print("\t on nodes : %s"%nodesList) global nbnodes outFN = 'listLocalHddUse.out' @@ -19,35 +22,35 @@ def listByUser(user, nodesList): cmd2 = "du -sh /local/%s* --exclude /local/lost+found " %user cmd="%s; %s"%(cmd1,cmd2) - print cmd + print(cmd) for inode in nodesList: - print "\n\tNode%03d\n\t========"%(inode) + print("\n\tNode%03d\n\t========"%(inode)) sshCmd = "ssh node%03d '%s'"%(inode, cmd) callOut = subprocess.call(sshCmd, shell=True) def delByUser(user, nodesList): - print "deleting local disk dirs for user %s"%user - print "\t on nodes : %s"%nodesList + print("deleting local disk dirs for user %s"%user) + print("\t on nodes : %s"%nodesList) - sure = raw_input("are you sure ? (yes/No) : ") + sure = input("are you sure ? (yes/No) : ") if sure=='' or sure.lower()!="yes": - print "Aborted ..." + print("Aborted ...") return global nbnodes outFN = 'listLocalHddUse.out' out = open(outFN, "w") cmd2 = "rm -rf /local/%s*" %user - print cmd2 + print(cmd2) for inode in nodesList: - print "\tNode%03d"%(inode) + print("\tNode%03d"%(inode)) cmd = "ssh node%03d '%s'"%(inode, cmd2) callOut = subprocess.call(cmd, shell=True) if __name__ == "__main__": nbnodes = 14 - nodesList = range(1,nbnodes+1) + nodesList = list(range(1,nbnodes+1)) from optparse import OptionParser parser = OptionParser() @@ -65,14 +68,14 @@ if __name__ == "__main__": (options, args) = parser.parse_args() - print "options.nodesList=",options.nodesList + print("options.nodesList=",options.nodesList) if options.nodesList: nodesList = options.nodesList if options.clean == True: if (options.user == '' or not options.nodesList)and options.force==False: - print "Error deleting for all users or on all nodes not allowed without --force option" + print("Error deleting for all users or on all nodes not allowed without --force option") else: delByUser(options.user, nodesList) else: diff --git a/comp.py b/comp.py index 098b550c651295506e6917eed356ad282df0b30a..98fd7e8fe85b636497f78644f8460663268bf7d5 100755 --- a/comp.py +++ b/comp.py @@ -5,6 +5,8 @@ # - compilation de Metafor # - lancement automatique de la batterie +from __future__ import print_function +from builtins import object from parametricJob import * import re @@ -26,9 +28,9 @@ class GitRepo(Repo): depth = '--depth %s' % pars['GIT_DEPTH'].val branch = pars['GIT_BRANCH'].val if not self.doesBranchExist(branch): - print 'INFO: branch %s does not exist on %s' % (branch, self.url) + print('INFO: branch %s does not exist on %s' % (branch, self.url)) branch = 'master' - print 'INFO: checking out %s' % branch + print('INFO: checking out %s' % branch) cmd = "git clone --branch %s %s --quiet %s %s" % \ (branch, depth, self.url, self.name) @@ -137,7 +139,7 @@ class CompJob(ParametricJob): def touchFiles(self): for repo in self.repos: - print "touching %s" % repo.name + print("touching %s" % repo.name) for path, dirs, files in os.walk(repo.name): for file in files: os.utime(os.path.join(path,file), None) # touch file @@ -145,9 +147,9 @@ class CompJob(ParametricJob): def checkOut(self): for repo in self.repos: if not os.path.isdir(repo.name): - print 'checking out "%s" from %s...' % (repo.name, repo.url) + print('checking out "%s" from %s...' % (repo.name, repo.url)) cmd = repo.co_cmd(self.pars) - print cmd + print(cmd) os.system(cmd) def doClean(self): @@ -155,11 +157,11 @@ class CompJob(ParametricJob): dirs.append('oo_metaB') for dir in dirs: if os.path.isdir(dir): - print "removing old %s directory" % dir + print("removing old %s directory" % dir) shutil.rmtree(dir) def doUnzip(self): - print "unzipping files..." + print("unzipping files...") file = self.pars['ARC_NAME'].val if not os.path.isfile(os.path.expanduser(file)): self.error("archive %s is not here!" % file) @@ -197,29 +199,29 @@ class CompJob(ParametricJob): self.error('oo_meta not here!') # create bin dir if os.path.isdir('oo_metaB'): - print 'removing old %s directory' % 'oo_metaB' + print('removing old %s directory' % 'oo_metaB') shutil.rmtree('oo_metaB') os.mkdir('oo_metaB') os.chdir('oo_metaB') # configure - print "configuring oo_meta" + print("configuring oo_meta") cmfile = '../oo_meta/CMake/%s' % self.pars['CMAKELIST'].val #print cmfile if not os.path.isfile(cmfile): msg = '%s not found!' % cmfile - print msg + print(msg) self.mailmsg(msg) cmd = 'cmake -C %s %s ../oo_meta >autocf.log 2>&1' % (cmfile, dflag) #print cmd os.system(cmd) # compile ncpu = int(self.pars['NB_TASKS'].val) * int(self.pars['NB_THREADS'].val) - print 'compiling %s using %s cpu(s) (have a coffee)' % (exe, ncpu) + print('compiling %s using %s cpu(s) (have a coffee)' % (exe, ncpu)) os.system('make -j %d >compile.log 2>&1' % (ncpu)) # check exe if os.path.isfile(exe) and os.access(exe, os.X_OK): msg='compilation of %s OK' % exe - print msg + print(msg) self.mailmsg(msg, 'compile.log') else: msg='compilation of %s FAILED' % exe @@ -231,34 +233,34 @@ class CompJob(ParametricJob): def cleanBattery(self): os.chdir('oo_metaB/bin') - print "cleaning old results" + print("cleaning old results") os.system("python battery.py clean >/dev/null 2>&1") os.chdir('../..') def startBat(self): now = datetime.datetime.now() - print "starting battery at %s (come back tomorrow)" % now.ctime() + print("starting battery at %s (come back tomorrow)" % now.ctime()) os.chdir('oo_metaB/bin') cmd="nice -%s python battery.py -j %s -k %s >battery.log 2>&1" % (self.pars['NICE_VALUE'].val, self.pars['NB_TASKS'].val, self.pars['NB_THREADS'].val) p = subprocess.Popen(cmd, shell=True) p.wait() # finish script now = datetime.datetime.now() - print "battery completed at %s" % now.ctime() + print("battery completed at %s" % now.ctime()) self.mailmsg("battery complete", file='battery.log') os.chdir('../..') def checkResults(self): # pars indep os.chdir('oo_metaB/bin') - print "diff'ing results" + print("diff'ing results") cmd="python battery.py diff" - print "checkResults: cmd = %s" % cmd + print("checkResults: cmd = %s" % cmd) os.system(cmd) file='verif/%s-diffs.html' % machineid() - print "verif file name = %s" % file + print("verif file name = %s" % file) #self.mailhtml(file, "html report") self.mailHtmlAsAttachement(file, "html report") - print "file %s sent as attachement ..." % file + print("file %s sent as attachement ..." % file) os.chdir('../..') def getJobName(self): @@ -297,7 +299,7 @@ class CompJob(ParametricJob): if os.path.isfile(self.cfgfile): os.remove(self.cfgfile) - print "done." + print("done.") # -- main ---------------------------------------------------------------------- diff --git a/externalProgramPath.py b/externalProgramPath.py index b268356d394eaac8a298636deadbe80197b367e2..208b8c9da5ec57e52325102856dd0c67d3d29627 100755 --- a/externalProgramPath.py +++ b/externalProgramPath.py @@ -3,6 +3,7 @@ # # Define external program paths according to local configuration +from __future__ import print_function from prmClasses import * import os, os.path, distutils.spawn @@ -59,20 +60,20 @@ class ExtProgs(PRMSet): def checkValidity(self, key): if key not in self.pars : - print "%s is not a valid external program key from externalProgramPath... "%key - print "Solutions : " - print "\t 1. Check the key validity (must be in capital letter)." - print "\t 2. Update the 'externalProgramPath' Script and rerun the configuration." - print "\t 3. If %s not in the Script last version, add it program in the Script..."%key + print("%s is not a valid external program key from externalProgramPath... "%key) + print("Solutions : ") + print("\t 1. Check the key validity (must be in capital letter).") + print("\t 2. Update the 'externalProgramPath' Script and rerun the configuration.") + print("\t 3. If %s not in the Script last version, add it program in the Script..."%key) return False if distutils.spawn.find_executable(os.path.splitext(self.pars[key].val)[0]) : return True else: - print "%s is not found in %s ..."%(key, self.pars[key].val) - print "\t Check installation and accessibility..." - print "\t Use 'externalProgramPathGui' to define the full program path (recommanded)" - print "\t or add %s in your user path (not recommanded)"%key + print("%s is not found in %s ..."%(key, self.pars[key].val)) + print("\t Check installation and accessibility...") + print("\t Use 'externalProgramPathGui' to define the full program path (recommanded)") + print("\t or add %s in your user path (not recommanded)"%key) return False def configAction(self): diff --git a/launch.py b/launch.py index 223fa114f2b498f8bb35d4395034736ad79cf9e4..f884d57b2f3dc333e57e8241ff93d611e62b4199 100755 --- a/launch.py +++ b/launch.py @@ -3,6 +3,7 @@ # # Script "launch.py": aide au lancement d'un job Metafor +from __future__ import print_function import sys, os, os.path, subprocess import shutil, socket, platform, glob, fnmatch, re import datetime, tarfile, signal @@ -123,15 +124,15 @@ class LaunchJob(ParametricJob): def applyDependencies(self): ret = False if self.debug: - print "applyDependecies: " - print " self.pars['ALGORITHM'].val = ", self.pars['ALGORITHM'].val + print("applyDependecies: ") + print(" self.pars['ALGORITHM'].val = ", self.pars['ALGORITHM'].val) #if self.pars['ALGORITHM'].val=='restart': # self.pars['MULTITEST'].val = False # ret = True if self.debug: - print " self.pars['MULTITEST'].val = ", self.pars['MULTITEST'].val + print(" self.pars['MULTITEST'].val = ", self.pars['MULTITEST'].val) return ret def configActions(self): @@ -200,7 +201,7 @@ class LaunchJob(ParametricJob): def getOutFileName(self): outFileName = "%s.%s-%s.txt" % (self.pars['OUTFILE'].val, self.pars['ALGORITHM'].val,self.jobId) if self.debug: - print "outFileName = ", outFileName + print("outFileName = ", outFileName) return outFileName # RUN Functions @@ -229,14 +230,14 @@ class LaunchJob(ParametricJob): if (self.pars['MULTITEST'].val==True): #check if not os.path.isdir(self.pars['TEST_DIR'].val): - print "Error: 'TEST_DIR' %s non existant directory" % self.pars['TEST_DIR'].val + print("Error: 'TEST_DIR' %s non existant directory" % self.pars['TEST_DIR'].val) return asciiname = self.pars['TEST_DIR'].val.encode('ascii','ignore') # convert unicode (from PyQt) outRun = self.startMultipleTests(asciiname) else: #check if not os.path.isfile(self.pars['TEST_NAME'].val): - print "Error: 'TEST_NAME' %s non existant file" % self.pars['TEST_NAME'].val + print("Error: 'TEST_NAME' %s non existant file" % self.pars['TEST_NAME'].val) return #run asciiname = self.pars['TEST_NAME'].val.encode('ascii','ignore') # convert unicode (from PyQt) @@ -251,7 +252,7 @@ class LaunchJob(ParametricJob): #tar facs cdir=os.path.basename(os.getcwd()) tarname="%s.tar.gz" % cdir - print "creating %s" % tarname + print("creating %s" % tarname) os.chdir('..') tar = tarfile.open(tarname,'w:gz') for path, dirs, files in os.walk(cdir): @@ -260,7 +261,7 @@ class LaunchJob(ParametricJob): tar.close() #ftp - print "sending results to %s" % self.pars['FTP_HOST'].val + print("sending results to %s" % self.pars['FTP_HOST'].val) import ftplib ftp = ftplib.FTP() ftp.connect(self.pars['FTP_HOST'].val, self.pars['FTP_PORT'].val) @@ -292,13 +293,13 @@ class LaunchJob(ParametricJob): for fil in fNames: if os.path.isfile(fil): os.remove(fil) - print "done." + print("done.") def startMultipleTests(self, tests): - print "startMultipleTests" + print("startMultipleTests") # starting timer now = datetime.datetime.now() - print "starting Multiple test at %s (come back later)" % now.ctime() + print("starting Multiple test at %s (come back later)" % now.ctime()) # opening outfile self.outFile = open(self.getOutFileName(),"w") @@ -357,16 +358,16 @@ class LaunchJob(ParametricJob): m = exp1.match(os.path.basename(tests)) if m: # chaining tests pin.write('battery.addCplxExecPath(r"%s_*")\n'%os.path.join(os.path.dirname(tests), m.group(1))) - print 'battery.cplx_exec = [r"%s_*"]\n'%os.path.join(os.path.dirname(tests), m.group(1)) + print('battery.cplx_exec = [r"%s_*"]\n'%os.path.join(os.path.dirname(tests), m.group(1))) else: pin.write('battery.addCplxExecPath(r"%s")\n'%tests) - print 'battery.cplx_exec = [r"%s"]\n'%tests + print('battery.cplx_exec = [r"%s"]\n'%tests) elif (self.pars['ALGORITHM'].val == "import" ): pin.write('battery.addCplxImportPath(r"%s")\n'%tests) - print 'battery.cplx_import = [r"%s"]\n'%tests + print('battery.cplx_import = [r"%s"]\n'%tests) elif (self.pars['ALGORITHM'].val == "restart" ): pin.write('battery.addRestartPath(r"%s")\n'%tests) - print 'battery.restart = [r"%s"]\n'%tests + print('battery.restart = [r"%s"]\n'%tests) pin.write('battery.verifsrc = "verif"\n') pin.write('battery.codes = [ "FAILED", "STP", "ITE", "INW", "EXT", "EXW", "LKS", "CPU", "MEM" ]\n') @@ -415,11 +416,11 @@ class LaunchJob(ParametricJob): # post pro cmd if ((self.pars['RUNMETHOD'].val == 'sge' or self.pars['RUNMETHOD'].val == 'slurm') and self.pars['LOCALDISK'].val == True): - print "Trying to get back local workspace to home" + print("Trying to get back local workspace to home") self.moveLocalDir2Home(self.jobId) now = datetime.datetime.now() - print "battery completed at %s" % now.ctime() + print("battery completed at %s" % now.ctime()) if self.pars['SEND_MAIL'].val == True: self.mailmsg("multipleTests complete", file=self.getOutFileName()) return retcode diff --git a/parametricJob.py b/parametricJob.py index ffb8ec087dd9733c357c79ce3ad8f25dd59e8bf2..f351a8a9339c34ae693407f2277d05a4cde4dd8c 100644 --- a/parametricJob.py +++ b/parametricJob.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- +from __future__ import print_function +from future import standard_library +standard_library.install_aliases() +from builtins import input import sys, os, os.path, shutil, socket, platform, glob, fnmatch import datetime, tarfile, subprocess, string from prmClasses import * @@ -41,14 +45,14 @@ class ParametricJob(PRMSet): toAddr = "%s@%s"%(self.pars['MAIL_ADDR'].val,socket.gethostbyaddr(socket.gethostname())[0]) smtpServ = "localhost" if 0: - print "mailData:" - print "\tfromAddr = %s" % fromAddr - print "\ttoAddr = %s" % toAddr - print "\tsmtpServ = %s" % smtpServ + print("mailData:") + print("\tfromAddr = %s" % fromAddr) + print("\ttoAddr = %s" % toAddr) + print("\tsmtpServ = %s" % smtpServ) return fromAddr, toAddr, smtpServ def mailmsg(self, msg="no subject", file=None, text=None): - print 'mailmsg with subject "%s"' % msg + print('mailmsg with subject "%s"' % msg) # getting address & smtp servers fromA, toA, smtpServ = self.getMailData() #subject = "[%s] %s : %s" % (os.path.basename(sys.argv[0]), socket.gethostname(), msg) @@ -68,7 +72,7 @@ class ParametricJob(PRMSet): server.quit() def mailhtml(self, file, subject): - print 'mailhtml with subject "%s"' % subject + print('mailhtml with subject "%s"' % subject) # getting address & smtp servers fromA, toA, smtpServ = self.getMailData() # building email @@ -89,7 +93,7 @@ class ParametricJob(PRMSet): smtp.close() def mailHtmlAsAttachement(self, fileName, subject): - print 'mailHtmlAsAttachement with subject "%s"' % subject + print('mailHtmlAsAttachement with subject "%s"' % subject) import mimetypes from email import encoders from email.mime.multipart import MIMEMultipart @@ -119,11 +123,11 @@ class ParametricJob(PRMSet): maintype, subtype = ctype.split('/', 1) try: file = open(fileName,'r') - print "file %s correctly opened for mailing" % fileName + print("file %s correctly opened for mailing" % fileName) msg = MIMEBase(maintype, subtype) msg.set_payload(file.read()) file.close() - print "file %s correctly closed after mailing" % fileName + print("file %s correctly closed after mailing" % fileName) except smtplib.SMTPException: text="file %s not found"%fileName # Encode the payload using Base64 @@ -133,19 +137,19 @@ class ParametricJob(PRMSet): msg.add_header('Content-Disposition', 'attachment', filename=newFileName) mail.attach(msg) try: - print "opening smtp" + print("opening smtp") import smtplib smtp = smtplib.SMTP(smtpServ) #smtp.set_debuglevel(True) smtp.sendmail(fromA, toA, mail.as_string()) smtp.close() - print "closing smtp" - except smtplib.SMTPException, e: - print "mailHtmlAsAttachement: error during smtp sendmail !!!" - print "smtplib.SMTPException returned: %s"%e + print("closing smtp") + except smtplib.SMTPException as e: + print("mailHtmlAsAttachement: error during smtp sendmail !!!") + print("smtplib.SMTPException returned: %s"%e) def error(self, msg="error", file=None): - print "**ERROR: %s" % msg + print("**ERROR: %s" % msg) self.mailmsg(msg, file) sys.exit(1) @@ -160,8 +164,8 @@ class ParametricJob(PRMSet): return cfgfile def hasSysCmd(self, cmd): - import commands - status, result = commands.getstatusoutput("which %s" % cmd) + import subprocess + status, result = subprocess.getstatusoutput("which %s" % cmd) return status==0 # Use Local Disk SPECIFIC (SGE / SLURM) @@ -195,7 +199,7 @@ class ParametricJob(PRMSet): file.write("\tprint 'Check manually what went wrong before cleaning %s local disk'\n"%(nodeHost)) file.write("sys.exit(outCp)") file.close() - os.chmod(filename,0700) + os.chmod(filename,0o700) def rmNodeResultsScript(self, jobId): nodeHost = socket.gethostname() @@ -218,12 +222,12 @@ class ParametricJob(PRMSet): file.write("\tprint 'Check manually how to clean %s local disk'\n"%(nodeHost)) file.write("sys.exit(outRm)") file.close() - os.chmod(filename,0700) + os.chmod(filename,0o700) def moveLocalDir2Home(self, jobId): localNodeDir = self.getLocalDiskDir(jobId) homeDir=os.getcwd() - print "trying to move %s/* to %s"%(localNodeDir,homeDir) + print("trying to move %s/* to %s"%(localNodeDir,homeDir)) #shutil.move(localNodeDir,homeDir) # do not work if dst dir exist or it exist => use of "cp -r"!!! try: # -R : recursif / p : preserve attribut (owner/mode/timestamp) / u : update (copy only if source is newer than target)/ v : verbose ##cmd1 = "cp -Rpuv %s/* %s"%(localNodeDir, homeDir) @@ -232,7 +236,7 @@ class ParametricJob(PRMSet): cmd1 = "rsync -avz %s/* %s"%(localNodeDir, homeDir) #--remove-source-files permet de nettoyer la source, mais ca risque de poser problème avec le check ci dessous # qui plus est, ne supprime pas l'arborescence, juste les fichiers => nettoyage incomplet - print "cmd1 = ", cmd1 + print("cmd1 = ", cmd1) subprocess.call([cmd1],stderr=subprocess.STDOUT, shell=True) #use of subprocess to be able to catch errors #execfile(self.cpNodeResultsScriptName(jobId)) # check que la copie soit bonne (même fichiers des 2 cotés) @@ -240,22 +244,22 @@ class ParametricJob(PRMSet): cmp = filecmp.dircmp(localNodeDir,homeDir) #print "cmp.report() = ",cmp.report() if recCmp(cmp): # copie parfaite => nettoyage brutal de l'arborescence - print "copie parfaite => nettoyage brutal de l'arborescence " + print("copie parfaite => nettoyage brutal de l'arborescence ") cmd2 = "rm -rf %s"%localNodeDir - print "cmd2 = ", cmd2 + print("cmd2 = ", cmd2) subprocess.call([cmd2],stderr=subprocess.STDOUT, shell=True) # 2 commands for not deleting files if copy throw an exception os.remove(self.cpNodeResultsScriptName(jobId)) os.remove(self.rmNodeResultsScriptName(jobId)) # suppression des scripts else: # on va au moins nettoyer ce qui est commun - print "copie imparfaite => nettoyage de ce qui est commun" + print("copie imparfaite => nettoyage de ce qui est commun") os.chdir(localNodeDir) rmCommonFiles(cmp) os.chdir(homeDir) - except OSError, e: #except OSError as e: dont work on blueberry - print "unable to get back files from local directory" - print "subprocess returned error: ",e - print "get back result files using %s "%self.cpNodeResultsScriptName(jobId) + except OSError as e: #except OSError as e: dont work on blueberry + print("unable to get back files from local directory") + print("subprocess returned error: ",e) + print("get back result files using %s "%self.cpNodeResultsScriptName(jobId)) # End of Use Local Disk Specific #=========================================================================================================================== # AT/BATCH SPECIFIC @@ -272,24 +276,24 @@ class ParametricJob(PRMSet): file.write("echo \"at JobId = $jobId\"\n") file.write('%s -x -i $jobId -d "%s"\n' % (sys.argv[0], os.getcwd()) ) file.close() - os.chmod(scriptname,0700) - print "starting script in batch mode: %s" % scriptname + os.chmod(scriptname,0o700) + print("starting script in batch mode: %s" % scriptname) #shcmd="at %s -f %s" % (self.pars['BATCHTIME'].val, scriptname) if self.pars['RUNMETHOD'].val == "at" : shcmd="echo \"/bin/bash %s\" | at %s" % (scriptname, self.pars['AT_TIME'].val) # keep it like that else job may start in dash !!! elif self.pars['RUNMETHOD'].val == "batch" : shcmd="echo \"/bin/bash %s\" | batch " % (scriptname) # keep it like that else job may start in dash !!! else : - print "error runBatch must be batch or at" + print("error runBatch must be batch or at") #shcmd="at now + 1 minutes -f %s" % (scriptname) #shcmd="at now %s" % scriptname - print "shcmd = ", shcmd - import commands - status, result = commands.getstatusoutput(shcmd) + print("shcmd = ", shcmd) + import subprocess + status, result = subprocess.getstatusoutput(shcmd) if status!=0: - print "Job submission FAILED!" + print("Job submission FAILED!") else: - print "Submission SUCCESSFUL!" + print("Submission SUCCESSFUL!") #print "result = ", result import re m = re.search('job ([0-9]+)',result) @@ -299,11 +303,11 @@ class ParametricJob(PRMSet): cfgFileName, cfgFileExtension = os.path.splitext(self.cfgfile) os.system("cp %s %s%s%s"%(self.cfgfile, cfgFileName, batchId, cfgFileExtension)) self.atrmScript(batchId) - print "\tuse 'atq' and find job number %s to check the status of your job" % batchId - print "\t\t - 'a' means waiting in the queue 'a'" - print "\t\t - '=' means running" - print "\tuse 'atrm %s to kill the job" % batchId - print "\t\t or 'atrm%s.py' to kill the job" % batchId + print("\tuse 'atq' and find job number %s to check the status of your job" % batchId) + print("\t\t - 'a' means waiting in the queue 'a'") + print("\t\t - '=' means running") + print("\tuse 'atrm %s to kill the job" % batchId) + print("\t\t or 'atrm%s.py' to kill the job" % batchId) sys.exit() def atrmScript(self, pid): @@ -315,7 +319,7 @@ class ParametricJob(PRMSet): file.write("if os.path.isfile('kill%s.py'):\n" % pid) file.write("\texecfile('kill%s.py')\n" % pid) file.close() - os.chmod(filename,0700) + os.chmod(filename,0o700) def killScript(self, jobId, pid): filename = "kill%s.py"%jobId @@ -324,7 +328,7 @@ class ParametricJob(PRMSet): s="import os, signal; os.killpg(%d, signal.SIGKILL)\n" % pid file.write(s) file.close() - os.chmod(filename,0700) + os.chmod(filename,0o700) #=========================================================================================================================== # SGE SPECIFIC @@ -333,7 +337,7 @@ class ParametricJob(PRMSet): cfgfile = self.guessProfile() # do some checks specific to SGE (may change pars) if self.pars['NICE_VALUE'].val!='0': - print "Warning: NICE_VALUE will be ignored (reset to 0)!!" + print("Warning: NICE_VALUE will be ignored (reset to 0)!!") self.pars['NICE_VALUE'].val='0' # build script @@ -367,15 +371,15 @@ class ParametricJob(PRMSet): file.write("%s -x -i $JOB_ID\n" % (sys.argv[0])) file.close() # send to sge - print "sending job '%s' to SGE" % jobname + print("sending job '%s' to SGE" % jobname) shcmd="qsub ./%s" % scriptname - import commands - status, result = commands.getstatusoutput(shcmd) - print result + import subprocess + status, result = subprocess.getstatusoutput(shcmd) + print(result) if status!=0: - print "Job submission FAILED!" + print("Job submission FAILED!") else: - print "Submission SUCCESSFUL!" + print("Submission SUCCESSFUL!") import re m = re.compile('Your job ([0-9]+)').match(result) sgeId = m.group(1) @@ -383,11 +387,11 @@ class ParametricJob(PRMSet): cfgFileName, cfgFileExtension = os.path.splitext(self.cfgfile) os.system("cp %s %s%s%s"%(self.cfgfile, cfgFileName, sgeId, cfgFileExtension)) self.qDelScript(sgeId) - print "\tuse 'qstat -f -j %s' to check the status of your job" % sgeId - print "\tuse 'qdel %s' to kill your job" % sgeId - print "\tuse './%s' to get results from node disk" % self.cpNodeResultsScriptName(sgeId) - print "\tuse './%s' to clean results from node disk" % self.rmNodeResultsScriptName(sgeId) - print "\tuse './qDel%s.py' to kill your job, get results and clean node disk" % sgeId + print("\tuse 'qstat -f -j %s' to check the status of your job" % sgeId) + print("\tuse 'qdel %s' to kill your job" % sgeId) + print("\tuse './%s' to get results from node disk" % self.cpNodeResultsScriptName(sgeId)) + print("\tuse './%s' to clean results from node disk" % self.rmNodeResultsScriptName(sgeId)) + print("\tuse './qDel%s.py' to kill your job, get results and clean node disk" % sgeId) sys.exit() def qDelScriptName(self, jobId): @@ -427,7 +431,7 @@ class ParametricJob(PRMSet): file.write("os.remove('./%s')\n"%(filename)) file.write("sys.exit(0)\n") file.close() - os.chmod(filename,0700) + os.chmod(filename,0o700) # END OF SGE SPECIFIC #=========================================================================== # SLURM SPECIFIC INTERFACE @@ -458,17 +462,17 @@ class ParametricJob(PRMSet): file.write(". %s %s\n" % (cfgfile, socket.gethostname())) file.write("srun %s -x -i $SLURM_JOB_ID \n" % (sys.argv[0])) file.close() - os.chmod(scriptname,0700) + os.chmod(scriptname,0o700) # send to slurm - print "sending job '%s' to Slurm" % jobname + print("sending job '%s' to Slurm" % jobname) shcmd="sbatch ./%s" % scriptname - import commands - status, result = commands.getstatusoutput(shcmd) - print result + import subprocess + status, result = subprocess.getstatusoutput(shcmd) + print(result) if status!=0: - print "Job submission FAILED!" + print("Job submission FAILED!") else: - print "Submission SUCCESSFUL!" + print("Submission SUCCESSFUL!") import re m = re.compile('Submitted batch job ([0-9]+)').match(result) slurmId = m.group(1) @@ -476,11 +480,11 @@ class ParametricJob(PRMSet): cfgFileName, cfgFileExtension = os.path.splitext(self.cfgfile) os.system("cp %s %s%s%s"%(self.cfgfile, cfgFileName, slurmId, cfgFileExtension)) self.sCancelScript(slurmId) - print "\tuse ' squeue -l -j %s ' to check the status of the SLURM scheduling queue of your job" % slurmId - print "\tuse ' sprio -l -j %s ' to check the factor priority of your job" % slurmId - print "\tuse ' sstat -a --format=JobID,NTasks,MaxRSS,MaxVMSize -j %s ' to get information about your running job (adapt format to your needs)" % slurmId - print "\tuse ' scancel %s ' to kill your job" % slurmId - print "\tuse ' sacct --format=JobID,NTasks,NCPUS,CPUTime,Elapsed,MaxRSS,MaxVMSize -j %s ' to get information about your finished job (adapt format to your needs)" % slurmId + print("\tuse ' squeue -l -j %s ' to check the status of the SLURM scheduling queue of your job" % slurmId) + print("\tuse ' sprio -l -j %s ' to check the factor priority of your job" % slurmId) + print("\tuse ' sstat -a --format=JobID,NTasks,MaxRSS,MaxVMSize -j %s ' to get information about your running job (adapt format to your needs)" % slurmId) + print("\tuse ' scancel %s ' to kill your job" % slurmId) + print("\tuse ' sacct --format=JobID,NTasks,NCPUS,CPUTime,Elapsed,MaxRSS,MaxVMSize -j %s ' to get information about your finished job (adapt format to your needs)" % slurmId) sys.exit() def sCancelScriptName(self, jobId): @@ -521,7 +525,7 @@ class ParametricJob(PRMSet): file.write("os.remove('./%s')\n"%(filename)) file.write("sys.exit(0)\n") file.close() - os.chmod(filename,0700) + os.chmod(filename,0o700) # END OF SLURM SPECIFIC INTERFACE #=========================================================================== # PBS SPECIFIC INTERFACE (Zenobe) @@ -540,8 +544,8 @@ class ParametricJob(PRMSet): # Mail if ( self.pars['MAIL_ADDR'].val == os.getenv('USER') or string.find(self.pars['MAIL_ADDR'].val, "@") < 0 ): - print "ERROR : PBSPro : Only external email allowed : correct it before launch" - raw_input("press enter to get back to configuration before launch") + print("ERROR : PBSPro : Only external email allowed : correct it before launch") + input("press enter to get back to configuration before launch") return file.write('#PBS -M %s\n'%self.pars['MAIL_ADDR'].val) file.write('#PBS -m abe\n') # mail abort/begin/end @@ -584,26 +588,26 @@ class ParametricJob(PRMSet): file.write('%s -x -i $PBS_JOBID\n' % (sys.argv[0])) file.write('qstat -f $PBS_JOBID\n') file.close() - os.chmod(scriptname,0700) + os.chmod(scriptname,0o700) # send to PBS - print 'sending job "%s" to PBS' % jobname + print('sending job "%s" to PBS' % jobname) shcmd='qsub ./%s' % scriptname - import commands - status, result = commands.getstatusoutput(shcmd) - print 'command status = ', status - print 'command result = ', result + import subprocess + status, result = subprocess.getstatusoutput(shcmd) + print('command status = ', status) + print('command result = ', result) if status!=0: - print "Job submission FAILED!" + print("Job submission FAILED!") else: - print "Submission SUCCESSFUL!" + print("Submission SUCCESSFUL!") # le pbsid est le retour de qsub : 3067709.frontal2 PbsId = result cfgFileName, cfgFileExtension = os.path.splitext(self.cfgfile) os.system("cp %s %s%s%s"%(self.cfgfile, cfgFileName, PbsId, cfgFileExtension)) self.PBSCancelScript(PbsId) - print "\tuse ' qstat -f %s ' to check the status of the PBS scheduling queue of your job" % PbsId - print "\tuse ' qdel %s ' to kill your job" % PbsId + print("\tuse ' qstat -f %s ' to check the status of the PBS scheduling queue of your job" % PbsId) + print("\tuse ' qdel %s ' to kill your job" % PbsId) ''' #import re #m = re.compile('waiting for job ([0-9]+) to start').match(result) @@ -637,23 +641,23 @@ class ParametricJob(PRMSet): file.write("os.remove('./%s')\n"%(filename)) file.write("sys.exit(0)\n") file.close() - os.chmod(filename,0700) + os.chmod(filename,0o700) # END OF PBS SPECIFIC INTERFACE #=========================================================================== # interface virtuelle... def run(self, sgeId=0): - print "run : not implemented" + print("run : not implemented") def setDefaultPars(self): - print "setDefaultPars : not implemented" + print("setDefaultPars : not implemented") def configActions(self): - print "configActions : not implemented" + print("configActions : not implemented") def applyDependencies(self): - print "applyDependencies : not implemented" + print("applyDependencies : not implemented") def go(self): self.savePars() - print "go in %s" % self.pars['RUNMETHOD'].val + print("go in %s" % self.pars['RUNMETHOD'].val) if (self.pars['RUNMETHOD'].val == 'at' or self.pars['RUNMETHOD'].val == 'batch'): self.runBatch() @@ -671,12 +675,12 @@ def recCmp(cmp): copyOk = True #print "cmp.left = ", cmp.left if len(cmp.left_only) != 0:# on a des fichiers manquants ou différents - print "local files only : ", cmp.left_only + print("local files only : ", cmp.left_only) copyOk = False if len(cmp.diff_files) != 0: # fichiers differents - print "files differents : ", cmp.diff_files + print("files differents : ", cmp.diff_files) copyOk = False - for subDirCmp in cmp.subdirs.values() : + for subDirCmp in list(cmp.subdirs.values()) : copyOk = recCmp(subDirCmp) # recursive function #if not copyOk : faire un break pour limiter le calcul #return copyOk @@ -688,9 +692,9 @@ def rmCommonFiles(cmp): os.chdir(subdir) subDirCmp = cmp.subdirs[subdir] rmCommonFiles(subDirCmp) # recursive function - print "fileToBeRemoved in %s: "%os.getcwd() + print("fileToBeRemoved in %s: "%os.getcwd()) for f in subDirCmp.common_files: - print f + print(f) os.remove(f) os.chdir('..') if os.path.isempty(subdir): @@ -698,9 +702,9 @@ def rmCommonFiles(cmp): # -- Misc Utilities -- def getUsername(): - if os.environ.has_key('USER'): + if 'USER' in os.environ: return os.environ['USER'] - elif os.environ.has_key('USERNAME'): + elif 'USERNAME' in os.environ: return os.environ['USERNAME'] else: return "unknown" @@ -741,9 +745,9 @@ def all_files(root, patterns, skips, single_level, yield_folders): def dos2unix( roots, patterns ): """ example: dos2unix([ 'copra5' ], '*.CPE;*.CRE') """ - print "dos2unix: analysing %s" % patterns + print("dos2unix: analysing %s" % patterns) for root in roots: - print "\t=> processing %s" % root + print("\t=> processing %s" % root) for file in all_files(root, patterns, '.svn', single_level=False, yield_folders=False): fromfile = os.path.abspath(file) diff --git a/postProLoop.py b/postProLoop.py index 4c80473d37193eacf514eec3e247bc8f8f4df39d..5dc628cb99db305874034b8a4fc01050bf257ad2 100755 --- a/postProLoop.py +++ b/postProLoop.py @@ -5,6 +5,7 @@ # # Define externals program path according to local configuration # +from __future__ import print_function from prmClasses import * import os, os.path, distutils.spawn, time, sys, re, glob import imp @@ -150,7 +151,7 @@ class PostProLoop(PRMSet): def loopOnDirTree(self, wDir): # change to workingDirectory - print "loopOnDirTree running on ", wDir + print("loopOnDirTree running on ", wDir) oldDir = os.getcwd() os.chdir(wDir) # loop on dir @@ -177,69 +178,69 @@ class PostProLoop(PRMSet): return False def execInDir(self): - print "execInDir running in ", os.path.abspath('.') + print("execInDir running in ", os.path.abspath('.')) if self.pars['MATLABRUN'].val and self.checkRequest(self.pars['MATLABREQUEST'].val): - print "launch matlab..." + print("launch matlab...") out = execMatlabScript(self.pars['MATLABEXE'].val, self.pars['MATLABCMD'].val , self.pars['MATLABPATH'].val) #out = matlab.execMatlabScript(self.pars['MATLABCMD'].val , self.pars['MATLABPATH'].val) - print "matlab done" + print("matlab done") if self.pars['GSRUN'].val and self.checkRequest(self.pars['GSREQUEST'].val): - print "launch Ghostscript translation of eps files..." + print("launch Ghostscript translation of eps files...") #execGhostScriptPostPro(device='png256', outExt='png',res=300) out = execGhostScript(self.pars['GSEXE'].val ,self.pars['GSFILTER'].val ,self.pars['GSOUTPUTFORMAT'].val , int(self.pars['GSDEFINITION'].val)) #out = ghostScript.execGhostScriptPostPro(self.pars['GSOUTPUTFORMAT'].val , 'png', int(self.pars['GSDEFINITION'].val)) - print "Ghostscript done " + print("Ghostscript done ") if self.pars['PYTHONRUN'].val and self.checkRequest(self.pars['PYTHONREQUEST'].val): - print "launch Python script..." + print("launch Python script...") out = execPythonScript(self.pars['PYTHONMODULE'].val, self.pars['PYTHONSCRIPT'].val) - print "Python script done " + print("Python script done ") if self.pars['LATEXRUN'].val and self.checkRequest(self.pars['LATEXREQUEST'].val): - print "launch latex building file ..." - print "NOT IMPLEMENTED YIET " + print("launch latex building file ...") + print("NOT IMPLEMENTED YIET ") #out = latex.execMatlabScript(self.pars['MATLABCMD'].val , self.pars['MATLABPATH'].val) - print "latex building file done " + print("latex building file done ") #================================================================================= #================================================================================= def execMatlabScript(matlabExe, matlabCmd, mFilePath=None): tim0 = time.time() - print "entering MatlabPostPro compute" + print("entering MatlabPostPro compute") inirep = os.getcwd() - print "inirep = ", inirep + print("inirep = ", inirep) moutfile = os.path.abspath(os.path.join("matlab.log")) - print "moutfile = ",moutfile + print("moutfile = ",moutfile) mCmd = "" if mFilePath: - print "mFilePath = ",mFilePath + print("mFilePath = ",mFilePath) mCmd = mCmd + "addpath('%s'); " %mFilePath mCmd = mCmd + "cd '%s'; "%inirep mCmd = mCmd + matlabCmd - print "mCmd = ", mCmd + print("mCmd = ", mCmd) if isUnix(): cmd = '"%s" -nodisplay -logfile "%s" -r "%s; quit" > pipo 2>&1' % (matlabExe, moutfile, mCmd) else: cmd = '"%s" -automation -noFigureWindows -nodesktop -wait -logfile "%s" -r "%s; quit"' % (matlabExe, moutfile, mCmd) # work - print 'complete Matlab subprocess command: ', cmd + print('complete Matlab subprocess command: ', cmd) # acquire matlock matlock.acquire() try: - print "Running Matlab..." + print("Running Matlab...") import subprocess matlabProcess = subprocess.call(cmd, shell=True) - print "Running Matlab... Done" + print("Running Matlab... Done") a = 1.0 except: # fonction objectif doit retourner un double - print "problem during matlab run" + print("problem during matlab run") a = 0.0 # release matlock matlock.release() - print " matlab time = %d sec" %(time.time()-tim0) + print(" matlab time = %d sec" %(time.time()-tim0)) return a #================================================================================= #================================================================================= @@ -258,7 +259,7 @@ def execGhostScript(gsExe, inFilter = '*.eps', device='png256', res=300): elif re.match('tiff', device): outExt = 'tiff' else: - print "device extension not implented!!! => exit" + print("device extension not implented!!! => exit") return # fonction objectif doit retourner un double a = 0 @@ -280,11 +281,11 @@ def execGhostScript(gsExe, inFilter = '*.eps', device='png256', res=300): print "subprocess return = ", ret ''' except: - print "problem during ghostScript run" + print("problem during ghostScript run") a = 0.0 fileOut.write("\nexecGhostScriptPostPro done in %d sec\n"%(time.time()-tim0)) fileOut.close() - print "execGhostScriptPostPro: %d files translated in %d sec"%(a, (time.time()-tim0)) + print("execGhostScriptPostPro: %d files translated in %d sec"%(a, (time.time()-tim0))) return a #------------------------------------------------------------------------------------ @@ -297,19 +298,19 @@ def execPythonScript(pythonModule, pythonCmd): [baseFName,extFName] = os.path.splitext(fName) (file, pathName, description) = imp.find_module(baseFName, [pName]) module = imp.load_module(baseFName, file, pathName, description) - print "python module loaded: ", pythonModule + print("python module loaded: ", pythonModule) except: - print "execPythonScript Error: unable to find or load pythonModule: ", pythonModule + print("execPythonScript Error: unable to find or load pythonModule: ", pythonModule) try: - print "method to search in module: ", pythonCmd + print("method to search in module: ", pythonCmd) m = re.match('(.*)\((.*)\)', pythonCmd) #print "re.match(...) = ", m if m: - print "method ", repr(m.group(1)), " found... with arg " ,repr(m.group(2)) + print("method ", repr(m.group(1)), " found... with arg " ,repr(m.group(2))) method = getattr(module, m.group(1)) - print dir(method) - print "method ", m.group(1), " loaded..." + print(dir(method)) + print("method ", m.group(1), " loaded...") if len(m.group(2)) > 0: args = m.group(2).split(',') argsVals=[] @@ -318,21 +319,21 @@ def execPythonScript(pythonModule, pythonCmd): method(*argsVals) else: method() - print "method ", m.group(1),'(',m.group(2),')', " successfully called..." + print("method ", m.group(1),'(',m.group(2),')', " successfully called...") else: method = getattr(module, pythonCmd) method() - print "method ", pythonCmd,'()', " called..." + print("method ", pythonCmd,'()', " called...") - print "execPythonScript: compute done" + print("execPythonScript: compute done") a=1.0 except: - print "problem during python cmd ",pythonCmd, " from module ", pythonModule, " run" + print("problem during python cmd ",pythonCmd, " from module ", pythonModule, " run") a = 0.0 fileOut.write("\execPythonScript done in %d sec\n"%(time.time()-tim0)) fileOut.close() - print "execPythonScript: done in %d sec"%(time.time()-tim0) + print("execPythonScript: done in %d sec"%(time.time()-tim0)) return a #------------------------------------------------------------------------------------ diff --git a/prmClasses.py b/prmClasses.py index 398876447ac67b475f9e2a46a385b44ff1b3e59c..79eee79afefc6b08d0e0356480a64e6fa6bb7570 100644 --- a/prmClasses.py +++ b/prmClasses.py @@ -3,6 +3,9 @@ # Classes de gestion des parametres +from __future__ import print_function +from builtins import input +from builtins import object import sys, os, os.path, shutil, socket, platform, glob, fnmatch import datetime, tarfile, subprocess @@ -32,7 +35,7 @@ class PRMSet(object): self.debug = _verb def printPars(self): - for par in self.pars.itervalues(): + for par in self.pars.values(): par.writePRM(sys.stdout) def loadPaths(self): @@ -46,7 +49,7 @@ class PRMSet(object): pth = self.savePath() fname = os.path.join(pth, self.cfgfile) file = open(fname,"w") - for par in self.pars.itervalues(): + for par in self.pars.values(): par.writePRM(file) file.close() @@ -59,14 +62,14 @@ class PRMSet(object): cmds = file.readlines() for cmd in cmds: try: - exec cmd + exec(cmd) except: pass file.close() break def setDefaultPars(self): # RB: inutile - d'autant plus que ca ne declanche aucune exception - print "PureVirtual Class PRMSet" + print("PureVirtual Class PRMSet") def applyDependencies(self): # no dependencies @@ -81,12 +84,12 @@ class PRMSet(object): while True: self.configActions() cls() - print "Actions:" + print("Actions:") for act in self.actions: if act.enabled(): act.disp() - print msg.rjust(78) - print "Your choice?", + print(msg.rjust(78)) + print("Your choice?", end=' ') c = getch() #print c, if c!='': @@ -128,8 +131,8 @@ class TextPRM(PRM): def input(self): cls() - print "%s [def=%s]:" % (self.desc, self.defval) - self.val = self.typecheck(raw_input()) + print("%s [def=%s]:" % (self.desc, self.defval)) + self.val = self.typecheck(input()) def typecheck(self, val): if type(val)!=str: @@ -197,14 +200,14 @@ class PRMAction(Action): ext = "[DISABLED]" else: ext = repr(self.prm.val) - print " %s/ %s : %s" % (self.key, self.prm.desc.ljust(35,' '), ext) + print(" %s/ %s : %s" % (self.key, self.prm.desc.ljust(35,' '), ext)) def enabled(self): return self.prm.enabled class NoAction(Action): def __init__(self, set): Action.__init__(self, set, '') def disp(self): - print + print() class QuitAction(Action): def __init__(self, set, key): @@ -212,7 +215,7 @@ class QuitAction(Action): def execute(self,job): sys.exit() def disp(self): - print " %s/ QUIT" % self.key + print(" %s/ QUIT" % self.key) class GoAction(Action): def __init__(self, set, key): @@ -221,7 +224,7 @@ class GoAction(Action): job.go() sys.exit() def disp(self): - print " %s/ GO" % self.key + print(" %s/ GO" % self.key) class SaveAction(Action): def __init__(self, set, key): @@ -229,7 +232,7 @@ class SaveAction(Action): def execute(self,job): job.savePars() def disp(self): - print " %s/ SAVE" % self.key + print(" %s/ SAVE" % self.key) @@ -294,7 +297,7 @@ def cls(): os.system("clear") def sigbreak(sig, arg): - print "SIG-BREAK!" + print("SIG-BREAK!") sys.exit() def quit(): # RB: ou cette fct est-elle utilisée???? diff --git a/prmClassesGui.py b/prmClassesGui.py index 68b5b1b0ab1277aec54ed35b77ed313eec2224f0..ad2743caa468c16788c0d7a143d733da9226d068 100644 --- a/prmClassesGui.py +++ b/prmClassesGui.py @@ -4,6 +4,8 @@ # # Classe d'interface PyQt des prmClasses +from __future__ import print_function +from builtins import object import os,sys ## Qt ## @@ -32,7 +34,7 @@ from prmClasses import * #======================================================================================== -class PRMLine(): +class PRMLine(object): def __init__(self, _win, _grpLayout, _prm): self.win = _win self.grpLayout = _grpLayout @@ -374,15 +376,15 @@ class BaseMultiDirPathLine(MultiPathLine): def postAction(self): self.checkValidity() # specific changeBaseDir - print "new base dir =",self.param.val + print("new base dir =",self.param.val) try: #self.win.launch.printPars() os.chdir(self.param.val) self.win.launch.loadPars() #self.win.launch.printPars() - except Exception, e: - print "change of base dir failed on error : " - print e + except Exception as e: + print("change of base dir failed on error : ") + print(e) # update enabled/disabled of options self.win.updateWidgetsVisibility() self.win.updateWidgetsValues() @@ -406,7 +408,7 @@ class BaseDirMultiPRM(MultiPRM): return key, desc, vals, defval def printPars(self): - for k,v in self.pars.items(): + for k,v in list(self.pars.items()): print ("pars['%s'].val=%s\n" % (k,repr(v.val)) ) def savePars(self): @@ -426,7 +428,7 @@ class BaseDirMultiPRM(MultiPRM): cmds = file.readlines() for cmd in cmds: try: - exec cmd + exec(cmd) except: pass file.close()