Package PyFoam :: Package Execution :: Module BasicRunner
[hide private]
[frames] | no frames]

Source Code for Module PyFoam.Execution.BasicRunner

  1  #  ICE Revision: $Id$ 
  2  """Run a OpenFOAM command""" 
  3   
  4  import sys 
  5  import string 
  6  import gzip 
  7  from os import path 
  8  from platform import uname 
  9  from threading import Timer 
 10  from time import time,asctime 
 11   
 12  from PyFoam.FoamInformation import oldAppConvention as oldApp 
 13  from PyFoam.ThirdParty.six import print_ 
 14  from PyFoam.Basics.DataStructures import makePrimitiveString 
 15   
 16  if not 'curdir' in dir(path) or not 'sep' in dir(path): 
 17      print_("Warning: Inserting symbols into os.path (Python-Version<2.3)") 
 18      path.curdir='.' 
 19      path.sep   ='/' 
 20   
 21  from PyFoam.Execution.FoamThread import FoamThread 
 22  from PyFoam.Infrastructure.FoamServer import FoamServer 
 23  from PyFoam.Infrastructure.Logging import foamLogger 
 24  from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory 
 25  from PyFoam.RunDictionary.ParameterFile import ParameterFile 
 26  from PyFoam.Error import warning,error,debug 
 27  from PyFoam import configuration as config 
 28   
29 -def restoreControlDict(ctrl,runner):
30 """Timed function to avoid time-stamp-problems""" 31 warning("Restoring the controlDict") 32 ctrl.restore() 33 runner.controlDict=None
34
35 -class BasicRunner(object):
36 """Base class for the running of commands 37 38 When the command is run the output is copied to a LogFile and 39 (optionally) standard-out 40 41 The argument list assumes for the first three elements the 42 OpenFOAM-convention: 43 44 <cmd> <dir> <case> 45 46 The directory name for outputs is therefor created from <dir> and 47 <case> 48 49 Provides some handle-methods that are to be overloaded for 50 additional functionality""" 51
52 - def __init__(self, 53 argv=None, 54 silent=False, 55 logname=None, 56 compressLog=False, 57 lam=None, 58 server=False, 59 restart=False, 60 noLog=False, 61 logTail=None, 62 remark=None, 63 jobId=None, 64 parameters=None, 65 writeState=True, 66 echoCommandLine=None):
67 """@param argv: list with the tokens that are the command line 68 if not set the standard command line is used 69 @param silent: if True no output is sent to stdout 70 @param logname: name of the logfile 71 @param compressLog: Compress the logfile into a gzip 72 @param lam: Information about a parallel run 73 @param server: Whether or not to start the network-server 74 @type lam: PyFoam.Execution.ParallelExecution.LAMMachine 75 @param noLog: Don't output a log file 76 @param logTail: only the last lines of the log should be written 77 @param remark: User defined remark about the job 78 @param parameters: User defined dictionary with parameters for 79 documentation purposes 80 @param jobId: Job ID of the controlling system (Queueing system) 81 @param writeState: Write the state to some files in the case 82 @param echoCommandLine: Prefix that is printed with the command line. If unset nothing is printed 83 """ 84 85 if sys.version_info < (2,3): 86 # Python 2.2 does not have the capabilities for the Server-Thread 87 if server: 88 warning("Can not start server-process because Python-Version is too old") 89 server=False 90 91 if argv==None: 92 self.argv=sys.argv[1:] 93 else: 94 self.argv=argv 95 96 if oldApp(): 97 self.dir=path.join(self.argv[1],self.argv[2]) 98 if self.argv[2][-1]==path.sep: 99 self.argv[2]=self.argv[2][:-1] 100 else: 101 self.dir=path.curdir 102 if "-case" in self.argv: 103 self.dir=self.argv[self.argv.index("-case")+1] 104 105 if logname==None: 106 logname="PyFoam."+path.basename(argv[0]) 107 108 try: 109 sol=self.getSolutionDirectory() 110 except OSError: 111 e = sys.exc_info()[1] # compatible with 2.x and 3.x 112 error("Solution directory",self.dir,"does not exist. No use running. Problem:",e) 113 114 self.echoCommandLine=echoCommandLine 115 self.silent=silent 116 self.lam=lam 117 self.origArgv=self.argv 118 self.writeState=writeState 119 self.__lastLastSeenWrite=0 120 self.__lastNowTimeWrite=0 121 122 if self.lam!=None: 123 self.argv=lam.buildMPIrun(self.argv) 124 if config().getdebug("ParallelExecution"): 125 debug("Command line:"," ".join(self.argv)) 126 self.cmd=" ".join(self.argv) 127 foamLogger().info("Starting: "+self.cmd+" in "+path.abspath(path.curdir)) 128 self.logFile=path.join(self.dir,logname+".logfile") 129 130 self.noLog=noLog 131 self.logTail=logTail 132 if self.logTail: 133 if self.noLog: 134 warning("Log tail",self.logTail,"and no-log specified. Using logTail") 135 self.noLog=True 136 self.lastLines=[] 137 138 self.compressLog=compressLog 139 if self.compressLog: 140 self.logFile+=".gz" 141 142 self.fatalError=False 143 self.fatalFPE=False 144 self.fatalStackdump=False 145 146 self.warnings=0 147 self.started=False 148 149 self.isRestarted=False 150 if restart: 151 self.controlDict=ParameterFile(path.join(self.dir,"system","controlDict"),backup=True) 152 self.controlDict.replaceParameter("startFrom","latestTime") 153 self.isRestarted=True 154 else: 155 self.controlDict=None 156 157 self.run=FoamThread(self.cmd,self) 158 159 self.server=None 160 if server: 161 self.server=FoamServer(run=self.run,master=self) 162 self.server.setDaemon(True) 163 self.server.start() 164 try: 165 IP,PID,Port=self.server.info() 166 f=open(path.join(self.dir,"PyFoamServer.info"),"w") 167 print_(IP,PID,Port,file=f) 168 f.close() 169 except AttributeError: 170 warning("There seems to be a problem with starting the server:",self.server,"with attributes",dir(self.server)) 171 self.server=None 172 173 self.createTime=None 174 self.nowTime=None 175 self.startTimestamp=time() 176 177 self.stopMe=False 178 self.writeRequested=False 179 180 self.endTriggers=[] 181 182 self.lastLogLineSeen=None 183 self.lastTimeStepSeen=None 184 185 self.remark=remark 186 self.jobId=jobId 187 188 self.data={"lines":0} # self.data={"lines":0L} 189 self.data["logfile"]=self.logFile 190 self.data["casefullname"]=path.abspath(self.dir) 191 self.data["casename"]=path.basename(path.abspath(self.dir)) 192 self.data["solver"]=path.basename(self.argv[0]) 193 self.data["solverFull"]=self.argv[0] 194 self.data["commandLine"]=self.cmd 195 self.data["hostname"]=uname()[1] 196 if remark: 197 self.data["remark"]=remark 198 else: 199 self.data["remark"]="No remark given" 200 if jobId: 201 self.data["jobId"]=jobId 202 parameterFile=sol.getParametersFromFile() 203 if len(parameterFile): 204 self.data["parameters"]={} 205 for k,v in parameterFile.items(): 206 self.data["parameters"][k]=makePrimitiveString(v) 207 if parameters: 208 if "parameters" not in self.data: 209 self.data["parameters"]={} 210 self.data["parameters"].update(parameters) 211 self.data["starttime"]=asctime()
212
213 - def appendTailLine(self,line):
214 """Append lines to the tail of the log""" 215 if len(self.lastLines)>10*self.logTail: 216 # truncate the lines, but not too often 217 self.lastLines=self.lastLines[-self.logTail:] 218 self.writeTailLog() 219 220 self.lastLines.append(line+"\n")
221
222 - def writeTailLog(self):
223 """Write the last lines to the log""" 224 fh=open(self.logFile,"w") 225 if len(self.lastLines)<=self.logTail: 226 fh.writelines(self.lastLines) 227 else: 228 fh.writelines(self.lastLines[-self.logTail:]) 229 fh.close()
230
231 - def start(self):
232 """starts the command and stays with it till the end""" 233 234 self.started=True 235 if not self.noLog: 236 if self.compressLog: 237 fh=gzip.open(self.logFile,"w") 238 else: 239 fh=open(self.logFile,"w") 240 241 self.startHandle() 242 243 self.writeStartTime() 244 self.writeTheState("Running") 245 246 check=BasicRunnerCheck() 247 248 if self.echoCommandLine: 249 print_(self.echoCommandLine+" "+" ".join(self.argv)) 250 251 self.run.start() 252 interrupted=False 253 254 totalWarningLines=0 255 addLinesToWarning=0 256 collectWarnings=True 257 258 while self.run.check(): 259 try: 260 self.run.read() 261 if not self.run.check(): 262 break 263 264 line=self.run.getLine() 265 266 if "errorText" in self.data: 267 self.data["errorText"]+=line+"\n" 268 269 if addLinesToWarning>0: 270 self.data["warningText"]+=line+"\n" 271 addLinesToWarning-=1 272 totalWarningLines+=1 273 if totalWarningLines>500: 274 collectWarnings=False 275 addLinesToWarning=0 276 self.data["warningText"]+="No more warnings added because limit of 500 lines exceeded" 277 self.data["lines"]+=1 278 self.lastLogLineSeen=time() 279 self.writeLastSeen() 280 281 tmp=check.getTime(line) 282 if check.controlDictRead(line): 283 if self.writeRequested: 284 duration=config().getfloat("Execution","controlDictRestoreWait",default=30.) 285 warning("Preparing to reset controlDict to old glory in",duration,"seconds") 286 Timer(duration, 287 restoreControlDict, 288 args=[self.controlDict,self]).start() 289 self.writeRequested=False 290 291 if tmp!=None: 292 self.data["time"]=tmp 293 self.nowTime=tmp 294 self.writeTheState("Running",always=False) 295 self.writeNowTime() 296 self.lastTimeStepSeen=time() 297 if self.createTime==None: 298 # necessary because interFoam reports no creation time 299 self.createTime=tmp 300 try: 301 self.data["stepNr"]+=1 302 except KeyError: 303 self.data["stepNr"]=1 # =1L 304 305 self.data["lasttimesteptime"]=asctime() 306 307 tmp=check.getCreateTime(line) 308 if tmp!=None: 309 self.createTime=tmp 310 311 if not self.silent: 312 try: 313 print_(line) 314 except IOError: 315 e = sys.exc_info()[1] # compatible with 2.x and 3.x 316 if e.errno!=32: 317 raise e 318 else: 319 # Pipe was broken 320 self.run.interrupt() 321 322 if line.find("FOAM FATAL ERROR")>=0 or line.find("FOAM FATAL IO ERROR")>=0: 323 self.fatalError=True 324 self.data["errorText"]="PyFoam found a Fatal Error " 325 if "time" in self.data: 326 self.data["errorText"]+="at time "+str(self.data["time"])+"\n" 327 else: 328 self.data["errorText"]+="before time started\n" 329 self.data["errorText"]+="\n"+line+"\n" 330 331 if line.find("Foam::sigFpe::sigFpeHandler")>=0: 332 self.fatalFPE=True 333 if line.find("Foam::error::printStack")>=0: 334 self.fatalStackdump=True 335 336 if self.fatalError and line!="": 337 foamLogger().error(line) 338 339 if line.find("FOAM Warning")>=0: 340 self.warnings+=1 341 try: 342 self.data["warnings"]+=1 343 except KeyError: 344 self.data["warnings"]=1 345 if collectWarnings: 346 addLinesToWarning=20 347 if not "warningText" in self.data: 348 self.data["warningText"]="" 349 else: 350 self.data["warningText"]+=("-"*40)+"\n" 351 self.data["warningText"]+="Warning found by PyFoam on line " 352 self.data["warningText"]+=str(self.data["lines"])+" " 353 if "time" in self.data: 354 self.data["warningText"]+="at time "+str(self.data["time"])+"\n" 355 else: 356 self.data["warningText"]+="before time started\n" 357 self.data["warningText"]+="\n"+line+"\n" 358 359 if self.server!=None: 360 self.server._insertLine(line) 361 362 self.lineHandle(line) 363 364 if not self.noLog: 365 fh.write(line+"\n") 366 fh.flush() 367 elif self.logTail: 368 self.appendTailLine(line) 369 370 except KeyboardInterrupt: 371 e = sys.exc_info()[1] # compatible with 2.x and 3.x 372 foamLogger().warning("Keyboard Interrupt") 373 self.run.interrupt() 374 self.writeTheState("Interrupted") 375 interrupted=True 376 377 self.data["interrupted"]=interrupted 378 self.data["OK"]=self.runOK() 379 self.data["cpuTime"]=self.run.cpuTime() 380 self.data["cpuUserTime"]=self.run.cpuUserTime() 381 self.data["cpuSystemTime"]=self.run.cpuSystemTime() 382 self.data["wallTime"]=self.run.wallTime() 383 self.data["usedMemory"]=self.run.usedMemory() 384 self.data["endtime"]=asctime() 385 386 self.data["fatalError"]=self.fatalError 387 self.data["fatalFPE"]=self.fatalFPE 388 self.data["fatalStackdump"]=self.fatalStackdump 389 390 self.writeNowTime(force=True) 391 392 self.stopHandle() 393 394 if not interrupted: 395 self.writeTheState("Finished") 396 397 for t in self.endTriggers: 398 t() 399 400 if not self.noLog: 401 fh.close() 402 elif self.logTail: 403 self.writeTailLog() 404 405 if self.server!=None: 406 self.server.deregister() 407 self.server.kill() 408 409 foamLogger().info("Finished") 410 411 return self.data
412
413 - def writeToStateFile(self,fName,message):
414 """Write a message to a state file""" 415 if self.writeState: 416 open(path.join(self.dir,"PyFoamState."+fName),"w").write(message+"\n")
417
418 - def writeStartTime(self):
419 """Write the real time the run was started at""" 420 self.writeToStateFile("StartedAt",asctime())
421
422 - def writeTheState(self,state,always=True):
423 """Write the current state the run is in""" 424 if always or (time()-self.__lastLastSeenWrite)>9: 425 self.writeToStateFile("TheState",state)
426
427 - def writeLastSeen(self):
428 if (time()-self.__lastLastSeenWrite)>10: 429 self.writeToStateFile("LastOutputSeen",asctime()) 430 self.__lastLastSeenWrite=time()
431
432 - def writeNowTime(self,force=False):
433 if (time()-self.__lastNowTimeWrite)>10 or force: 434 self.writeToStateFile("CurrentTime",str(self.nowTime)) 435 self.__lastNowTimeWrite=time()
436
437 - def runOK(self):
438 """checks whether the run was successful""" 439 if self.started: 440 return not self.fatalError and not self.fatalFPE and not self.fatalStackdump # and self.run.getReturnCode()==0 441 else: 442 return False
443
444 - def startHandle(self):
445 """to be called before the program is started""" 446 pass
447
448 - def stopGracefully(self):
449 """Tells the runner to stop at the next convenient time""" 450 if not self.stopMe: 451 self.stopMe=True 452 if not self.isRestarted: 453 if self.controlDict: 454 warning("The controlDict has already been modified. Restoring will be problementic") 455 self.controlDict=ParameterFile(path.join(self.dir,"system","controlDict"),backup=True) 456 self.controlDict.replaceParameter("stopAt","writeNow") 457 warning("Stopping run at next write")
458
459 - def writeResults(self):
460 """Writes the next possible time-step""" 461 # warning("writeResult is not yet implemented") 462 if not self.writeRequested: 463 if not self.isRestarted: 464 if self.controlDict: 465 warning("The controlDict has already been modified. Restoring will be problementic") 466 self.controlDict=ParameterFile(path.join(self.dir,"system","controlDict"),backup=True) 467 self.controlDict.replaceParameter("writeControl","timeStep") 468 self.controlDict.replaceParameter("writeInterval","1") 469 self.writeRequested=True
470
471 - def stopHandle(self):
472 """called after the program has stopped""" 473 if self.stopMe or self.isRestarted: 474 self.controlDict.restore()
475
476 - def lineHandle(self,line):
477 """called every time a new line is read""" 478 pass
479
480 - def logName(self):
481 """Get the name of the logfiles""" 482 return self.logFile
483
484 - def getSolutionDirectory(self,archive=None):
485 """@return: The directory of the case 486 @rtype: PyFoam.RunDictionary.SolutionDirectory 487 @param archive: Name of the directory for archiving results""" 488 489 return SolutionDirectory(self.dir,archive=archive,parallel=True)
490
491 - def addEndTrigger(self,f):
492 """@param f: A function that is to be executed at the end of the simulation""" 493 self.endTriggers.append(f)
494 495 import re 496
497 -class BasicRunnerCheck(object):
498 """A small class that does primitve checking for BasicRunner 499 Duplicates other efforts, but ....""" 500 501 floatRegExp="[-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?" 502
503 - def __init__(self):
504 # self.timeExpr=re.compile("^Time = (%f%)$".replace("%f%",self.floatRegExp)) 505 self.timeExpr=config().getRegexp("SolverOutput","timeregexp") 506 self.createExpr=re.compile("^Create mesh for time = (%f%)$".replace("%f%",self.floatRegExp))
507
508 - def getTime(self,line):
509 """Does this line contain time information?""" 510 m=self.timeExpr.match(line) 511 if m: 512 try: 513 return float(m.group(2)) 514 except ValueError: 515 warning("Problem while converting",m.group(2),"to float") 516 return None 517 else: 518 return None
519
520 - def getCreateTime(self,line):
521 """Does this line contain mesh time information?""" 522 m=self.createExpr.match(line) 523 if m: 524 return float(m.group(1)) 525 else: 526 return None
527
528 - def controlDictRead(self,line):
529 """Was the controlDict reread?""" 530 phrases=["Reading object controlDict from file", 531 "Re-reading object controlDict from file"] 532 533 for p in phrases: 534 if line.find(p)>=0: 535 return True 536 537 return False
538 539 # Should work with Python3 and Python2 540