Package PyFoam :: Package Applications :: Module CompressCaseFiles
[hide private]
[frames] | no frames]

Source Code for Module PyFoam.Applications.CompressCaseFiles

  1  """ 
  2  Application-class that implements pyFoamCompressCaseFiles.py 
  3  """ 
  4  from optparse import OptionGroup 
  5  from os import path,listdir,stat 
  6  import time,datetime 
  7  from stat import ST_MTIME 
  8  import string 
  9  import subprocess 
 10  import re,sys 
 11  from glob import glob 
 12   
 13  from .PyFoamApplication import PyFoamApplication 
 14   
 15  from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory 
 16   
 17  from PyFoam import configuration 
 18   
 19  from PyFoam.ThirdParty.six import print_,iteritems,PY3 
 20   
 21  from PyFoam.Basics.Utilities import humanReadableSize 
 22   
 23  if PY3: 
 24      long=int 
 25   
26 -class CompressCaseFiles(PyFoamApplication):
27 - def __init__(self,args=None):
28 description="""\ 29 Gets a number of directories. If these are OpenFOAM-cases then it goes through them 30 and checks for large uncompressed files and gnuzips them 31 """ 32 PyFoamApplication.__init__(self, 33 args=args, 34 description=description, 35 usage="%prog [<directories>]", 36 interspersed=True, 37 changeVersion=False, 38 nr=1, 39 exactNr=False)
40
41 - def addOptions(self):
42 compress=OptionGroup(self.parser, 43 "Compression", 44 "Define what will be compressed") 45 self.parser.add_option_group(compress) 46 47 compress.add_option("--recursive", 48 action="store_true", 49 dest="recursive", 50 default=False, 51 help="Use the specified directories as a starting point and recursively search for cases") 52 53 compress.add_option("--big-size", 54 action="store", 55 type="int", 56 dest="bigSize", 57 default=4095, 58 help="Files with less bytes than this will not be compressed. Default: %default") 59 60 compress.add_option("--compressed-extensions", 61 action="append", 62 dest="extensions", 63 default=["gz","zip","tgz"], 64 help="File extensions for which we assume that they are already compressed. Default: %default") 65 66 compress.add_option("--logfiles", 67 action="store_true", 68 dest="logfile", 69 default=False, 70 help="Compress files in the case directory that end with .logfile (Assuming that these are logfiles generated by PyFoam)") 71 72 feedback=OptionGroup(self.parser, 73 "Feedback", 74 "What should be printed") 75 self.parser.add_option_group(feedback) 76 feedback.add_option("--verbose", 77 action="count", 78 dest="verbose", 79 default=1, 80 help="Print names of the directories processed. Use multiple times for higher verbosity") 81 feedback.add_option("--no-statistics", 82 action="store_false", 83 dest="statistics", 84 default=True, 85 help="Do not print a summary in the end") 86 feedback.add_option("--silent", 87 action="store_true", 88 dest="silent", 89 default=False, 90 help="Switch off all output except warnings")
91 92
93 - def compressFile(self,fName):
94 if self.verbose>1: 95 print_(" Compressing",fName) 96 zippedName=fName+".gz" 97 if path.exists(zippedName): 98 self.warning("Zipped file",zippedName,"already existing for",fName) 99 return 100 oldSize=path.getsize(fName) 101 if oldSize<self.bigSize: 102 if self.verbose>2: 103 print_(" Skipping because it is too small") 104 self.nrSkipped+=1 105 return 106 107 # use gzip because that way the responsibility of removing the old file is with a 'tried and testd' program 108 ret=subprocess.call(["gzip",fName]) 109 if ret!=0 or not path.exists(zippedName) or path.exists(fName): 110 self.warning("Problem compressing file",fName) 111 self.nrProblems+=1 112 return 113 newSize=path.getsize(zippedName) 114 if newSize>oldSize: 115 self.warning("Compression of",fName,"increased the filesize. Old:", 116 humanReadableSize(oldSize),"New:",humanReadableSize(newSize)) 117 118 if self.verbose>2: 119 print_(" Old size:",humanReadableSize(oldSize),"New size:",humanReadableSize(newSize)) 120 121 self.nrFiles+=1 122 self.prevSize+=oldSize 123 self.nowSize+=newSize
124
125 - def compressDirectory(self,dirName):
126 if self.verbose>1: 127 print_(" Checking",dirName,"for compressible files") 128 for f in listdir(dirName): 129 if path.isdir(path.join(dirName,f)): 130 self.compressDirectory(path.join(dirName,f)) 131 else: 132 name,ext=path.splitext(f) 133 if ext.lower() not in self.extensions: 134 self.compressFile(path.join(dirName,f)) 135 else: 136 self.nrCompressed+=1
137
138 - def compressCase(self,dirName,warn=False):
139 if not path.exists(dirName): 140 self.error("Directory",dirName,"does not exist") 141 s=SolutionDirectory(dirName, 142 archive=None, 143 paraviewLink=False, 144 parallel=True, 145 tolerant=True) 146 if not s.isValid(): 147 if warn: 148 print_("Directory",dirName,"is not an OpenFOAM-case") 149 return 150 151 self.nrDir+=1 152 oldNr=self.nrFiles 153 oldUnc=self.prevSize 154 oldCon=self.nowSize 155 156 if self.verbose>0: 157 print_("Processing case",dirName) 158 159 # compress meshes 160 for d in glob(path.join(dirName,"*","polyMesh"))+glob(path.join(dirName,"*","*","polyMesh")): 161 if path.isdir(d): 162 self.compressDirectory(d) 163 164 # compress times 165 for t in s: 166 self.compressDirectory(t.name) 167 168 # compress logfiles if requested 169 if self.opts.logfile: 170 for f in glob(path.join(dirName,"*.logfile")): 171 self.compressFile(path.join(dirName,f)) 172 173 # processor direcories 174 for p in s.procDirs: 175 self.compressDirectory(path.join(dirName,p)) 176 if self.nrFiles>oldNr and self.verbose>0: 177 print_(" -> ",self.nrFiles-oldNr,"files compressed.", 178 humanReadableSize((self.prevSize-oldUnc)-(self.nowSize-oldCon)),"gained")
179
180 - def recursiveCompress(self,dirName):
181 if self.verbose>1: 182 print_("Recursively checking",dirName) 183 if path.isdir(dirName): 184 s=SolutionDirectory(dirName,archive=None,paraviewLink=False,parallel=True) 185 if s.isValid(): 186 try: 187 self.compressCase(dirName) 188 except OSError: 189 e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' 190 self.warning("Problem processing",dirName,":",e) 191 return 192 193 for f in listdir(dirName): 194 name=path.join(dirName,f) 195 try: 196 if path.isdir(name): 197 self.recursiveCompress(name) 198 except OSError: 199 e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' 200 self.warning("Problem processing",name,":",e)
201
202 - def run(self):
203 dirs=self.parser.getArgs() 204 205 self.bigSize=self.opts.bigSize 206 self.verbose=self.opts.verbose 207 self.extensions=["."+e.lower() for e in self.opts.extensions] 208 209 if self.opts.silent: 210 self.verbose=0 211 self.opts.statistics=False 212 213 # for the statistics 214 self.nrDir=0 215 self.nrCompressed=0 216 self.nrFiles=0 217 self.prevSize=0 218 self.nowSize=0 219 self.nrSkipped=0 220 self.nrProblems=0 221 222 try: 223 for d in dirs: 224 p=path.abspath(d) 225 if self.opts.recursive: 226 self.recursiveCompress(p) 227 else: 228 self.compressCase(p,warn=True) 229 except KeyboardInterrupt: 230 print_("Rudely interrupted by Control-C") 231 232 if self.opts.statistics: 233 if self.verbose>0: 234 print_() 235 print_(self.nrDir,"case directories processed") 236 if self.nrFiles==0: 237 print_("No files to compress found") 238 else: 239 print_(self.nrFiles,"files processed") 240 print_("Reduced total size from",humanReadableSize(self.prevSize), 241 "to",humanReadableSize(self.nowSize), 242 ". This is",(100.*self.nowSize)/self.prevSize,"percent of the original") 243 if self.nrSkipped>0: 244 print_("Skipped",self.nrSkipped,"files because they were smaller than",self.bigSize) 245 if self.nrCompressed>0: 246 print_("Skipped",self.nrCompressed,"files because they are already compressed") 247 if self.nrProblems>0: 248 print_("Problems during the compression of",self.nrProblems,"files")
249 250 # Should work with Python3 and Python2 251