1 """
2 Application-class that implements pyFoamIPythonNotebook.py
3 """
4 from optparse import OptionGroup
5
6 from .PyFoamApplication import PyFoamApplication
7 from PyFoam.IPython.Notebook import Notebook
8 from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
9 from PyFoam.Basics.FoamOptionParser import Subcommand
10
11 from os import path
12 import sys,re
13
14 from PyFoam.ThirdParty.six import print_,u
15
17 - def __init__(self,
18 args=None,
19 **kwargs):
20 description="""\
21 This utility creates and manipulates IPython-Notebooks that are related to
22 OpenFOAM-cases. The Notebooks are only used as a start for the own evaluations of the user
23 """
24 PyFoamApplication.__init__(self,
25 args=args,
26 description=description,
27 usage="%prog COMMAND [<arguments>]",
28 changeVersion=False,
29 subcommands=True,
30 **kwargs)
31
33
34 createCmd=Subcommand(name='create',
35 help="Create a new IPython-notebook for a case",
36 aliases=("new","mk",),
37 nr=1,
38 exactNr=True)
39 self.parser.addSubcommand(createCmd,
40 usage="%prog COMMAND <caseDirectory>")
41
42 copyCmd=Subcommand(name='copy',
43 help="Gets an existing notebook and rewrites it to fit a new case (this assumes that the original notebook was built with this utility)",
44 aliases=("cp",),
45 nr=2,
46 exactNr=True)
47 self.parser.addSubcommand(copyCmd,
48 usage="%prog COMMAND <originalNotebook> <caseDirectory>")
49
50 infoCmd=Subcommand(name='info',
51 help="Check whether an IPython-Notebook is created by this Utility and print info",
52 aliases=("report",),
53 nr=1,
54 exactNr=False)
55 self.parser.addSubcommand(infoCmd,
56 usage="%prog COMMAND <notebookFile> [<more notebook files>]")
57
58 cleanCmd=Subcommand(name='clean',
59 help="Remove unneeded cells from the notebook",
60 aliases=("purge",),
61 nr=1,
62 exactNr=True)
63 self.parser.addSubcommand(cleanCmd,
64 usage="%prog COMMAND <notebookFile>")
65
66
67 for cmd in [copyCmd,createCmd]:
68 outOpts=OptionGroup(cmd.parser,
69 "Write Options",
70 "Where the Notebook should be created")
71 outOpts.add_option("--force-write",
72 action="store_true",
73 dest="forceWrite",
74 default=False,
75 help="Force writing if the file already exists")
76 outOpts.add_option("--destination-file",
77 action="store",
78 dest="destinationFile",
79 default=None,
80 help="Write to this filename. If unset the notebook is written to the case it is created for as <casename>.ipynb. If the destination is directory the file is created in this directory as <casename>.ipynb. Otherwise the fie is created according to specification")
81 outOpts.add_option("--relative-path",
82 action="store_false",
83 dest="absolutePath",
84 default=True,
85 help="Keep the relative path to the directory as specified by the user. Otherwise the path is rewritten as an absolute path")
86 outOpts.add_option("--case-variable-name",
87 action="store",
88 dest="caseVariable",
89 default="case",
90 help="Name of the variable representing the case in the notebook. Defaut: %default")
91 cmd.parser.add_option_group(outOpts)
92
93 for cmd in [cleanCmd]:
94 cleanOpts=OptionGroup(cmd.parser,
95 "Clean Options",
96 "What should be cleaned")
97 cleanOpts.add_option("--keep-selector",
98 action="store_false",
99 dest="cleanSelector",
100 default=True,
101 help="Keep the data selectors")
102 cleanOpts.add_option("--keep-developer",
103 action="store_false",
104 dest="cleanDeveloper",
105 default=True,
106 help="Clean out the developer stuff")
107 cleanOpts.add_option("--clean-comments",
108 action="store_true",
109 dest="cleanComment",
110 default=False,
111 help="Clean out the comments created by this utility")
112 cleanOpts.add_option("--clean-headings",
113 action="store_true",
114 dest="cleanHeading",
115 default=False,
116 help="Clean out the headings created by this utility")
117 cleanOpts.add_option("--clean-report",
118 action="store_true",
119 dest="cleanReport",
120 default=False,
121 help="Clean out the case report created by this utility")
122 cleanOpts.add_option("--clean-info",
123 action="store_true",
124 dest="cleanInfo",
125 default=False,
126 help="Clean out information statements")
127 cleanOpts.add_option("--clean-output",
128 action="store_true",
129 dest="cleanOutput",
130 default=False,
131 help="Strip out the output cells (results)")
132 cleanOpts.add_option("--clean-custom-tag",
133 action="append",
134 dest="customTags",
135 default=[],
136 help="Clean cells tagged with this custom tag. Can be specified more than once")
137 cmd.parser.add_option_group(cleanOpts)
138
139 outOpts=OptionGroup(cmd.parser,
140 "Write Options",
141 "How the cleaned notebook should be written")
142 outOpts.add_option("--overwrite",
143 action="store_true",
144 dest="overwrite",
145 default=False,
146 help="Overwrite the old notebook")
147 outOpts.add_option("--outfile",
148 action="store",
149 dest="outfile",
150 default=None,
151 help="Write to a new notebook here")
152 outOpts.add_option("--force",
153 action="store_true",
154 dest="force",
155 default=False,
156 help="If the outfile already exists overwrite it")
157 cmd.parser.add_option_group(outOpts)
158
159 for cmd in [createCmd]:
160 contentOpts=OptionGroup(cmd.parser,
161 "Content Options",
162 "What should be added to the notebook")
163 contentOpts.add_option("--no-case-report",
164 action="store_false",
165 dest="caseReport",
166 default=True,
167 help="Do not give a general overview of the case")
168 contentOpts.add_option("--no-additional-imports",
169 action="store_false",
170 dest="additional",
171 default=True,
172 help="Do not import packages that make the notebook neater")
173 contentOpts.add_option("--long-boundary-conditions",
174 action="store_true",
175 dest="longBCs",
176 default=False,
177 help="Long boundary conditions")
178 contentOpts.add_option("--no-parallel-report",
179 action="store_false",
180 dest="parallelReport",
181 default=True,
182 help="Do not report about parallelization")
183 contentOpts.add_option("--no-postprocessing",
184 action="store_false",
185 dest="postprocessing",
186 default=True,
187 help="Do not report about available postprocessing data")
188 contentOpts.add_option("--no-data-selectors",
189 action="store_false",
190 dest="selectors",
191 default=True,
192 help="Do not add data selectors for the available postprocessing data")
193 cmd.parser.add_option_group(contentOpts)
194
196 if self.cmdname in ["create","copy"]:
197 if self.cmdname=="create":
198 dest=self.parser.getArgs()[0]
199 else:
200 dest=self.parser.getArgs()[1]
201 sol=SolutionDirectory(dest,
202 paraviewLink=False,
203 archive=None)
204 fName=path.join(sol.name,path.basename(sol.name)+".ipynb")
205 if self.opts.destinationFile:
206 fName=self.opts.destinationFile
207 if path.isdir(fName):
208 fName=path.join(fName,path.basename(sol.name))
209 if path.splitext(fName)[1]!=".ipynb":
210 fName+=".ipynb"
211 if self.opts.absolutePath:
212 usedDest=sol.name
213 else:
214 usedDest=path.relpath(sol.name,
215 start=path.dirname(path.abspath(
216 fName)))
217 if path.exists(fName):
218 if not self.opts.forceWrite:
219 self.error("File",fName,"already existing")
220 else:
221 self.warning("Overwriting",fName)
222 nb=Notebook(name=path.basename(sol.name))
223 nb.pyFoamMetaData()["description"]="Created by "+self.parser.get_prog_name()
224 if self.cmdname=="create":
225 nb.addHeading("Imports and administrative stuff",
226 level=1,classes="heading")
227 if self.opts.developerMode:
228 nb.addMarkdown("This part only needed by developers (reload imports)",
229 classes=("comment","developer"))
230 nb.addCode("%load_ext autoreload",classes="developer")
231 nb.addCode("%autoreload 2",classes="developer")
232 nb.addMarkdown("Make sure that plots are inlined",
233 classes="comment")
234 nb.addCode("%matplotlib inline")
235 if self.opts.additional:
236 nb.addHeading("Additional imports for convenience",
237 level=2,classes=("heading","additional"))
238 nb.addMarkdown("Allow panning and zooming in plots. Slower than regular plotting so for big data you might want to use `mpld3.disable_notebook()` and erase this cell.",
239 classes=("comment","additional"))
240 nb.addCode(
241 """try:
242 import mpld3
243 mpld3.enable_notebook()
244 except ImportError:
245 print 'No mpld3-library. No interactive plots'""",classes="additional")
246 nb.addMarkdown(
247 """Wrapper with additional functionality to the regular Pandas-`DataFrame`:
248
249 * `addData()` for adding columns from other data sets (with resampling
250 * `integrals()` and `weightedAverage()`. Also extended `descripe()` that returns this data
251
252 Most Pandas-operations (like slicing) will return a Pandas-`DataFrame`. By enclosing this in `DataFrame(...)` you can 'add' this functionality to your data. PyFoam operations return this extended `DataFrame` automatically""",
253 classes=("comment","additional"))
254 nb.addCode("from PyFoam.Wrappers.Pandas import PyFoamDataFrame as DataFrame",classes="additional")
255 nb.addHeading("Data storage",
256 level=2,classes=("heading"))
257 nb.addMarkdown("This is the support for permanently storing data into the notebook",
258 classes="comment")
259 nb.addCode("from PyFoam.IPython import storage")
260 nb.addMarkdown("Due to technical problems the next line has to be executed 'by hand' (it will not work poperly if called from `Run All` or similar). When reopening the page the JavaScript-error is normal (it will go away once the cell is executed). Reading can take some time and the next command will appear to 'hang'",
261 classes="comment")
262 nb.addCode("store=storage()")
263 nb.addMarkdown("The next line switches on the behaviour that items specified with `store(name,func)` will be stored permanently in the notebook. Uncomment if you want this behaviour",
264 classes="comment")
265 nb.addCode("# store.autowriteOn()")
266 nb.addMarkdown("The next line switches off the default behaviour that for items specified with `store(name,func)` if `name` is already specified in the permant storage this value is used and `func` is ignored",
267 classes="comment")
268 nb.addCode("# store.autoreadOff()")
269 nb.addHeading("Case data",
270 level=2,classes=("heading"))
271 nb.addMarkdown("This class makes it easy to access case data. Use tab-completion for available methods",
272 classes="comment")
273 nb.addCode("from PyFoam.IPython.Case import Case")
274 nb.addHeading("The Case",classes="heading")
275 v=self.opts.caseVariable
276 nb.addCode("%s=Case('%s')" % (v,usedDest),classes="case",
277 pyFoam={"caseVar":v,"usedDirectory":usedDest,
278 "casePath":sol.name})
279 if self.opts.caseReport:
280 nb.addHeading("Case Report",level=2,
281 classes=("report","heading"))
282 regions=sorted(sol.getRegions(defaultRegion=True))
283 namedRegions=[r for r in regions if r!=None]
284 if len(namedRegions)>0:
285 nb.addMarkdown("Contains named regions *"+
286 ", ".join(namedRegions)+"*",
287 classes=("info","report"))
288 if sol.procNr>0:
289 nb.addMarkdown("Case seems to be decomposed to "+
290 str(sol.procNr)+" processors",
291 classes=("info","report"))
292 for region in regions:
293 if region==None:
294 level=3
295 regionStr=""
296 else:
297 nb.addHeading("Region "+region,
298 level=3,classes=("heading","report"))
299 level=4
300 regionStr="region='%s'," % region
301 nb.addCode("%s.size(%slevel=%d)" % (v,regionStr,level),
302 classes="report")
303 nb.addCode("%s.boundaryConditions(%slevel=%d)" % (v,regionStr,level),
304 classes="report")
305 nb.addCode("%s.dimensions(%slevel=%d)" % (v,regionStr,level),
306 classes="report")
307 nb.addCode("%s.internalField(%slevel=%d)" % (v,regionStr,level),
308 classes="report")
309 if self.opts.longBCs:
310 nb.addCode("%s.longBoundaryConditions(%slevel=%d)" % (regionStr,v,level),
311 classes="report")
312 if sol.procNr>0 and self.opts.parallelReport:
313 nb.addCode("%s.decomposition(%slevel=%d)" % (v,regionStr,level),
314 classes="report")
315 nb.addCode("%s.processorMatrix(%slevel=%d)" % (v,regionStr,level),
316 classes="report")
317 if self.opts.postprocessing:
318 nb.addHeading("Postprocessing data",classes="heading")
319 if len(sol.timelines)>0:
320 nb.addMarkdown("Timelines",classes="info")
321 nb.addCode("%s.sol.timelines" % v,classes="info")
322 if len(sol.samples)>0:
323 nb.addMarkdown("Samples",classes="info")
324 nb.addCode("%s.sol.samples" % v,classes="info")
325 if len(sol.surfaces)>0:
326 nb.addMarkdown("Surfaces",classes="info")
327 nb.addCode("%s.sol.surfaces" % v,classes="info")
328 if len(sol.distributions)>0:
329 nb.addMarkdown("Distributions",classes="info")
330 nb.addCode("%s.sol.distributions" % v,classes="info")
331 if len(sol.pickledData)>0:
332 nb.addMarkdown("Pickled data files",classes="info")
333 nb.addCode("%s.sol.pickledData" % v,classes="info")
334 if len(sol.pickledPlots)>0:
335 nb.addMarkdown("Pickled plot files",classes="info")
336 nb.addCode("%s.sol.pickledPlots" % v,classes="info")
337 if self.opts.selectors:
338 sel=[("timeline",sol.timelines),
339 ("sample",sol.samples),
340 ("distribution",sol.distributions)]
341 for desc,items in sel:
342 if len(items)>0:
343 nb.addHeading(desc.capitalize()+
344 " selectors",level=3,
345 classes=("heading","selector"))
346 for i in items:
347 nb.addCode("%s.%sSelector('%s')" %
348 (v,desc,i),
349 classes="selector")
350 if len(sol.pickledPlots)>0 or len(sol.pickledData)>0:
351 nb.addHeading("Data selectors",level=3,
352 classes=("heading","selector"))
353 if len(sol.pickledPlots)>0:
354 nb.addCode("%s.pickledPlotSelector()" % v,classes="selector")
355 if len(sol.pickledData)>0:
356 nb.addCode("%s.pickledDataSelector()" % v,classes="selector")
357
358 nb.addHeading("User evaluations",classes="heading")
359 nb.addMarkdown("Now add your own stuff",classes="comment")
360 elif self.cmdname=="copy":
361 src=self.parser.getArgs()[0]
362 nb=Notebook(src)
363 cnt=0
364 for c in nb:
365 if c.isClass("case"):
366 cnt+=1
367 if cnt>1:
368 self.error(src,"has more than one 'case'-cell")
369 py=c.meta()[u("pyFoam")]
370 used=py["usedDirectory"]
371 input=[]
372 changed=False
373 for l in c["input"]:
374 if l.find(used)>=0:
375 input.append(l.replace(used,usedDest))
376 changed=True
377 else:
378 input.append(l)
379 if not changed:
380 self.warning(used,"not found")
381 py["usedDirectory"]=usedDest
382 py["casePath"]=sol.name
383 c["input"]=input
384 else:
385 self.error("Unimplemented:",self.cmdname)
386 nb.writeToFile(fName)
387 elif self.cmdname=="info":
388 for n in self.parser.getArgs():
389 print_(n)
390 print_("-"*len(n))
391 nb=Notebook(n)
392 meta=nb.pyFoamMetaData()
393 try:
394 origin=meta["createdBy"]
395 except KeyError:
396 origin="unknown"
397 try:
398 created=meta["createdTime"]
399 except KeyError:
400 created="unknown"
401 try:
402 created=meta["createdTime"]
403 except KeyError:
404 created="unknown"
405 try:
406 modified=meta["modificationTime"]
407 except KeyError:
408 modified="unknown"
409 print_("Created by",origin,"at",created,
410 "modified",modified)
411 classes={}
412 cases={}
413 nrOutput=0
414 for c in nb:
415 if "outputs" in c:
416 if len(c["outputs"])>0:
417 nrOutput+=1
418 try:
419 py=c.meta()[u("pyFoam")]
420 except KeyError:
421 continue
422 try:
423 cl=py["classes"]
424 for c in cl:
425 try:
426 classes[c]+=1
427 except KeyError:
428 classes[c]=1
429 except KeyError:
430 pass
431 if "caseVar" in py:
432 try:
433 cases[py["caseVar"]]=py["casePath"]
434 except KeyError:
435 pass
436 print_(len(nb),"cells. Classes:",
437 ", ".join([k+":"+str(classes[k]) for k in sorted(classes.keys())]))
438 print_("Cells with output:",nrOutput)
439 print("Case-Variables:")
440 for k in sorted(cases.keys()):
441 print_(" ",k,":",cases[k])
442
443 print_()
444 elif self.cmdname=="clean":
445 nb=Notebook(self.parser.getArgs()[0])
446 if not self.opts.overwrite and not self.opts.outfile:
447 self.error("Either specify --overwrite or --outfile")
448 if self.opts.overwrite and self.opts.outfile:
449 self.error("Only specify --overwrite or --outfile")
450 if self.opts.outfile:
451 if path.exists(self.opts.outfile):
452 if not self.opts.force:
453 self.error("File",self.opts.outfile,"exists")
454 else:
455 self.warning("Overwriting",self.opts.outfile)
456 else:
457 if path.splitext(self.opts.outfile)[1]!=".ipynb":
458 self.warning("Appending '.ipynb' to",self.opts.outfile)
459 self.opts.outfile+=".ipynb"
460 if self.opts.overwrite:
461 toFile=self.parser.getArgs()[0]
462 else:
463 toFile=self.opts.outfile
464
465 removeClasses=self.opts.customTags[:]
466 if self.opts.cleanSelector:
467 removeClasses.append("selector")
468 if self.opts.cleanDeveloper:
469 removeClasses.append("developer")
470 if self.opts.cleanHeading:
471 removeClasses.append("heading")
472 if self.opts.cleanComment:
473 removeClasses.append("comment")
474 if self.opts.cleanReport:
475 removeClasses.append("report")
476 if self.opts.cleanInfo:
477 removeClasses.append("info")
478
479 print_("Cleaning cells tagged with: "+" ".join(sorted(removeClasses)))
480
481 nb.reset([c for c in nb if not c.isClass(removeClasses)])
482 if self.opts.cleanOutput:
483 print_("Removing output")
484 for c in nb:
485 if "outputs" in c:
486 c["outputs"]=[]
487
488 nb.writeToFile(toFile)
489 else:
490 self.error("Unimplemented command",self.cmdname)
491
492
493