1
2 """
3 Collects data about runs in a small SQLite database
4 """
5
6
7
8 import sqlite3
9 from os import path
10 import datetime
11 import re
12 import sys
13
14 from PyFoam.Error import error
15 from .CSVCollection import CSVCollection
16
17 from PyFoam.ThirdParty.six import print_,iteritems,integer_types
18 from PyFoam.ThirdParty.six import u as uniCode
19
21 """
22 Database with information about runs. To be queried etc
23 """
24
25 separator="//"
26
27 - def __init__(self,
28 name,
29 create=False,
30 verbose=False):
31 """@param name: name of the file
32 @param create: should the database be created if it does not exist"""
33
34 self.verbose=verbose
35 if not path.exists(name):
36 if create==False:
37 error("Database",name,"does not exist")
38 else:
39 self.initDatabase(name)
40
41 self.db=sqlite3.connect(name)
42 self.db.row_factory=sqlite3.Row
43
45 """Create a new database file"""
46 db=sqlite3.connect(name)
47 with db:
48 db.row_factory=sqlite3.Row
49 cursor=db.cursor()
50 cursor.execute("CREATE TABLE theRuns(runId INTEGER PRIMARY KEY, "+
51 "insertionTime TIMESTAMP)")
52
54 """Add a dictionary with data to the database"""
55 self.__adaptDatabase(data)
56
57 runData=dict([("insertionTime",datetime.datetime.now())]+ \
58 [(k,v) for k,v in iteritems(data) if type(v)!=dict])
59 runID=self.__addContent("theRuns",runData)
60
61 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict])
62 for tn,content in iteritems(subtables):
63 self.__addContent(tn+"Data",
64 dict(list(self.__flattenDict(content).items())+
65 [("runId",runID)]))
66
67 self.db.commit()
68
69 - def __addContent(self,table,data):
70 cursor=self.db.cursor()
71 runData={}
72 for k,v in iteritems(data):
73 if k=="runId":
74 runData[k]=v
75 elif isinstance(v,integer_types+(float,)):
76 runData[k]=float(v)
77 else:
78 runData[k]=uniCode(str(v))
79 cols=self.__getColumns(table)[1:]
80 addData=[]
81 for c in cols:
82 try:
83 addData.append(runData[c])
84 except KeyError:
85 addData.append(None)
86 addData=tuple(addData)
87 cSQL = "insert into "+table+" ("+ \
88 ",".join(['"'+c+'"' for c in cols])+ \
89 ") values ("+",".join(["?"]*len(addData))+")"
90 if self.verbose:
91 print_("Execute SQL",cSQL,"with",addData)
92 try:
93 cursor.execute(cSQL, addData)
94 except Exception:
95 e = sys.exc_info()[1]
96 print_("SQL-Expression:",cSQL)
97 print_("AddData:",addData)
98 raise e
99
100 return cursor.lastrowid
101
103 """Make sure that all the required columns and tables are there"""
104
105 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"')
106 tables=[ x["name"] for x in c.fetchall() ]
107
108 indata=dict([(k,v) for k,v in iteritems(data) if type(v)!=dict])
109 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict])
110
111 self.__addColumnsToTable("theRuns",indata)
112
113 for tn,content in iteritems(subtables):
114 if tn+"Data" not in tables:
115 if self.verbose:
116 print_("Adding table",tn)
117 self.db.execute("CREATE TABLE "+tn+"Data (dataId INTEGER PRIMARY KEY, runId INTEGER)")
118 self.__addColumnsToTable(tn+"Data",
119 self.__flattenDict(content))
120
122 data=[(prefix+k,v) for k,v in iteritems(oData) if type(v)!=dict]
123 subtables=dict([(k,v) for k,v in iteritems(oData) if type(v)==dict])
124 for name,val in iteritems(subtables):
125 data+=list(self.__flattenDict(val,prefix+name+self.separator).items())
126 if self.verbose:
127 print_("Flattened",oData,"to",data)
128 return dict(data)
129
131 c=self.db.execute('SELECT * from '+tablename)
132 return [desc[0] for desc in c.description]
133
147
148 - def dumpToCSV(self,
149 fname,
150 selection=None,
151 disableRunData=None,
152 pandasFormat=True,
153 excel=False):
154 """Dump the contents of the database to a csv-file
155 @param name: the CSV-file
156 @param selection: list of regular expressions. Only data
157 entries fitting those will be added to the CSV-file (except
158 for the basic run). If unset all data will be written"""
159 file=CSVCollection(fname)
160
161 runCursor=self.db.cursor()
162 runCursor.execute("SELECT * from theRuns")
163
164 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"')
165 tables=[ x["name"] for x in c.fetchall() ]
166
167 allData=set()
168 writtenData=set()
169
170 disabledStandard=set()
171
172 for d in runCursor:
173 id=d['runId']
174 if self.verbose:
175 print_("Dumping run",id)
176 for k in list(d.keys()):
177 writeEntry=True
178 if disableRunData:
179 for e in disableRunData:
180 exp=re.compile(e)
181 if exp.search(k):
182 writeEntry=False
183 break
184 if writeEntry:
185 file[k]=d[k]
186 else:
187 disabledStandard.add(k)
188 for t in tables:
189 if t=="theRuns":
190 continue
191 namePrefix=t[:-4]
192 dataCursor=self.db.cursor()
193 dataCursor.execute("SELECT * FROM "+t+" WHERE runId=?",
194 (str(id),))
195 data=dataCursor.fetchall()
196 if len(data)>1:
197 error(len(data),"data items found for id ",id,
198 "in table",t,".Need exactly 1")
199 elif len(data)<1:
200 continue
201 for k in list(data[0].keys()):
202 if k in ["dataId","runId"]:
203 continue
204 name=namePrefix+self.separator+k
205 allData.add(name)
206 writeEntry=True
207 if selection:
208 writeEntry=False
209 for e in selection:
210 exp=re.compile(e)
211 if exp.search(name):
212 writeEntry=True
213 break
214 if writeEntry:
215 writtenData.add(name)
216 file[name]=data[0][k]
217
218 file.write()
219
220 if self.verbose:
221 sep="\n "
222 if allData==writtenData:
223 print_("Added all data entries:",sep,sep.join(allData),sep="")
224 else:
225 print_("Added parameters:",sep,sep.join(writtenData),
226 "\nUnwritten data:",sep,sep.join(allData-writtenData),sep="")
227 if len(disabledStandard)>0:
228 print_("Disabled standard entries:",sep,sep.join(disabledStandard),sep="")
229
230 f=file(pandasFormat)
231 if excel:
232 file(True).to_excel(fname)
233
234 if f:
235 return f
236 else:
237
238 return file(False)
239
240
241