Mercurial > hg > drupalISMI
annotate importFromOpenMind/importer/check_ismi_log.py @ 60:1b520696760a default tip
new ismixml_splitter.py that splits openmind-data.xml into separate files per entity type.
| author | casties |
|---|---|
| date | Tue, 19 Jun 2018 21:46:49 +0200 |
| parents | 277ea02906f9 |
| children |
| rev | line source |
|---|---|
| 38 | 1 |
| 2 # coding: utf-8 | |
| 3 | |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
4 import sys, re |
| 38 | 5 |
| 39 | 6 # max number of lines to read (for testing) |
| 38 | 7 maxLinecnt = None |
| 39 | 8 |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
9 # do not output deleted nodes |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
10 omitDeleted = True |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
11 |
| 39 | 12 # active log levels for logging |
| 41 | 13 logLevels = {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'SYSMSG'} |
| 38 | 14 #logLevels = {'INFO', 'WARNING', 'ERROR', 'SYSMSG'} |
| 41 | 15 #logLevels = {'ERROR', 'SYSMSG'} |
| 38 | 16 |
| 17 def log(level, message): | |
| 18 if level in logLevels: | |
| 19 print("%s: %s"%(level, message)) | |
| 20 | |
| 21 | |
| 22 def prettyPrintNode(node): | |
| 23 nt = node['node-type'] | |
| 24 att = '' | |
| 25 if nt == 'ENTITY': | |
| 26 att = " %s=%s "%('oc',node['object-class']) | |
| 27 | |
| 28 elif nt == 'ATTRIBUTE': | |
| 29 att = " %s=%s "%('name',node['name']) | |
| 30 | |
| 31 elif nt == 'RELATION': | |
| 39 | 32 att = " %s=%s "%('oc',node['object-class']) |
| 38 | 33 |
| 34 s = "%s%s[%s]"%(nt, att, node) | |
| 35 return s | |
| 36 | |
| 37 | |
| 39 | 38 nodeCsvFieldList = ['node_type', 'id', 'row_id', 'object_class', 'user', 'public', 'type', |
| 39 'version', 'modification_time', 'system_status', 'content_type', | |
| 40 'source_id', 'source_modif', 'source_obj_class', 'target_id', 'target_modif', 'target_obj_class', | |
| 41 'own_value_b64'] | |
| 42 | |
| 43 nodeCsvFieldMap = {'node_type':'node-type', 'id':'id', 'row_id':'row-id', 'object_class':'object-class', 'user':'user', | |
| 44 'public':'public', 'type':'type', | |
| 45 'version':'version', 'modification_time':'mtime', 'system_status':'system-status', 'content_type':'content-type', | |
| 46 'source_id':'source-id', 'source_modif':'source-mtime', 'source_obj_class':'source-oc', | |
| 47 'target_id':'target-id', 'target_modif':'target-mtime', 'target_obj_class':'target-oc', | |
| 48 'own_value_b64':'b64-value'} | |
| 49 | |
| 50 | |
| 51 def printHeaderCsv(outFile): | |
| 52 s = "" | |
| 53 for cf in nodeCsvFieldList: | |
| 54 if s == "": | |
| 55 s += "%s"%cf | |
| 56 else: | |
| 57 s += ",%s"%cf | |
| 58 | |
| 59 print(s, file=outFile) | |
| 60 | |
| 61 | |
| 62 def printNodeCsv(node, outFile): | |
| 63 s = "" | |
| 64 for cf in nodeCsvFieldList: | |
| 65 nf = nodeCsvFieldMap[cf] | |
| 66 if s == "": | |
| 67 s += "%s"%(node.get(nf, '')) | |
| 68 else: | |
| 69 s += ",%s"%(node.get(nf, '')) | |
| 70 | |
| 71 print(s, file=outFile) | |
| 72 | |
| 73 | |
| 38 | 74 def parseStart(line): |
| 75 tstamp = None | |
| 76 tm = re.match('(\d+-\d+-\d+ \d+:\d+:\d+)', line) | |
| 77 if tm: | |
| 78 tstamp = tm.group(1) | |
| 79 | |
|
42
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
80 sm = re.search('START Saving (\w+) \[ID=(\w*)', line) |
| 38 | 81 if sm: |
| 82 return {'time': tstamp, 'oc': sm.group(1), 'id': sm.group(2)} | |
| 83 | |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
84 sm = re.search('Deleting entity \[ID=(\d*)', line) |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
85 if sm: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
86 return {'time': tstamp, 'id': sm.group(1)} |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
87 |
|
42
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
88 sm = re.search('END Saving (\w+) \[ID=(\d*)', line) |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
89 if sm: |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
90 return {'time': tstamp, 'oc': sm.group(1), 'id': sm.group(2)} |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
91 |
| 38 | 92 return None |
| 93 | |
| 94 | |
| 95 def parseSave(line): | |
| 96 match = re.search('([A-Z]+)\[([^\]]+)\]', line) | |
| 97 if match: | |
| 98 data = {'node-type': match.group(1)} | |
| 99 segs = match.group(2).split(', ') | |
| 100 for seg in segs: | |
| 101 k, v = seg.split('=', 1) | |
| 39 | 102 |
| 103 # fix bug with relation's object-class parameter | |
| 104 if k == 'object_class': | |
| 105 k = 'object-class' | |
| 106 | |
| 38 | 107 data[k] = v.strip('"') |
| 108 | |
| 39 | 109 # normalize attriute's name to object-class |
| 110 if k == 'name': | |
| 111 data['object-class'] = v.strip('"') | |
| 112 | |
| 38 | 113 return data |
| 114 | |
| 115 return None | |
| 116 | |
| 117 | |
| 118 def equalNodes(prev, cur): | |
| 119 log("DEBUG", "compare: %s vs %s"%(prev, cur)) | |
| 120 if prev['id'] != cur['id']: | |
| 121 log("INFO", "node id mismatch!") | |
| 122 return False | |
| 123 | |
| 124 if prev['node-type'] != cur['node-type']: | |
| 125 log("INFO", "node node-type mismatch!") | |
| 126 return False | |
| 127 | |
| 128 if prev.get('source-id', None) != cur.get('source-id', None): | |
| 129 log("INFO", "node source_id mismatch!") | |
| 130 return False | |
| 131 | |
| 132 if prev.get('target-id', None) != cur.get('target-id', None): | |
| 133 log("INFO", "node target_id mismatch!") | |
| 134 return False | |
| 135 | |
| 136 if prev['b64-value'] != cur['b64-value']: | |
| 137 log("INFO", "node ownvalue mismatch!") | |
| 138 return False | |
| 139 | |
| 140 return True | |
| 141 | |
| 142 | |
| 143 def getSimilarNode(prev, curList): | |
| 144 nt = prev['node-type'] | |
| 145 if nt == 'ATTRIBUTE': | |
| 146 for n in curList: | |
| 147 if n['node-type'] == 'ATTRIBUTE' \ | |
| 148 and prev['name'] == n['name']: | |
| 149 # attribute with same name | |
| 150 log("DEBUG", "similar attributes: %s vs %s"%(prev, n)) | |
| 151 return n | |
| 152 | |
| 153 elif nt == 'RELATION': | |
| 154 for n in curList: | |
| 155 if n['node-type'] == 'RELATION' \ | |
| 156 and prev['source-id'] == n['source-id'] \ | |
| 157 and prev['target-id'] == n['target-id'] \ | |
| 39 | 158 and prev['object-class'] == n['object-class']: |
| 38 | 159 # relation with same source, target and type |
| 160 log("DEBUG", "similar relations: %s vs %s"%(prev, n)) | |
| 161 return n | |
| 162 | |
| 163 return None | |
| 164 | |
| 165 | |
| 39 | 166 def compareNodeLists(prev, cur, ctx, lostFile=None): |
| 41 | 167 """compare list of previous and current nodes. |
| 168 | |
| 169 prints results to the log and output file. | |
| 170 """ | |
| 38 | 171 prevNodes = {} |
| 172 curNodes = {} | |
| 173 | |
| 174 # | |
| 175 # read nodes | |
| 176 # | |
| 177 for n in prev: | |
| 178 nid = n['id'] | |
| 179 if nid not in prevNodes: | |
| 180 prevNodes[nid] = n | |
| 181 else: | |
| 182 log("DEBUG", "duplicate save of prev node id="+nid) | |
| 183 if isinstance(prevNodes[nid], list): | |
| 184 prevNodes[nid].append(n) | |
| 185 else: | |
| 186 prevNodes[nid] = [prevNodes[nid], n] | |
| 187 | |
| 188 for n in cur: | |
| 189 nid = n['id'] | |
| 190 if nid not in curNodes: | |
| 191 curNodes[nid] = n | |
| 192 else: | |
| 193 log("DEBUG", "duplicate save of cur node id="+nid) | |
| 194 if isinstance(curNodes[nid], list): | |
| 195 curNodes[nid].append(n) | |
| 196 else: | |
| 197 curNodes[nid] = [curNodes[nid], n] | |
| 198 | |
| 199 # | |
| 200 # compare nodes | |
| 201 # | |
| 202 curNodeUnchecked = set(curNodes.keys()) | |
| 203 addPrevNodes = [] | |
| 204 addCurNodes = [] | |
| 205 | |
| 206 for nid in prevNodes: | |
| 207 prevNode = prevNodes[nid] | |
| 208 if isinstance(prevNode, list): | |
| 209 log("DEBUG", "multi-save prev node: %s"%prevNode) | |
| 210 # use the last version(?) | |
| 211 prevNode = prevNode[-1] | |
| 212 | |
| 213 if nid not in curNodes: | |
| 214 if prevNode['node-type'] == 'ATTRIBUTE' and prevNode['b64-value'] == '': | |
| 215 # emtpy attribute - ignore | |
| 216 continue | |
| 217 | |
| 218 else: | |
| 219 log("DEBUG", "node %s not in cur saves! %s"%(nid,prevNode)) | |
| 220 addPrevNodes.append(prevNode) | |
| 221 continue | |
| 222 | |
| 223 curNode = curNodes[nid] | |
| 224 if isinstance(curNode, list): | |
| 225 log("DEBUG", "multi-save cur node: %s"%curNode) | |
| 226 # use the last version? | |
| 227 curNode = curNode[-1] | |
| 228 | |
| 229 equalNodes(prevNode, curNode) | |
| 230 | |
| 231 curNodeUnchecked.remove(nid) | |
| 232 | |
| 233 # make list of additional current (=new) nodes | |
| 234 for nid in curNodeUnchecked: | |
| 41 | 235 curNode = curNodes[nid] |
| 236 # list can contain lists | |
| 237 if isinstance(curNode, list): | |
| 238 # use the last version(?) | |
| 239 curNode = curNode[-1] | |
| 240 | |
| 241 addCurNodes.append(curNode) | |
| 38 | 242 log("DEBUG", "new node %s"%curNodes[nid]) |
| 243 | |
| 244 # compare missing and new nodes | |
| 245 for n in addPrevNodes.copy(): | |
| 246 sn = getSimilarNode(n, addCurNodes) | |
| 247 if sn is not None: | |
| 248 # similar is good enough | |
| 249 addPrevNodes.remove(n) | |
| 250 addCurNodes.remove(sn) | |
| 251 | |
| 252 if len(addPrevNodes) > 0: | |
| 253 #print("ERROR: lost nodes: %s"%[prettyPrintNode(n) for n in addPrevNodes]) | |
| 254 log("ERROR", "in %s"%ctx) | |
| 255 for n in addPrevNodes: | |
| 256 log("ERROR","lost node: %s"%prettyPrintNode(n)) | |
| 39 | 257 if lostFile is not None: |
| 258 printNodeCsv(n, lostFile) | |
| 38 | 259 |
| 260 if len(addCurNodes) > 0: | |
| 261 #print("INFO: new nodes: %s"%[prettyPrintNode(n) for n in addCurNodes]) | |
| 262 for n in addCurNodes: | |
| 263 log("INFO", "new node: %s"%prettyPrintNode(n)) | |
| 264 | |
| 265 | |
| 266 | |
| 39 | 267 def analyseLogfile(inFilename, outFilename=None): |
| 268 outFile = None | |
| 269 if outFilename is not None: | |
| 270 outFile = open(outFilename, mode='w') | |
| 271 printHeaderCsv(outFile) | |
| 272 | |
| 273 with open(inFilename) as f: | |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
274 linecnt = 0 |
| 38 | 275 saving = 0 |
|
42
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
276 openSaves = {} |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
277 savingPrev = 0 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
278 deleting = 0 |
| 38 | 279 saveCtx = None |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
280 deleteCtx = None |
| 38 | 281 prevSaves = [] |
| 282 saves = [] | |
| 283 | |
| 284 for line in f: | |
| 285 linecnt += 1 | |
| 286 if '*************** START Saving' in line: | |
| 287 saving += 1 | |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
288 # make sure delete is off |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
289 deleting = 0 |
| 38 | 290 log('DEBUG', line) |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
291 # parse time and id |
| 38 | 292 saveCtx = parseStart(line) |
|
42
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
293 saveId = saveCtx['id'] |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
294 if saveId in openSaves: |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
295 log('ERROR', "Duplicate save for same id! %s"%saveCtx) |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
296 |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
297 openSaves[saveId] = saveCtx |
| 38 | 298 |
|
42
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
299 if len(openSaves) > 1: |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
300 log("ERROR", "Multiple open save (%s) in #%s of %s"%(saving, linecnt, line)) |
| 38 | 301 # TODO: what now? |
| 302 | |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
303 elif 'Deleting entity' in line: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
304 deleting += 1 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
305 log('DEBUG', line) |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
306 deleteCtx = parseStart(line) |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
307 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
308 if deleting > 1: |
| 45 | 309 log("ERROR", "Concurrent delete (%s) in #%s of %s"%(deleting, linecnt, line)) |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
310 # TODO: what now? |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
311 break |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
312 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
313 elif 'transactionlog' in line: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
314 if '* START save previous' in line: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
315 savingPrev += 1 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
316 |
| 45 | 317 elif '* END ...save previous' in line or '* End ...save previous' in line: |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
318 if deleting > 0 and savingPrev > 0: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
319 # this should be the end of the save prev from deleting |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
320 deleting -= 1 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
321 deleteCtx = None |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
322 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
323 savingPrev -= 1 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
324 |
| 45 | 325 if savingPrev < 0: |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
326 log("ERROR", "Too many END save previous!") |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
327 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
328 elif 'save previous' in line: |
| 38 | 329 data = parseSave(line) |
| 330 if data is None: | |
| 331 log("DEBUG", "Error parsing line: %s"%line) | |
| 332 continue | |
| 333 | |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
334 if omitDeleted and deleting > 0 and savingPrev > 0: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
335 # this should be a save prev from deleting |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
336 delId = deleteCtx['id'] |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
337 # check if node is related to deleted id |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
338 if data.get('id', None) == delId or data.get('source-id', None) == delId \ |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
339 or data.get('target-id', None) == delId: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
340 log('DEBUG', "intentionally deleted node: %s"%data) |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
341 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
342 else: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
343 log('ERROR', "Node without matching id in delete! %s"%data) |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
344 prevSaves.append(data) |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
345 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
346 else: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
347 prevSaves.append(data) |
| 38 | 348 |
| 349 elif 'save' in line: | |
| 350 data = parseSave(line) | |
| 351 if data is None: | |
| 352 log("DEBUG", "Error parsing line: %s"%line) | |
| 353 continue | |
| 354 | |
| 355 saves.append(parseSave(line)) | |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
356 |
| 38 | 357 elif '*************** END Saving' in line: |
|
42
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
358 log('DEBUG', line) |
| 38 | 359 saving -= 1 |
|
42
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
360 saveCtx = parseStart(line) |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
361 saveId = saveCtx['id'] |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
362 if saveId in openSaves: |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
363 log('DEBUG', "End save matches start save: %s"%saveCtx) |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
364 del openSaves[saveId] |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
365 |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
366 elif 'null' in openSaves: |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
367 log('DEBUG', "Assume end save (of %s) matches start save null: %s"%(len(openSaves), saveCtx)) |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
368 del openSaves['null'] |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
369 |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
370 else: |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
371 log("ERROR", "End save without start save! %s"%saveCtx) |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
372 |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
373 # make sure delete is off |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
374 deleting = 0 |
| 38 | 375 |
|
42
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
376 if len(openSaves) > 0: |
|
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
377 log('WARNING', "Still open saves (%s) during end save in #%s of %s"%(saving, linecnt, line)) |
| 38 | 378 |
| 379 elif saving < 0: | |
|
42
4dee9586cc44
added checks for matching ids in start/end save.
Robert Casties <casties@mpiwg-berlin.mpg.de>
parents:
41
diff
changeset
|
380 log('ERROR', "Too many END saves!") |
| 38 | 381 break |
| 382 | |
| 383 log("INFO", "saving %s"%saveCtx) | |
| 384 log("INFO", "prev saves: %s"%len(prevSaves)) | |
| 385 log("INFO", "saves: %s"%len(saves)) | |
| 386 | |
| 387 if len(prevSaves) > 0: | |
| 39 | 388 compareNodeLists(prevSaves, saves, saveCtx, outFile) |
| 38 | 389 |
| 390 prevSaves = [] | |
| 391 saves = [] | |
| 392 | |
| 393 if maxLinecnt is not None and linecnt >= maxLinecnt: | |
| 394 break | |
| 395 | |
| 396 log("SYSMSG", "%s lines of logfile scanned"%linecnt) | |
| 397 | |
|
40
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
398 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
399 # |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
400 # public static void main :-) |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
401 # |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
402 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
403 input_fn = None |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
404 output_fn = None |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
405 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
406 # parse command line parameters |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
407 if len(sys.argv) > 2: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
408 input_fn = sys.argv[1] |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
409 output_fn = sys.argv[2] |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
410 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
411 # run analysis |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
412 analyseLogfile(input_fn, output_fn) |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
413 |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
414 else: |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
415 print("ERROR: missing parameters!") |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
416 print("use: check_ismi_log logfile csvfile") |
|
f38ca3eb1088
check_ismi_log analyser ignores deleted entities now.
casties
parents:
39
diff
changeset
|
417 exit(1) |