drupalISMI: importFromOpenMind/importer/check_ismi

annotate importFromOpenMind/importer/check_ismi_log.py @ 42:4dee9586cc44

added checks for matching ids in start/end save.

author	Robert Casties <casties@mpiwg-berlin.mpg.de>
date	Fri, 11 Nov 2016 17:20:46 +0100
parents	5b3cd0b66b30
children	277ea02906f9

rev	line source
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	1
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	2 # coding: utf-8
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	3
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	4 import sys, re
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	5
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	6 # max number of lines to read (for testing)
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	7 maxLinecnt = None
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	8
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	9 # do not output deleted nodes
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	10 omitDeleted = True
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	11
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	12 # active log levels for logging
41 5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	13 logLevels = {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'SYSMSG'}
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	14 #logLevels = {'INFO', 'WARNING', 'ERROR', 'SYSMSG'}
41 5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	15 #logLevels = {'ERROR', 'SYSMSG'}
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	16
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	17 def log(level, message):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	18 if level in logLevels:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	19 print("%s: %s"%(level, message))
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	20
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	21
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	22 def prettyPrintNode(node):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	23 nt = node['node-type']
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	24 att = ''
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	25 if nt == 'ENTITY':
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	26 att = " %s=%s "%('oc',node['object-class'])
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	27
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	28 elif nt == 'ATTRIBUTE':
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	29 att = " %s=%s "%('name',node['name'])
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	30
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	31 elif nt == 'RELATION':
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	32 att = " %s=%s "%('oc',node['object-class'])
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	33
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	34 s = "%s%s[%s]"%(nt, att, node)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	35 return s
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	36
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	37
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	38 nodeCsvFieldList = ['node_type', 'id', 'row_id', 'object_class', 'user', 'public', 'type',
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	39 'version', 'modification_time', 'system_status', 'content_type',
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	40 'source_id', 'source_modif', 'source_obj_class', 'target_id', 'target_modif', 'target_obj_class',
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	41 'own_value_b64']
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	42
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	43 nodeCsvFieldMap = {'node_type':'node-type', 'id':'id', 'row_id':'row-id', 'object_class':'object-class', 'user':'user',
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	44 'public':'public', 'type':'type',
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	45 'version':'version', 'modification_time':'mtime', 'system_status':'system-status', 'content_type':'content-type',
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	46 'source_id':'source-id', 'source_modif':'source-mtime', 'source_obj_class':'source-oc',
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	47 'target_id':'target-id', 'target_modif':'target-mtime', 'target_obj_class':'target-oc',
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	48 'own_value_b64':'b64-value'}
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	49
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	50
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	51 def printHeaderCsv(outFile):
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	52 s = ""
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	53 for cf in nodeCsvFieldList:
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	54 if s == "":
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	55 s += "%s"%cf
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	56 else:
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	57 s += ",%s"%cf
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	58
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	59 print(s, file=outFile)
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	60
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	61
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	62 def printNodeCsv(node, outFile):
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	63 s = ""
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	64 for cf in nodeCsvFieldList:
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	65 nf = nodeCsvFieldMap[cf]
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	66 if s == "":
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	67 s += "%s"%(node.get(nf, ''))
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	68 else:
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	69 s += ",%s"%(node.get(nf, ''))
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	70
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	71 print(s, file=outFile)
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	72
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	73
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	74 def parseStart(line):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	75 tstamp = None
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	76 tm = re.match('(\d+-\d+-\d+ \d+:\d+:\d+)', line)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	77 if tm:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	78 tstamp = tm.group(1)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	79
42 4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	80 sm = re.search('START Saving (\w+) \[ID=(\w*)', line)
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	81 if sm:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	82 return {'time': tstamp, 'oc': sm.group(1), 'id': sm.group(2)}
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	83
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	84 sm = re.search('Deleting entity \[ID=(\d*)', line)
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	85 if sm:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	86 return {'time': tstamp, 'id': sm.group(1)}
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	87
42 4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	88 sm = re.search('END Saving (\w+) \[ID=(\d*)', line)
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	89 if sm:
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	90 return {'time': tstamp, 'oc': sm.group(1), 'id': sm.group(2)}
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	91
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	92 return None
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	93
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	94
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	95 def parseSave(line):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	96 match = re.search('([A-Z]+)\[([^\]]+)\]', line)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	97 if match:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	98 data = {'node-type': match.group(1)}
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	99 segs = match.group(2).split(', ')
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	100 for seg in segs:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	101 k, v = seg.split('=', 1)
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	102
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	103 # fix bug with relation's object-class parameter
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	104 if k == 'object_class':
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	105 k = 'object-class'
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	106
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	107 data[k] = v.strip('"')
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	108
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	109 # normalize attriute's name to object-class
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	110 if k == 'name':
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	111 data['object-class'] = v.strip('"')
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	112
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	113 return data
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	114
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	115 return None
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	116
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	117
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	118 def equalNodes(prev, cur):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	119 log("DEBUG", "compare: %s vs %s"%(prev, cur))
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	120 if prev['id'] != cur['id']:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	121 log("INFO", "node id mismatch!")
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	122 return False
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	123
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	124 if prev['node-type'] != cur['node-type']:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	125 log("INFO", "node node-type mismatch!")
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	126 return False
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	127
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	128 if prev.get('source-id', None) != cur.get('source-id', None):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	129 log("INFO", "node source_id mismatch!")
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	130 return False
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	131
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	132 if prev.get('target-id', None) != cur.get('target-id', None):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	133 log("INFO", "node target_id mismatch!")
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	134 return False
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	135
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	136 if prev['b64-value'] != cur['b64-value']:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	137 log("INFO", "node ownvalue mismatch!")
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	138 return False
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	139
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	140 return True
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	141
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	142
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	143 def getSimilarNode(prev, curList):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	144 nt = prev['node-type']
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	145 if nt == 'ATTRIBUTE':
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	146 for n in curList:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	147 if n['node-type'] == 'ATTRIBUTE' \
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	148 and prev['name'] == n['name']:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	149 # attribute with same name
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	150 log("DEBUG", "similar attributes: %s vs %s"%(prev, n))
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	151 return n
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	152
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	153 elif nt == 'RELATION':
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	154 for n in curList:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	155 if n['node-type'] == 'RELATION' \
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	156 and prev['source-id'] == n['source-id'] \
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	157 and prev['target-id'] == n['target-id'] \
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	158 and prev['object-class'] == n['object-class']:
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	159 # relation with same source, target and type
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	160 log("DEBUG", "similar relations: %s vs %s"%(prev, n))
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	161 return n
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	162
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	163 return None
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	164
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	165
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	166 def compareNodeLists(prev, cur, ctx, lostFile=None):
41 5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	167 """compare list of previous and current nodes.
5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	168
5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	169 prints results to the log and output file.
5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	170 """
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	171 prevNodes = {}
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	172 curNodes = {}
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	173
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	174 #
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	175 # read nodes
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	176 #
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	177 for n in prev:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	178 nid = n['id']
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	179 if nid not in prevNodes:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	180 prevNodes[nid] = n
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	181 else:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	182 log("DEBUG", "duplicate save of prev node id="+nid)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	183 if isinstance(prevNodes[nid], list):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	184 prevNodes[nid].append(n)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	185 else:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	186 prevNodes[nid] = [prevNodes[nid], n]
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	187
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	188 for n in cur:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	189 nid = n['id']
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	190 if nid not in curNodes:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	191 curNodes[nid] = n
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	192 else:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	193 log("DEBUG", "duplicate save of cur node id="+nid)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	194 if isinstance(curNodes[nid], list):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	195 curNodes[nid].append(n)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	196 else:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	197 curNodes[nid] = [curNodes[nid], n]
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	198
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	199 #
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	200 # compare nodes
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	201 #
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	202 curNodeUnchecked = set(curNodes.keys())
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	203 addPrevNodes = []
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	204 addCurNodes = []
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	205
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	206 for nid in prevNodes:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	207 prevNode = prevNodes[nid]
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	208 if isinstance(prevNode, list):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	209 log("DEBUG", "multi-save prev node: %s"%prevNode)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	210 # use the last version(?)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	211 prevNode = prevNode[-1]
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	212
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	213 if nid not in curNodes:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	214 if prevNode['node-type'] == 'ATTRIBUTE' and prevNode['b64-value'] == '':
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	215 # emtpy attribute - ignore
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	216 continue
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	217
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	218 else:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	219 log("DEBUG", "node %s not in cur saves! %s"%(nid,prevNode))
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	220 addPrevNodes.append(prevNode)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	221 continue
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	222
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	223 curNode = curNodes[nid]
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	224 if isinstance(curNode, list):
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	225 log("DEBUG", "multi-save cur node: %s"%curNode)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	226 # use the last version?
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	227 curNode = curNode[-1]
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	228
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	229 equalNodes(prevNode, curNode)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	230
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	231 curNodeUnchecked.remove(nid)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	232
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	233 # make list of additional current (=new) nodes
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	234 for nid in curNodeUnchecked:
41 5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	235 curNode = curNodes[nid]
5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	236 # list can contain lists
5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	237 if isinstance(curNode, list):
5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	238 # use the last version(?)
5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	239 curNode = curNode[-1]
5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	240
5b3cd0b66b30 fix bug with multiple-saved nodes. casties parents: 40 diff changeset	241 addCurNodes.append(curNode)
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	242 log("DEBUG", "new node %s"%curNodes[nid])
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	243
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	244 # compare missing and new nodes
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	245 for n in addPrevNodes.copy():
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	246 sn = getSimilarNode(n, addCurNodes)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	247 if sn is not None:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	248 # similar is good enough
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	249 addPrevNodes.remove(n)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	250 addCurNodes.remove(sn)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	251
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	252 if len(addPrevNodes) > 0:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	253 #print("ERROR: lost nodes: %s"%[prettyPrintNode(n) for n in addPrevNodes])
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	254 log("ERROR", "in %s"%ctx)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	255 for n in addPrevNodes:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	256 log("ERROR","lost node: %s"%prettyPrintNode(n))
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	257 if lostFile is not None:
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	258 printNodeCsv(n, lostFile)
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	259
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	260 if len(addCurNodes) > 0:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	261 #print("INFO: new nodes: %s"%[prettyPrintNode(n) for n in addCurNodes])
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	262 for n in addCurNodes:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	263 log("INFO", "new node: %s"%prettyPrintNode(n))
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	264
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	265
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	266
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	267 def analyseLogfile(inFilename, outFilename=None):
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	268 outFile = None
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	269 if outFilename is not None:
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	270 outFile = open(outFilename, mode='w')
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	271 printHeaderCsv(outFile)
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	272
1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	273 with open(inFilename) as f:
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	274 linecnt = 0
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	275 saving = 0
42 4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	276 openSaves = {}
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	277 savingPrev = 0
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	278 deleting = 0
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	279 saveCtx = None
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	280 deleteCtx = None
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	281 prevSaves = []
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	282 saves = []
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	283
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	284 for line in f:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	285 linecnt += 1
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	286 if '*************** START Saving' in line:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	287 saving += 1
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	288 # make sure delete is off
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	289 deleting = 0
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	290 log('DEBUG', line)
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	291 # parse time and id
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	292 saveCtx = parseStart(line)
42 4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	293 saveId = saveCtx['id']
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	294 if saveId in openSaves:
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	295 log('ERROR', "Duplicate save for same id! %s"%saveCtx)
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	296
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	297 openSaves[saveId] = saveCtx
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	298
42 4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	299 if len(openSaves) > 1:
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	300 log("ERROR", "Multiple open save (%s) in #%s of %s"%(saving, linecnt, line))
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	301 # TODO: what now?
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	302
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	303 elif 'Deleting entity' in line:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	304 deleting += 1
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	305 log('DEBUG', line)
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	306 deleteCtx = parseStart(line)
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	307
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	308 if deleting > 1:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	309 log("ERROR", "Concurrent delete (%s) in #%s of %s"%(saving, linecnt, line))
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	310 # TODO: what now?
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	311 break
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	312
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	313 elif 'transactionlog' in line:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	314 if '* START save previous' in line:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	315 savingPrev += 1
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	316
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	317 elif '* End ...save previous' in line:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	318 if deleting > 0 and savingPrev > 0:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	319 # this should be the end of the save prev from deleting
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	320 deleting -= 1
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	321 deleteCtx = None
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	322
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	323 savingPrev -= 1
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	324
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	325 if saving < 0:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	326 log("ERROR", "Too many END save previous!")
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	327
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	328 elif 'save previous' in line:
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	329 data = parseSave(line)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	330 if data is None:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	331 log("DEBUG", "Error parsing line: %s"%line)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	332 continue
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	333
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	334 if omitDeleted and deleting > 0 and savingPrev > 0:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	335 # this should be a save prev from deleting
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	336 delId = deleteCtx['id']
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	337 # check if node is related to deleted id
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	338 if data.get('id', None) == delId or data.get('source-id', None) == delId \
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	339 or data.get('target-id', None) == delId:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	340 log('DEBUG', "intentionally deleted node: %s"%data)
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	341
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	342 else:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	343 log('ERROR', "Node without matching id in delete! %s"%data)
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	344 prevSaves.append(data)
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	345
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	346 else:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	347 prevSaves.append(data)
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	348
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	349 elif 'save' in line:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	350 data = parseSave(line)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	351 if data is None:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	352 log("DEBUG", "Error parsing line: %s"%line)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	353 continue
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	354
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	355 saves.append(parseSave(line))
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	356
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	357 elif '*************** END Saving' in line:
42 4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	358 log('DEBUG', line)
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	359 saving -= 1
42 4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	360 saveCtx = parseStart(line)
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	361 saveId = saveCtx['id']
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	362 if saveId in openSaves:
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	363 log('DEBUG', "End save matches start save: %s"%saveCtx)
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	364 del openSaves[saveId]
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	365
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	366 elif 'null' in openSaves:
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	367 log('DEBUG', "Assume end save (of %s) matches start save null: %s"%(len(openSaves), saveCtx))
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	368 del openSaves['null']
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	369
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	370 else:
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	371 log("ERROR", "End save without start save! %s"%saveCtx)
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	372
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	373 # make sure delete is off
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	374 deleting = 0
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	375
42 4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	376 if len(openSaves) > 0:
4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	377 log('WARNING', "Still open saves (%s) during end save in #%s of %s"%(saving, linecnt, line))
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	378
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	379 elif saving < 0:
42 4dee9586cc44 added checks for matching ids in start/end save. Robert Casties <casties@mpiwg-berlin.mpg.de> parents: 41 diff changeset	380 log('ERROR', "Too many END saves!")
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	381 break
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	382
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	383 log("INFO", "saving %s"%saveCtx)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	384 log("INFO", "prev saves: %s"%len(prevSaves))
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	385 log("INFO", "saves: %s"%len(saves))
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	386
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	387 if len(prevSaves) > 0:
39 1867bc2180c5 better check_ismi_log analyser. casties parents: 38 diff changeset	388 compareNodeLists(prevSaves, saves, saveCtx, outFile)
38 9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	389
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	390 prevSaves = []
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	391 saves = []
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	392
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	393 if maxLinecnt is not None and linecnt >= maxLinecnt:
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	394 break
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	395
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	396 log("SYSMSG", "%s lines of logfile scanned"%linecnt)
9ab136f412a1 new first version of check_ismi_log analyser. casties parents: diff changeset	397
40 f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	398
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	399 #
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	400 # public static void main :-)
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	401 #
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	402
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	403 input_fn = None
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	404 output_fn = None
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	405
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	406 # parse command line parameters
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	407 if len(sys.argv) > 2:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	408 input_fn = sys.argv[1]
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	409 output_fn = sys.argv[2]
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	410
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	411 # run analysis
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	412 analyseLogfile(input_fn, output_fn)
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	413
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	414 else:
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	415 print("ERROR: missing parameters!")
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	416 print("use: check_ismi_log logfile csvfile")
f38ca3eb1088 check_ismi_log analyser ignores deleted entities now. casties parents: 39 diff changeset	417 exit(1)

Mercurial > hg > drupalISMI

annotate importFromOpenMind/importer/check_ismi_log.py @ 42:4dee9586cc44