--- a/importFromOpenMind/importer/check_ismi_log.py	Fri Oct 28 19:55:19 2016 +0200
+++ b/importFromOpenMind/importer/check_ismi_log.py	Mon Oct 31 18:33:12 2016 +0100
@@ -1,12 +1,16 @@
 
 # coding: utf-8
 
-import re
+import sys, re
 
 # max number of lines to read (for testing)
 maxLinecnt = None
 
+# do not output deleted nodes
+omitDeleted = True
+
 # active log levels for logging
+#logLevels = {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'SYSMSG'}
 #logLevels = {'INFO', 'WARNING', 'ERROR', 'SYSMSG'}
 logLevels = {'ERROR', 'SYSMSG'}
 
@@ -77,6 +81,10 @@
     if sm:
         return {'time': tstamp, 'oc': sm.group(1), 'id': sm.group(2)}
     
+    sm = re.search('Deleting entity \[ID=(\d*)', line)
+    if sm:
+        return {'time': tstamp, 'id': sm.group(1)}
+    
     return None
 
 
@@ -249,9 +257,12 @@
         printHeaderCsv(outFile)
     
     with open(inFilename) as f:
+        linecnt = 0
         saving = 0
-        linecnt = 0
+        savingPrev = 0
+        deleting = 0
         saveCtx = None
+        deleteCtx = None
         prevSaves = []
         saves = []
         
@@ -259,21 +270,61 @@
             linecnt += 1
             if '*************** START Saving' in line:
                 saving += 1
+                # make sure delete is off
+                deleting = 0
                 log('DEBUG', line)
+                # parse time and id
                 saveCtx = parseStart(line)
                 
                 if saving > 1:
                     log("ERROR", "Concurrent save (%s) in #%s of %s"%(saving, linecnt, line))
                     # TODO: what now?
                     
-            elif 'INFO transactionlog' in line:
-                if 'save previous' in line:
+            elif 'Deleting entity' in line:
+                deleting += 1
+                log('DEBUG', line)
+                deleteCtx = parseStart(line)
+                
+                if deleting > 1:
+                    log("ERROR", "Concurrent delete (%s) in #%s of %s"%(saving, linecnt, line))
+                    # TODO: what now?
+                    break
+                    
+            elif 'transactionlog' in line:
+                if '* START save previous' in line:
+                    savingPrev += 1
+
+                elif '* End ...save previous' in line:
+                    if deleting > 0 and savingPrev > 0:
+                        # this should be the end of the save prev from deleting
+                        deleting -= 1
+                        deleteCtx = None
+                    
+                    savingPrev -= 1
+                    
+                    if saving < 0:
+                        log("ERROR", "Too many END save previous!")
+                
+                elif 'save previous' in line:
                     data = parseSave(line)
                     if data is None:
                         log("DEBUG", "Error parsing line: %s"%line)
                         continue
                         
-                    prevSaves.append(data)
+                    if omitDeleted and deleting > 0 and savingPrev > 0:
+                        # this should be a save prev from deleting
+                        delId = deleteCtx['id']
+                        # check if node is related to deleted id
+                        if data.get('id', None) == delId or data.get('source-id', None) == delId \
+                            or data.get('target-id', None) == delId:
+                            log('DEBUG', "intentionally deleted node: %s"%data)
+                        
+                        else:
+                            log('ERROR', "Node without matching id in delete! %s"%data)
+                            prevSaves.append(data)
+                            
+                    else:
+                        prevSaves.append(data)
                     
                 elif 'save' in line:
                     data = parseSave(line)
@@ -282,9 +333,11 @@
                         continue
                     
                     saves.append(parseSave(line))
-                
+
             elif '*************** END Saving' in line:
                 saving -= 1
+                # make sure delete is off
+                deleting = 0
                 log('DEBUG', line)
                 
                 if saving > 0:
@@ -304,12 +357,28 @@
                 prevSaves = []
                 saves = []
                             
-                    
             if maxLinecnt is not None and linecnt >= maxLinecnt:
                 break
                 
         log("SYSMSG", "%s lines of logfile scanned"%linecnt)
         
-        
-# run analysis
-analyseLogfile('ismi-161011.log', 'ismi-161011-lost.csv')
+
+#        
+# public static void main :-)
+#
+
+input_fn = None
+output_fn = None
+
+# parse command line parameters
+if len(sys.argv) > 2:
+    input_fn = sys.argv[1]
+    output_fn = sys.argv[2]
+
+    # run analysis
+    analyseLogfile(input_fn, output_fn)
+
+else:
+    print("ERROR: missing parameters!")
+    print("use: check_ismi_log logfile csvfile")
+    exit(1)