# HG changeset patch # User casties # Date 1486143976 -3600 # Node ID f3945ef1e6a45c181183e9726cb7b681ce899ec5 # Parent 277ea02906f98c1d8d1426126df20468516b8638 new importer for OM4XML dump file. diff -r 277ea02906f9 -r f3945ef1e6a4 importFromOpenMind/importer/ismi2model.py --- a/importFromOpenMind/importer/ismi2model.py Fri Dec 09 12:24:21 2016 +0100 +++ b/importFromOpenMind/importer/ismi2model.py Fri Feb 03 18:46:16 2017 +0100 @@ -9,7 +9,7 @@ output_fn = "ismi_graph.gpickle" # OpenMind base URL -#baseURL="http://localhost:18080/om4-ismi/jsonInterface?" +#baseURL="http://ismi.mpiwg-berlin.mpg.de//om4-ismi/jsonInterface?" baseURL="http://localhost:18080/ismi-richfaces/jsonInterface?" # node types to exclude from the graph @@ -282,8 +282,6 @@ # return -# In[119]: - def importAllEnts(etypes): for etype in etypes: diff -r 277ea02906f9 -r f3945ef1e6a4 importFromOpenMind/importer/ismixml2model.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/importFromOpenMind/importer/ismixml2model.py Fri Feb 03 18:46:16 2017 +0100 @@ -0,0 +1,359 @@ +import xml.etree.ElementTree as ET +import json +import networkx +import sys + +## configure behaviour + +# output filename +output_fn = "ismi_graph.gpickle" + +input_fn = "openmind-data.xml" + + +# node types to exclude from the graph +exclude_objects_of_type = ['DIGITALIZATION', 'REFERENCE'] + +# attributes to exclude +exclude_attributes_of_type = [ + 'lw', + 'node_type', + 'nov', + 'notes_old' +] + +# name of type attribute +node_type_attribute = '_type' +rel_type_attribute = '_type' + +#ismi_types=["PERSON","WITNESS","CODEX","PLACE","COLLECTION","REPOSITORY"] + + +nx_graph = networkx.MultiDiGraph() + +nx_nodes = {} +ismi_relations = {} +nx_relations = {} + +# active log levels for logging +#logLevels = {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'SYSMSG'} +#logLevels = {'INFO', 'WARNING', 'ERROR', 'SYSMSG'} +logLevels = {'INFO', 'ERROR', 'SYSMSG'} + +def log(level, message): + if level in logLevels: + print("%s: %s"%(level, message)) + + +def fixName(name, is_src_rel=False, is_tar_rel=False, att_from_rel=False): + if is_src_rel: + #name = name + '>' + pass + + if is_tar_rel: + name = '<' + name + + if att_from_rel: + # clean up relations as attribute names + name = name.replace('is_', '') + name = name.replace('has_', '') + name = name.replace('was_', '') + name = name.replace('_of', '') + + return name + + + +def parseYear(val): + year = None + try: + date_json = json.loads(val) + if 'from' in date_json: + year = date_json['from'].get('year', None) + elif 'date' in date_json: + year = date_json['date'].get('year', None) + else: + log("WARNING", "don't know what to do with date %s"%(val)) + + except: + pass + + return year + + +def nodeFromEnt(ent_elem): + """Create a graph node from the given XML entity. + + Creates the node in gdb and returns the node. + """ + # text content of entity element + ov = ent_elem.text or '' + + attrs = {} + + # get attributes element + atts_elem = ent_elem.find('attributes') + + if atts_elem is None: + log('DEBUG', "entity has no attributes: %s"%ent_elem) + + else: + # go through all attributes + for att_elem in atts_elem: + if att_elem.tail is not None: + # tail belongs to parent + ov += att_elem.tail + + ct = att_elem.get('content-type', None) + name = att_elem.get('name', None) + if name in exclude_attributes_of_type: + # exclude attribute + continue + + if ct is None or ct.lower() in ['text', 'arabic', 'bool', 'boolean', 'url', 'language']: + # normal text attribute (assume no content_type is text too...) + val = att_elem.text + + if val is not None and val[0] == '{': + # try to parse as date + year = parseYear(val) + if year is not None: + val = year + + # keep attribute + attrs[name] = val + #if 'nov' in att: + # # add normalized value + # attrs['_n_'+name] = att['nov'] + + elif ct == 'date': + # date attribute + val = att_elem.text + if val is not None: + # try to parse date object to get gregorian year + year = parseYear(val) + if year is not None: + attrs[name] = year + + elif ct == 'num': + # number attribute + val = att_elem.text + if val is not None: + # keep attribute, assume num is int + attrs[name] = int(val) + + elif ct == 'old': + # ignore attribute + continue + + else: + log("WARN", "attribute with unknown content_type: %s"%repr(att_elem)) + # ignore other content types + continue + + # process base attributes + oc = ent_elem.get('object-class') + + # set type + attrs[node_type_attribute] = fixName(oc) + + ismi_id = ent_elem.get('id') + # rename id to ismi_id + attrs['ismi_id'] = ismi_id + + if len(ov) > 0: + # save ov as label + attrs['label'] = ov + #if 'nov' in ent: + # # add normalized value + # attrs['_n_label'] = ent.get('nov') + + # create node + #log('DEBUG', "new node(%s, %s)"%(ismi_id, attrs)) + nx_graph.add_node(ismi_id, **attrs) + node = nx_graph.node[ismi_id] + + return node + + +def relationFromRel(rel_elem): + """Create graph relation from etree element. + """ + rel_id = rel_elem.get('id') + rel_name = rel_elem.get('object-class') + src_id = rel_elem.get('source-id') + tar_id = rel_elem.get('target-id') + if not src_id in nx_nodes: + log("ERROR", "relation %s src node %s missing!"%(rel_id,src_id)) + return None + + if not tar_id in nx_nodes: + log("ERROR", "relation %s tar node %s missing!"%(rel_id,tar_id)) + return None + + ov = rel_elem.text or '' + + attrs = {} + + # get attributes element + atts_elem = rel_elem.find('attributes') + + if atts_elem is not None: + # go through all attributes + for att_elem in atts_elem: + if att_elem.tail is not None: + # tail belongs to parent + ov += att_elem.tail + + ct = att_elem.get('content-type', None) + name = att_elem.get('name', None) + if name in exclude_attributes_of_type: + # exclude attribute + continue + + if ct is None or ct.lower() in ['text', 'arabic', 'bool', 'boolean', 'url', 'language']: + # normal text attribute (assume no content_type is text too...) + val = att_elem.text + + if val is not None and val[0] == '{': + # try to parse as date + year = parseYear(val) + if year is not None: + val = year + + # keep attribute + attrs[name] = val + #if 'nov' in att: + # # add normalized value + # attrs['_n_'+name] = att['nov'] + + elif ct == 'date': + # date attribute + val = att_elem.text + if val is not None: + # try to parse date object to get gregorian year + year = parseYear(val) + if year is not None: + attrs[name] = year + + elif ct == 'num': + # number attribute + val = att_elem.text + if val is not None: + # keep attribute, assume num is int + attrs[name] = int(val) + + elif ct == 'old': + # ignore attribute + continue + + else: + log("WARN", "attribute with unknown content_type: %s"%repr(att_elem)) + # ignore other content types + continue + + #if len(ov) > 0: + # # own value of relation is not useful + # attrs['ov'] = ov + + attrs[rel_type_attribute] = fixName(rel_name) + attrs['ismi_id'] = rel_id + log('DEBUG', "new edge(%s, %s, %s)"%(src_id, tar_id, attrs)) + # create relation with type + nx_rel = nx_graph.add_edge(src_id, tar_id, attr_dict=attrs) + + return nx_rel + + +def importEnts(ents_elem): + """Import all entities from etree element elem. + """ + cnt = 0 + xml_num = ents_elem.get('number') + log('INFO', "XML says %s entities"%xml_num) + + # iterate through entities element + for ent_elem in ents_elem: + cnt += 1 + ismi_id = ent_elem.get('id') + log('DEBUG', "reading entity[%s]"%ismi_id) + + if ismi_id in nx_nodes: + log("ERROR", "entity with id=%s exists!"%ismi_id) + return + + # create networkx node + node = nodeFromEnt(ent_elem) + + # save node reference + nx_nodes[ismi_id] = node + + # debug + #if cnt >= 100: + # return + + +def importRels(rels_elem): + """Import all entities from etree element elem. + """ + cnt = 0 + xml_num = rels_elem.get('number') + log('INFO', "XML says %s relations"%xml_num) + + # iterate through entities element + for rel_elem in rels_elem: + cnt += 1 + ismi_id = rel_elem.get('id') + log('DEBUG', "reading relation[%s]"%ismi_id) + + if ismi_id in nx_relations: + print("ERROR: relation with id=%s exists!"%ismi_id) + return + + # create networkx relation + relation = relationFromRel(rel_elem) + + # save relation reference + nx_relations[ismi_id] = relation + + # debug + #if cnt >= 100: + # return + + +def importAll(): + # parse XML file + log('INFO', "parsing XML file %s"%input_fn) + tree = ET.parse(input_fn) + log('DEBUG', "etree ready") + root = tree.getroot() + ents = root.find('entities') + importEnts(ents) + + rels = root.find('relations') + importRels(rels) + +## main + +print("Copy graph from OpenMind-XML to networkx pickle") + +# parse command line parameters +if len(sys.argv) > 1: + input_fn = sys.argv[1] + +if len(sys.argv) > 2: + output_fn = sys.argv[2] + +# import everything +print("Reading graph from OpenMind-XML file %s"%input_fn) +if len(exclude_objects_of_type) > 0: + print(" Skipping objects of type %s"%exclude_objects_of_type); + +importAll() + +print("Graph info: %s"%networkx.info(nx_graph)) +#print(" nodes:%s"%repr(nx_graph.nodes(data=True))) + +# export pickle +networkx.write_gpickle(nx_graph, output_fn) +print("Wrote networkx pickle file %s"%output_fn)