# HG changeset patch # User casties # Date 1443111461 -7200 # Node ID 5bdcb5805d29c69bf234373eab9d780911e61f35 # Parent 97f2da68fb5f40590a28b13b80e81b105b18ed3a updated openmind-networkx-neo4j conversion with dates, locations and links. diff -r 97f2da68fb5f -r 5bdcb5805d29 importFromOpenMind/importer/ismi2model.py --- a/importFromOpenMind/importer/ismi2model.py Wed Sep 23 19:47:02 2015 +0200 +++ b/importFromOpenMind/importer/ismi2model.py Thu Sep 24 18:17:41 2015 +0200 @@ -8,13 +8,12 @@ # output filename output_fn = "ismi_graph.gpickle" -# contract relations to these objects into attributes with the relations' name -#contract_relations_into_attributes = ['PLACE', 'ALIAS'] -contract_relations_into_attributes = [] - # OpenMind base URL baseURL="http://localhost:18080/ismi-richfaces/jsonInterface?" +# node types to exclude from the graph +exclude_objects_of_type = ['DIGITALIZATION', 'REFERENCE'] + entsURL=baseURL+"method=get_ents&oc=%s" @@ -111,7 +110,22 @@ # date attribute key = att['name'] val = att['ov'] - print("don't know what to do with date: %s=%s"%(key,val)) + # try to parse date object to get gregorian year + try: + year = None + date_json = json.loads(val) + if 'from' in date_json: + year = date_json['from'].get('year', None) + elif 'date' in date_json: + year = date_json['date'].get('year', None) + else: + print("don't know what to do with date on %s: %s=%s"%(ent['id'],key,val)) + + if year is not None: + attrs[key] = year + + except: + print("ERROR: invalid JSON in date: %s"%repr(val)) elif ct == 'old': # ignore attribute @@ -128,6 +142,11 @@ print("ERROR: entity type doesn't match!") return null + # rename if type attr exists + if 'type' in attrs: + attrs['type2'] = attrs['type'] + + # set type attrs['type'] = fixName(oc) ismi_id = ent['id'] @@ -154,6 +173,12 @@ # go through src_rels and tar_rels rels = ent.get('src_rels', []) + ent.get('tar_rels', []) for rel in rels: + src_type = rel['src_oc'] + tar_type = rel['tar_oc'] + if src_type in exclude_objects_of_type or tar_type in exclude_objects_of_type: + # skip relation to excluded objects + continue + rel_id = rel['id'] if rel_id in relations: old_rel = relations[rel_id] @@ -195,41 +220,6 @@ print("ERROR: relation %s tar node %s missing!"%(rel_id,tar_id)) continue - if contract_relations_into_attributes: - # contract source relations - tar_type = rel['tar_oc'] - if tar_type in contract_relations_into_attributes: - att_name = fixName(rel_name, att_from_rel=True) - # TODO: clean up attribute names - while src.get(att_name, None) is not None: - # attribute exists - if att_name[-1].isnumeric(): - # increment last digit - att_name = att_name[:-1] + str(int(att_name[-1]) + 1) - else: - att_name += '2' - - # add target node's label as attribute - #print("contracting tar to attribute %s on id=%s"%(att_name, src_id)) - nx_graph.node[src_id][att_name] = nx_graph.node[tar_id]['label'] - - # contract target relations - src_type = rel['src_oc'] - if src_type in contract_relations_into_attributes: - att_name = fixName(rel_name, att_from_rel=True) - # TODO: clean up attribute names - while tar.get(att_name, None) is not None: - # attribute exists - if att_name[-1].isnumeric(): - # increment last digit - att_name = att_name[:-1] + str(int(att_name[-1]) + 1) - else: - att_name += '2' - - # add target node's label as attribute - #print("contracting src to attribute %s on id=%s"%(att_name, tar_id)) - nx_graph.node[tar_id][att_name] = nx_graph.node[src_id]['label'] - # create relation with type nx_rel = nx_graph.add_edge(src_id, tar_id, type=fixName(rel_name), ismi_id=rel_id) @@ -267,7 +257,7 @@ print("ERROR: entity with id=%s exists!"%ismi_id) return - # create neo4j node + # create networkx node node = nodeFromEnt(ent_data, etype) # save node reference @@ -285,6 +275,10 @@ def importAllEnts(etypes): for etype in etypes: + if etype in exclude_objects_of_type: + # skip this type + continue + importEnts(etype) relationsFromRels(ismi_relations, nx_nodes) @@ -300,6 +294,9 @@ # import everything print("Reading graph from OpenMind at %s"%baseURL) +if len(exclude_objects_of_type) > 0: + print(" Skipping objects of type %s"%exclude_objects_of_type); + importAllEnts(ismi_defs) #importAllEnts(['TEXT']) diff -r 97f2da68fb5f -r 5bdcb5805d29 importFromOpenMind/importer/model2model.py --- a/importFromOpenMind/importer/model2model.py Wed Sep 23 19:47:02 2015 +0200 +++ b/importFromOpenMind/importer/model2model.py Thu Sep 24 18:17:41 2015 +0200 @@ -1,5 +1,6 @@ import networkx as nx import sys +import csv ## configure behaviour @@ -8,13 +9,24 @@ output_fn = 'ismi_graph_mod.gpickle' # operations -ops = ['contract', 'inv_rels'] +ops = ['locate', 'contract', 'inv_rels', 'add_links'] + +# types of object to locate +locate_objects_of_type = ['PLACE'] -# add relations to these objects as attributes with the relations name -contract_relations_into_attributes = {'PLACE': ['label'], +# file with place location information +places_fn = 'ismi_places_loc.csv' + +# node types to remove from the graph +#remove_objects_of_type = ['DIGITALIZATION', 'REFERENCE'] + +# add relations to these objects as attributes with the relation's name +contract_relations_into_attributes = {'PLACE': ['label', 'latitude', 'longitude'], 'ALIAS': ['label']} +# add URLs to nodes using an attribute in a pattern +add_link_attributes = {'ismi_id': 'https://ismi-dev.mpiwg-berlin.mpg.de/drupal-ismi/entity/%s'} def fixName(name, is_src_rel=False, is_tar_rel=False, att_from_rel=False): @@ -42,11 +54,158 @@ return name +def locatePlaces(nx_graph): + """add location information to objects in the graph""" + + print("Adding location information from %s to %s."%(places_fn, locate_objects_of_type)) + cnt = 0 + + # read place location file + locations = {} + with open(places_fn, encoding='utf-8') as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + lat = row['Latitude'] + lon = row['Longitude'] + name = row['Address'] + if lat and lon: + locations[name] = {'latitude': lat, 'longitude': lon} + + # iterate all nodes + for n in nx.nodes_iter(nx_graph): + attrs = nx_graph.node[n] + if attrs['type'] in locate_objects_of_type: + # locatable object + name = attrs['label'] + if name in locations: + # place name match + location = locations[name] + attrs['latitude'] = location['latitude'] + attrs['longitude'] = location['longitude'] + + else: + print("WARNING: no location for name '%s'"%name) + + cnt += 1 + if cnt % 100 == 0: + print(" %s nodes"%cnt) + + + +def genAttName(attrs, name): + """Generate new attribute name. + """ + while attrs.get(name, None) is not None: + # attribute exists + if name[-1].isnumeric(): # increment last digit + name = name[:-1] + str(int(name[-1]) + 1) + else: + name += '2' + + return name + + def contractRelations(nx_graph): """contract relations into attributes""" + print("Contracting relations to attributes.") + cnt = 0 + for nx_edge in nx.edges_iter(nx_graph): + (nx_src, nx_tar) = nx_edge + # get attributes of edge + rel_attrs = nx_graph.edge[nx_src][nx_tar][0] + rel_type = rel_attrs['type'] + # get attributes of source and target nodes + src_attrs = nx_graph.node[nx_src] + tar_attrs = nx_graph.node[nx_tar] + + # contract source relations + tar_type = tar_attrs['type'] + if tar_type in contract_relations_into_attributes: + # get list of attributes to transfer + transfer_atts = contract_relations_into_attributes[tar_type] + for transfer_att in transfer_atts: + if transfer_att not in tar_attrs: + # target has no attribute + continue + + # name for new attribute starts with relation name + att_name = fixName(rel_type, att_from_rel=True) + # then attribute name + if transfer_att != 'label': + att_name += "_%s"%transfer_att + + # then generate unique name + att_name = genAttName(src_attrs, att_name) + # add target node's attribute + src_attrs[att_name] = tar_attrs.get(transfer_att) + + # contract target relations + src_type = src_attrs['type'] + if src_type in contract_relations_into_attributes: + # get list of attributes to transfer + transfer_atts = contract_relations_into_attributes[src_type] + for transfer_att in transfer_atts: + if transfer_att not in src_attrs: + # target has no attribute + continue + + # name for new attribute starts with relation name + att_name = fixName(rel_type, att_from_rel=True) + # then attribute name + if transfer_att != 'label': + att_name += "_%s"%transfer_att + + # then generate unique name + att_name = genAttName(tar_attrs, att_name) + # add target node's attribute + tar_attrs[att_name] = src_attrs.get(transfer_att) + + cnt += 1 + if cnt % 100 == 0: + print(" %s relations"%cnt) + + +def invertRelations(nx_graph): + """Add inverse relations to each relation""" + print("Adding inverse relations.") + # copy list of edges because we add edges in the loop + edges = nx.edges(nx_graph)[:] + # iterate list + cnt = 0 + for nx_edge in edges: + (nx_src, nx_tar) = nx_edge + # get attributes of edge + rel_attrs = nx_graph.edge[nx_src][nx_tar][0] + rel_type = rel_attrs['type'] + rel_id = rel_attrs['ismi_id'] + # create new relation + nx_graph.add_edge(nx_tar, nx_src, type=fixName(rel_type, is_tar_rel=True), ismi_id=-rel_id) + + cnt += 1 + if cnt % 100 == 0: + print(" %s relations"%cnt) + +def addLinks(nx_graph): + """Add link attributes to all nodes.""" + + print("Adding links: %s"%repr(add_link_attributes)) + cnt = 0 + for link_att, link_pattern in add_link_attributes.items(): + # iterate all nodes + for n in nx.nodes_iter(nx_graph): + attrs = nx_graph.node[n] + if link_att in attrs: + url = link_pattern%attrs[link_att] + # TODO: which target attribute for multiple? + attrs['link'] = url + + cnt += 1 + if cnt % 100 == 0: + print(" %s nodes"%cnt) + ## main @@ -65,11 +224,19 @@ # operate for op in ops: - if op == 'contract': + if op == 'locate': + locatePlaces(nx_graph) + + elif op == 'contract': contractRelations(nx_graph) elif op == 'inv_rels': invertRelations(nx_graph) + elif op == 'add_links': + addLinks(nx_graph) + +print("Writing graph to %s"%output_fn) +nx_graph = nx.write_gpickle(nx_graph, output_fn) print("Done.") diff -r 97f2da68fb5f -r 5bdcb5805d29 importFromOpenMind/importer/model2neo4j.py --- a/importFromOpenMind/importer/model2neo4j.py Wed Sep 23 19:47:02 2015 +0200 +++ b/importFromOpenMind/importer/model2neo4j.py Thu Sep 24 18:17:41 2015 +0200 @@ -8,7 +8,7 @@ input_fn = 'ismi_graph.gpickle' # label added to all nodes -project_label = '_ismi4' +project_label = '_ismi6' # neo4j base URL neo4jBaseURL = "http://localhost:7474/db/data/" @@ -63,7 +63,7 @@ cnt += 1 if cnt % 100 == 0: - print(" %s"%cnt) + print(" %s nodes"%cnt) def copyRelations(nx_graph, n4j_graph): @@ -97,7 +97,7 @@ cnt += 1 if cnt % 100 == 0: - print(" %s"%cnt) + print(" %s relations"%cnt) ## main