Mercurial > hg > drupalISMI

annotate importFromOpenMind/importer/ismi2model.py @ 24:97f2da68fb5f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first version of model2model graph manipulation tool. doesn't work yet.
author casties
date Wed, 23 Sep 2015 19:47:02 +0200
parents 45a823b5bf33
children 5bdcb5805d29
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
1 import urllib.request
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
2 import json
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
3 import networkx
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
4 import sys
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
5
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
6 ## configure behaviour
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
7
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
8 # output filename
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
9 output_fn = "ismi_graph.gpickle"
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
10
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
11 # contract relations to these objects into attributes with the relations' name
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
12 #contract_relations_into_attributes = ['PLACE', 'ALIAS']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
13 contract_relations_into_attributes = []
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
14
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
15 # OpenMind base URL
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
16 baseURL="http://localhost:18080/ismi-richfaces/jsonInterface?"
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
17
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
18
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
19 entsURL=baseURL+"method=get_ents&oc=%s"
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
20
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
21 entsByIdURL = baseURL+"method=get_ents&include_content=True&ids=%s"
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
22
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
23 entURL=baseURL+"method=get_ent&id=%s&include_content=True"
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
24
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
25
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
26 def readJSON(url):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
27 #print("JSON loading %s"%url)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
28 wsh=urllib.request.urlopen(url)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
29 txt = wsh.read()
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
30 return json.loads(txt.decode("utf-8"))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
31
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
32 defs_json = readJSON(baseURL+"method=get_defs")
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
33
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
34 # current list of all definitions
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
35 ismi_defs = [atts['ov'] for atts in defs_json['defs']]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
36
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
37 #ismi_types=["PERSON","WITNESS","CODEX","PLACE","COLLECTION","REPOSITORY"]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
38
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
39
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
40 nx_graph = networkx.MultiDiGraph()
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
41
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
42 nx_nodes = {}
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
43 ismi_relations = {}
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
44 nx_relations = {}
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
45
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
46 ent_exclude_attrs = [
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
47 'lw',
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
48 'node_type',
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
49 'nov'
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
50 ]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
51
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
52 def fixName(name, is_src_rel=False, is_tar_rel=False, att_from_rel=False):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
53 # these are too embarrassing...
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
54 if 'FLORUIT' in name:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
55 name = name.replace('FLORUIT', 'FLOURISH')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
56
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
57 elif 'floruit' in name:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
58 name = name.replace('floruit', 'flourish')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
59
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
60 if is_src_rel:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
61 #name = name + '>'
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
62 pass
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
63
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
64 if is_tar_rel:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
65 name = '<' + name
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
66
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
67 if att_from_rel:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
68 # clean up relations as attribute names
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
69 name = name.replace('is_', '')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
70 name = name.replace('has_', '')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
71 name = name.replace('was_', '')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
72 name = name.replace('_of', '')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
73
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
74 return name
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
75
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
76
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
77 def nodeFromEnt(ent, etype):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
78 """Create a Neo4J node from the given JSON entity.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
79
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
80 Creates the node in gdb and returns the node.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
81 """
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
82 attrs = {}
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
83 # go through all attributes
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
84 for att in ent['atts']:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
85 ct = att.get('content_type', None)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
86 if ct is None or ct.lower() in ['text', 'arabic', 'bool', 'boolean', 'url', 'language']:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
87 # normal text attribute (assume no content_type is text too...)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
88 key = att['name']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
89 val = att['ov']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
90
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
91 if key in ent_exclude_attrs:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
92 # exclude attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
93 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
94
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
95 # keep attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
96 attrs[key] = val
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
97
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
98 elif ct == 'num':
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
99 # number attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
100 key = att['name']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
101 val = att['ov']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
102
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
103 if key in ent_exclude_attrs:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
104 # exclude attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
105 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
106
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
107 # keep attribute, assume num is int
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
108 attrs[key] = int(val)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
109
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
110 elif ct == 'date':
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
111 # date attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
112 key = att['name']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
113 val = att['ov']
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
114 print("don't know what to do with date: %s=%s"%(key,val))
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
115
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
116 elif ct == 'old':
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
117 # ignore attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
118 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
119
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
120 else:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
121 print("WARN: attribute with unknown content_type: %s"%repr(att))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
122 # ignore other content types
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
123 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
124
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
125 # process base attributes
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
126 oc = ent['oc']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
127 if oc != etype:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
128 print("ERROR: entity type doesn't match!")
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
129 return null
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
130
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
131 attrs['type'] = fixName(oc)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
132
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
133 ismi_id = ent['id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
134 # rename id to ismi_id
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
135 attrs['ismi_id'] = ismi_id
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
136
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
137 ov = ent.get('ov', None)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
138 if ov is not None:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
139 # save ov as label
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
140 attrs['label'] = ov
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
141
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
142 # create node with attributes
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
143 nx_graph.add_node(ismi_id, **attrs)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
144 node = nx_graph.node[ismi_id]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
145
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
146 return node
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
147
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
148
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
149 def relsFromEnt(ent, relations):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
150 """Extract all relations from JSON entity.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
151
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
152 Adds JSON to dict relations under relation's id.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
153 """
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
154 # go through src_rels and tar_rels
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
155 rels = ent.get('src_rels', []) + ent.get('tar_rels', [])
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
156 for rel in rels:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
157 rel_id = rel['id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
158 if rel_id in relations:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
159 old_rel = relations[rel_id]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
160 if rel != old_rel:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
161 print("ERROR: relation is different: %s != %s"%(repr(rel), repr(old_rel)))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
162 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
163
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
164 relations[rel_id] = rel
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
165
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
166 return relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
167
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
168
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
169 def relationsFromRels(rels, nodes):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
170 """Create relations in Neo4J.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
171
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
172 Args:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
173 rels: dict of JSON relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
174 nodes: dict of existing Neo4J nodes
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
175 Returns:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
176 dict of Neo4J relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
177 """
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
178 # go through all rels
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
179 print("importing %s relations"%len(rels))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
180 cnt = 0
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
181 for rel in rels.values():
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
182 cnt += 1
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
183 if cnt % 100 == 0:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
184 print(" %s relations"%cnt)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
185
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
186 rel_id = rel['id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
187 rel_name = rel['name']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
188 src_id = rel['src_id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
189 tar_id = rel['tar_id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
190 if not src_id in nodes:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
191 print("ERROR: relation %s src node %s missing!"%(rel_id,src_id))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
192 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
193
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
194 if not tar_id in nodes:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
195 print("ERROR: relation %s tar node %s missing!"%(rel_id,tar_id))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
196 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
197
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
198 if contract_relations_into_attributes:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
199 # contract source relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
200 tar_type = rel['tar_oc']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
201 if tar_type in contract_relations_into_attributes:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
202 att_name = fixName(rel_name, att_from_rel=True)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
203 # TODO: clean up attribute names
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
204 while src.get(att_name, None) is not None:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
205 # attribute exists
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
206 if att_name[-1].isnumeric():
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
207 # increment last digit
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
208 att_name = att_name[:-1] + str(int(att_name[-1]) + 1)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
209 else:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
210 att_name += '2'
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
211
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
212 # add target node's label as attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
213 #print("contracting tar to attribute %s on id=%s"%(att_name, src_id))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
214 nx_graph.node[src_id][att_name] = nx_graph.node[tar_id]['label']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
215
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
216 # contract target relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
217 src_type = rel['src_oc']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
218 if src_type in contract_relations_into_attributes:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
219 att_name = fixName(rel_name, att_from_rel=True)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
220 # TODO: clean up attribute names
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
221 while tar.get(att_name, None) is not None:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
222 # attribute exists
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
223 if att_name[-1].isnumeric():
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
224 # increment last digit
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
225 att_name = att_name[:-1] + str(int(att_name[-1]) + 1)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
226 else:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
227 att_name += '2'
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
228
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
229 # add target node's label as attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
230 #print("contracting src to attribute %s on id=%s"%(att_name, tar_id))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
231 nx_graph.node[tar_id][att_name] = nx_graph.node[src_id]['label']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
232
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
233 # create relation with type
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
234 nx_rel = nx_graph.add_edge(src_id, tar_id, type=fixName(rel_name), ismi_id=rel_id)
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
235
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
236 nx_relations[rel_id] = nx_rel
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
237
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
238 return nx_relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
239
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
240
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
241 def importEnts(etype):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
242 """Import all entities of the given type.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
243 """
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
244 # read json for all entities of given type
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
245 json = readJSON(entsURL%etype)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
246 ents = json['ents']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
247 print("importing %s %ss"%(len(ents),etype))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
248 size = 100
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
249 batches = [ents[pos:pos + size] for pos in range(0, len(ents), size)]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
250 cnt = 0
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
251 for batch in batches:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
252 cnt += size
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
253 if cnt % 100 == 0:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
254 print(" %s %ss"%(cnt, etype))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
255
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
256 # extract list of ismi ids
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
257 ismi_ids = [str(ent['id']) for ent in batch]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
258
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
259 # fetch full data for list of entities
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
260 ent_json = readJSON(entsByIdURL%','.join(ismi_ids))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
261 ents_data = ent_json['ents']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
262
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
263 # iterate through results batch
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
264 for ent_data in ents_data:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
265 ismi_id = ent_data['id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
266 if ismi_id in nx_nodes:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
267 print("ERROR: entity with id=%s exists!"%ismi_id)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
268 return
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
269
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
270 # create neo4j node
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
271 node = nodeFromEnt(ent_data, etype)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
272
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
273 # save node reference
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
274 nx_nodes[ismi_id] = node
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
275
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
276 # extract relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
277 relsFromEnt(ent_data, ismi_relations)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
278
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
279 #if cnt >= 100:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
280 # return
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
281
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
282
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
283 # In[119]:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
284
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
285 def importAllEnts(etypes):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
286
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
287 for etype in etypes:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
288 importEnts(etype)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
289
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
290 relationsFromRels(ismi_relations, nx_nodes)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
291
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
292
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
293 ## main
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
294
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
295 print("Copy graph from OpenMind to networkx pickle")
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
296
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
297 # parse command line parameters
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
298 if len(sys.argv) > 1:
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
299 output_fn = sys.argv[1]
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
300
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
301 # import everything
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
302 print("Reading graph from OpenMind at %s"%baseURL)
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
303 importAllEnts(ismi_defs)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
304 #importAllEnts(['TEXT'])
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
305
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
306 print("Graph info: %s"%networkx.info(nx_graph))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
307 #print(" nodes:%s"%repr(nx_graph.nodes(data=True)))
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
308
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
309 # export pickle
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
310 networkx.write_gpickle(nx_graph, output_fn)
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
311 print("Wrote networkx pickle file %s"%output_fn)