Mercurial > hg > drupalISMI

annotate importFromOpenMind/importer/ismi2model.py @ 32:ce12475d2109

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix bug with normalized type attribute.
author casties
date Fri, 22 Jan 2016 20:41:25 +0100
parents 1a1877812757
children 7e2e344c3b87
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
1 import urllib.request
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
2 import json
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
3 import networkx
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
4 import sys
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
5
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
6 ## configure behaviour
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
7
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
8 # output filename
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
9 output_fn = "ismi_graph.gpickle"
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
10
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
11 # OpenMind base URL
28
a9bfd49355f8 updated config for ismi-dev.
casties
parents: 27
diff changeset
12 baseURL="http://localhost:18080/om4-ismi/jsonInterface?"
32
ce12475d2109 fix bug with normalized type attribute.
casties
parents: 29
diff changeset
13 #baseURL="http://localhost:18080/ismi-richfaces/jsonInterface?"
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
14
25
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
15 # node types to exclude from the graph
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
16 exclude_objects_of_type = ['DIGITALIZATION', 'REFERENCE']
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
17
27
3fce3fa9097e ignore notes_old.
casties
parents: 25
diff changeset
18 # attributes to exclude
3fce3fa9097e ignore notes_old.
casties
parents: 25
diff changeset
19 exclude_attributes_of_type = [
3fce3fa9097e ignore notes_old.
casties
parents: 25
diff changeset
20 'lw',
3fce3fa9097e ignore notes_old.
casties
parents: 25
diff changeset
21 'node_type',
3fce3fa9097e ignore notes_old.
casties
parents: 25
diff changeset
22 'nov',
3fce3fa9097e ignore notes_old.
casties
parents: 25
diff changeset
23 'notes_old'
3fce3fa9097e ignore notes_old.
casties
parents: 25
diff changeset
24 ]
3fce3fa9097e ignore notes_old.
casties
parents: 25
diff changeset
25
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
26
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
27 entsURL=baseURL+"method=get_ents&oc=%s"
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
28
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
29 entsByIdURL = baseURL+"method=get_ents&include_content=True&ids=%s"
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
30
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
31 entURL=baseURL+"method=get_ent&id=%s&include_content=True"
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
32
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
33
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
34 def readJSON(url):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
35 #print("JSON loading %s"%url)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
36 wsh=urllib.request.urlopen(url)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
37 txt = wsh.read()
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
38 return json.loads(txt.decode("utf-8"))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
39
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
40 defs_json = readJSON(baseURL+"method=get_defs")
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
41
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
42 # current list of all definitions
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
43 ismi_defs = [atts['ov'] for atts in defs_json['defs']]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
44
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
45 #ismi_types=["PERSON","WITNESS","CODEX","PLACE","COLLECTION","REPOSITORY"]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
46
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
47
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
48 nx_graph = networkx.MultiDiGraph()
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
49
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
50 nx_nodes = {}
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
51 ismi_relations = {}
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
52 nx_relations = {}
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
53
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
54
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
55 def fixName(name, is_src_rel=False, is_tar_rel=False, att_from_rel=False):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
56 # these are too embarrassing...
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
57 if 'FLORUIT' in name:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
58 name = name.replace('FLORUIT', 'FLOURISH')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
59
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
60 elif 'floruit' in name:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
61 name = name.replace('floruit', 'flourish')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
62
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
63 if is_src_rel:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
64 #name = name + '>'
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
65 pass
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
66
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
67 if is_tar_rel:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
68 name = '<' + name
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
69
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
70 if att_from_rel:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
71 # clean up relations as attribute names
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
72 name = name.replace('is_', '')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
73 name = name.replace('has_', '')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
74 name = name.replace('was_', '')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
75 name = name.replace('_of', '')
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
76
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
77 return name
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
78
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
79
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
80 def nodeFromEnt(ent, etype):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
81 """Create a Neo4J node from the given JSON entity.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
82
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
83 Creates the node in gdb and returns the node.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
84 """
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
85 attrs = {}
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
86 # go through all attributes
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
87 for att in ent['atts']:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
88 ct = att.get('content_type', None)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
89 if ct is None or ct.lower() in ['text', 'arabic', 'bool', 'boolean', 'url', 'language']:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
90 # normal text attribute (assume no content_type is text too...)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
91 key = att['name']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
92 val = att['ov']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
93
27
3fce3fa9097e ignore notes_old.
casties
parents: 25
diff changeset
94 if key in exclude_attributes_of_type:
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
95 # exclude attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
96 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
97
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
98 # keep attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
99 attrs[key] = val
29
1a1877812757 include normalized attributes in neo4j with prefix "_n_"
casties
parents: 28
diff changeset
100 if 'nov' in att:
1a1877812757 include normalized attributes in neo4j with prefix "_n_"
casties
parents: 28
diff changeset
101 # add normalized value
1a1877812757 include normalized attributes in neo4j with prefix "_n_"
casties
parents: 28
diff changeset
102 attrs['_n_'+key] = att['nov']
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
103
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
104 elif ct == 'num':
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
105 # number attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
106 key = att['name']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
107 val = att['ov']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
108
27
3fce3fa9097e ignore notes_old.
casties
parents: 25
diff changeset
109 if key in exclude_attributes_of_type:
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
110 # exclude attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
111 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
112
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
113 # keep attribute, assume num is int
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
114 attrs[key] = int(val)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
115
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
116 elif ct == 'date':
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
117 # date attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
118 key = att['name']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
119 val = att['ov']
25
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
120 # try to parse date object to get gregorian year
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
121 try:
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
122 year = None
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
123 date_json = json.loads(val)
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
124 if 'from' in date_json:
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
125 year = date_json['from'].get('year', None)
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
126 elif 'date' in date_json:
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
127 year = date_json['date'].get('year', None)
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
128 else:
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
129 print("don't know what to do with date on %s: %s=%s"%(ent['id'],key,val))
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
130
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
131 if year is not None:
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
132 attrs[key] = year
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
133
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
134 except:
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
135 print("ERROR: invalid JSON in date: %s"%repr(val))
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
136
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
137 elif ct == 'old':
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
138 # ignore attribute
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
139 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
140
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
141 else:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
142 print("WARN: attribute with unknown content_type: %s"%repr(att))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
143 # ignore other content types
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
144 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
145
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
146 # process base attributes
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
147 oc = ent['oc']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
148 if oc != etype:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
149 print("ERROR: entity type doesn't match!")
29
1a1877812757 include normalized attributes in neo4j with prefix "_n_"
casties
parents: 28
diff changeset
150 return None
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
151
25
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
152 # rename if type attr exists
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
153 if 'type' in attrs:
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
154 attrs['type2'] = attrs['type']
32
ce12475d2109 fix bug with normalized type attribute.
casties
parents: 29
diff changeset
155 if '_n_type' in attrs:
ce12475d2109 fix bug with normalized type attribute.
casties
parents: 29
diff changeset
156 # rename normalized attribute
ce12475d2109 fix bug with normalized type attribute.
casties
parents: 29
diff changeset
157 attrs['_n_type2'] = attrs['_n_type']
ce12475d2109 fix bug with normalized type attribute.
casties
parents: 29
diff changeset
158 del attrs['n_type']
25
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
159
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
160 # set type
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
161 attrs['type'] = fixName(oc)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
162
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
163 ismi_id = ent['id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
164 # rename id to ismi_id
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
165 attrs['ismi_id'] = ismi_id
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
166
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
167 ov = ent.get('ov', None)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
168 if ov is not None:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
169 # save ov as label
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
170 attrs['label'] = ov
29
1a1877812757 include normalized attributes in neo4j with prefix "_n_"
casties
parents: 28
diff changeset
171 if 'nov' in ent:
1a1877812757 include normalized attributes in neo4j with prefix "_n_"
casties
parents: 28
diff changeset
172 # add normalized value
1a1877812757 include normalized attributes in neo4j with prefix "_n_"
casties
parents: 28
diff changeset
173 attrs['_n_label'] = ent.get('nov')
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
174
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
175 nx_graph.add_node(ismi_id, **attrs)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
176 node = nx_graph.node[ismi_id]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
177
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
178 return node
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
179
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
180
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
181 def relsFromEnt(ent, relations):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
182 """Extract all relations from JSON entity.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
183
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
184 Adds JSON to dict relations under relation's id.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
185 """
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
186 # go through src_rels and tar_rels
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
187 rels = ent.get('src_rels', []) + ent.get('tar_rels', [])
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
188 for rel in rels:
25
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
189 src_type = rel['src_oc']
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
190 tar_type = rel['tar_oc']
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
191 if src_type in exclude_objects_of_type or tar_type in exclude_objects_of_type:
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
192 # skip relation to excluded objects
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
193 continue
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
194
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
195 rel_id = rel['id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
196 if rel_id in relations:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
197 old_rel = relations[rel_id]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
198 if rel != old_rel:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
199 print("ERROR: relation is different: %s != %s"%(repr(rel), repr(old_rel)))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
200 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
201
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
202 relations[rel_id] = rel
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
203
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
204 return relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
205
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
206
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
207 def relationsFromRels(rels, nodes):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
208 """Create relations in Neo4J.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
209
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
210 Args:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
211 rels: dict of JSON relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
212 nodes: dict of existing Neo4J nodes
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
213 Returns:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
214 dict of Neo4J relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
215 """
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
216 # go through all rels
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
217 print("importing %s relations"%len(rels))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
218 cnt = 0
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
219 for rel in rels.values():
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
220 cnt += 1
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
221 if cnt % 100 == 0:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
222 print(" %s relations"%cnt)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
223
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
224 rel_id = rel['id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
225 rel_name = rel['name']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
226 src_id = rel['src_id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
227 tar_id = rel['tar_id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
228 if not src_id in nodes:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
229 print("ERROR: relation %s src node %s missing!"%(rel_id,src_id))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
230 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
231
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
232 if not tar_id in nodes:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
233 print("ERROR: relation %s tar node %s missing!"%(rel_id,tar_id))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
234 continue
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
235
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
236 # create relation with type
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
237 nx_rel = nx_graph.add_edge(src_id, tar_id, type=fixName(rel_name), ismi_id=rel_id)
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
238
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
239 nx_relations[rel_id] = nx_rel
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
240
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
241 return nx_relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
242
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
243
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
244 def importEnts(etype):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
245 """Import all entities of the given type.
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
246 """
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
247 # read json for all entities of given type
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
248 json = readJSON(entsURL%etype)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
249 ents = json['ents']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
250 print("importing %s %ss"%(len(ents),etype))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
251 size = 100
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
252 batches = [ents[pos:pos + size] for pos in range(0, len(ents), size)]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
253 cnt = 0
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
254 for batch in batches:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
255 cnt += size
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
256 if cnt % 100 == 0:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
257 print(" %s %ss"%(cnt, etype))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
258
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
259 # extract list of ismi ids
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
260 ismi_ids = [str(ent['id']) for ent in batch]
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
261
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
262 # fetch full data for list of entities
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
263 ent_json = readJSON(entsByIdURL%','.join(ismi_ids))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
264 ents_data = ent_json['ents']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
265
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
266 # iterate through results batch
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
267 for ent_data in ents_data:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
268 ismi_id = ent_data['id']
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
269 if ismi_id in nx_nodes:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
270 print("ERROR: entity with id=%s exists!"%ismi_id)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
271 return
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
272
25
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
273 # create networkx node
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
274 node = nodeFromEnt(ent_data, etype)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
275
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
276 # save node reference
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
277 nx_nodes[ismi_id] = node
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
278
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
279 # extract relations
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
280 relsFromEnt(ent_data, ismi_relations)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
281
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
282 #if cnt >= 100:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
283 # return
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
284
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
285
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
286 # In[119]:
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
287
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
288 def importAllEnts(etypes):
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
289
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
290 for etype in etypes:
25
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
291 if etype in exclude_objects_of_type:
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
292 # skip this type
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
293 continue
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
294
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
295 importEnts(etype)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
296
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
297 relationsFromRels(ismi_relations, nx_nodes)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
298
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
299
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
300 ## main
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
301
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
302 print("Copy graph from OpenMind to networkx pickle")
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
303
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
304 # parse command line parameters
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
305 if len(sys.argv) > 1:
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
306 output_fn = sys.argv[1]
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
307
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
308 # import everything
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
309 print("Reading graph from OpenMind at %s"%baseURL)
25
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
310 if len(exclude_objects_of_type) > 0:
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
311 print(" Skipping objects of type %s"%exclude_objects_of_type);
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
312
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
313 importAllEnts(ismi_defs)
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
314 #importAllEnts(['TEXT'])
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
315
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
316 print("Graph info: %s"%networkx.info(nx_graph))
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
317 #print(" nodes:%s"%repr(nx_graph.nodes(data=True)))
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
318
19
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
319 # export pickle
ca1e02a2a9c4 unfilteredIsmi: openmind to json exporter like filterISMI.
casties
parents:
diff changeset
320 networkx.write_gpickle(nx_graph, output_fn)
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents: 19
diff changeset
321 print("Wrote networkx pickle file %s"%output_fn)