Mercurial > hg > drupalISMI

annotate importFromOpenMind/importer/model2neo4j.py @ 34:74dfaed3600b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
keep relation attributes but no attributes from openmind. change relation type attribute to '_type' too.
author casties
date Tue, 02 Feb 2016 15:16:29 +0100
parents 7e2e344c3b87
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
1 import networkx as nx
29
1a1877812757 include normalized attributes in neo4j with prefix "_n_"
casties
parents: 28
diff changeset
2 from neo4jrestclient.client import GraphDatabase
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
3 import sys
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
4
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
5 ## configure behaviour
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
6
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
7 # metworkx graph file
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
8 input_fn = 'ismi_graph.gpickle'
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
9
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
10 # label added to all nodes
28
a9bfd49355f8 updated config for ismi-dev.
casties
parents: 25
diff changeset
11 project_label = '_ismi'
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
12
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
13 # neo4j base URL
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
14 neo4jBaseURL = "http://localhost:7474/db/data/"
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
15
33
7e2e344c3b87 make name of type attribute configurable. default '_type' for nodes.
casties
parents: 31
diff changeset
16 # name of type attribute
7e2e344c3b87 make name of type attribute configurable. default '_type' for nodes.
casties
parents: 31
diff changeset
17 node_type_attribute = '_type'
34
74dfaed3600b keep relation attributes but no attributes from openmind. change relation type attribute to '_type' too.
casties
parents: 33
diff changeset
18 rel_type_attribute = '_type'
33
7e2e344c3b87 make name of type attribute configurable. default '_type' for nodes.
casties
parents: 31
diff changeset
19
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
20
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
21 ## setup
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
22
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
23 n4j_nodes = {}
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
24 n4j_relations = {}
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
25
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
26
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
27 def copyNodes(nx_graph, n4j_graph):
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
28 """copy all nodes from nx_graph to n4j_graph"""
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
29
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
30 print("Copying nodes to Neo4J")
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
31 cnt = 0
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
32 for node_id in nx.nodes_iter(nx_graph):
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
33 attrs = nx_graph.node[node_id]
33
7e2e344c3b87 make name of type attribute configurable. default '_type' for nodes.
casties
parents: 31
diff changeset
34 ntype = attrs[node_type_attribute]
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
35 ismi_id = attrs['ismi_id']
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
36 # create node with attributes
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
37 n4j_node = n4j_graph.nodes.create(**attrs)
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
38 # add labels
29
1a1877812757 include normalized attributes in neo4j with prefix "_n_"
casties
parents: 28
diff changeset
39 n4j_node.labels.add([project_label, ntype])
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
40 # save reference
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
41 n4j_nodes[ismi_id] = n4j_node
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
42
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
43 cnt += 1
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
44 if cnt % 100 == 0:
25
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
45 print(" %s nodes"%cnt)
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
46
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
47
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
48 def copyRelations(nx_graph, n4j_graph):
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
49 """copy all relations from nx_graph to n4j_graph"""
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
50
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
51 print("Copying relations to Neo4J")
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
52 cnt = 0
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
53 for nx_edge in nx.edges_iter(nx_graph):
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
54 (nx_src, nx_tar) = nx_edge
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
55 # get attributes of edge
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
56 attrs = nx_graph.edge[nx_src][nx_tar][0]
33
7e2e344c3b87 make name of type attribute configurable. default '_type' for nodes.
casties
parents: 31
diff changeset
57 etype = attrs[rel_type_attribute]
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
58 # get ismi_id of source and target nodes
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
59 src_id = nx_graph.node[nx_src]['ismi_id']
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
60 tar_id = nx_graph.node[nx_tar]['ismi_id']
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
61 # get Neo4J nodes
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
62 src = n4j_nodes.get(src_id, None)
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
63 if src is None:
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
64 print("ERROR: src node %s missing!"%src_id)
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
65 break
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
66
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
67 tar = n4j_nodes.get(tar_id, None)
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
68 if tar is None:
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
69 print("ERROR: tar node %s missing!"%tar_id)
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
70 break
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
71
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
72 # create Neo4J relation
33
7e2e344c3b87 make name of type attribute configurable. default '_type' for nodes.
casties
parents: 31
diff changeset
73 n4j_rel = n4j_graph.relationships.create(src, etype, tar)
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
74 # add attributes
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
75 n4j_rel.properties = attrs
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
76
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
77 cnt += 1
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
78 if cnt % 100 == 0:
25
5bdcb5805d29 updated openmind-networkx-neo4j conversion with dates, locations and links.
casties
parents: 23
diff changeset
79 print(" %s relations"%cnt)
23
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
80
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
81
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
82 ## main
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
83
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
84 print("Copy graph from networkx to Neo4J")
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
85
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
86 # read commandline parameters
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
87 if len(sys.argv) > 1:
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
88 input_fn = sys.argv[1]
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
89
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
90 # read networkx graph from pickle
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
91 print("Reading graph from %s"%input_fn)
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
92 nx_graph = nx.read_gpickle(input_fn)
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
93 print("Graph info: %s"%nx.info(nx_graph))
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
94
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
95 # open neo4j graph db
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
96 print("Opening Neo4J db at %s"%neo4jBaseURL)
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
97 n4j_graph = GraphDatabase(neo4jBaseURL, username="neo4j", password="neo5j")
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
98
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
99 copyNodes(nx_graph, n4j_graph)
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
100
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
101 copyRelations(nx_graph, n4j_graph)
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
102
45a823b5bf33 updated ismi2model importer and model2neo4j exporter.
casties
parents:
diff changeset
103 print("Done.")