# HG changeset patch
# User casties
# Date 1477592479 -7200
# Node ID 2d669fdd0a8be652440289539f4dc92b25325f5e
# Parent 69c233694e765adfa25b7c0a6989059862962e66
New CsvNodeListReader for import-merge.
diff -r 69c233694e76 -r 2d669fdd0a8b pom.xml
--- a/pom.xml Tue Oct 25 18:42:05 2016 +0200
+++ b/pom.xml Thu Oct 27 20:21:19 2016 +0200
@@ -163,7 +163,17 @@
2.5
provided
-->
-
+
+
+ org.apache.commons
+ commons-csv
+ 1.4
+
+
+ commons-codec
+ commons-codec
+ 1.10
+
diff -r 69c233694e76 -r 2d669fdd0a8b src/main/java/org/mpi/openmind/repository/bo/Attribute.java
--- a/src/main/java/org/mpi/openmind/repository/bo/Attribute.java Tue Oct 25 18:42:05 2016 +0200
+++ b/src/main/java/org/mpi/openmind/repository/bo/Attribute.java Thu Oct 27 20:21:19 2016 +0200
@@ -246,4 +246,9 @@
}
return "!!!ENCODING-ERROR!!!";
}
+
+ @Override
+ public String getNodeType() {
+ return "ATTRIBUTE";
+ }
}
diff -r 69c233694e76 -r 2d669fdd0a8b src/main/java/org/mpi/openmind/repository/bo/Entity.java
--- a/src/main/java/org/mpi/openmind/repository/bo/Entity.java Tue Oct 25 18:42:05 2016 +0200
+++ b/src/main/java/org/mpi/openmind/repository/bo/Entity.java Thu Oct 27 20:21:19 2016 +0200
@@ -1286,4 +1286,9 @@
}
}
+
+ @Override
+ public String getNodeType() {
+ return "ENTITY";
+ }
}
diff -r 69c233694e76 -r 2d669fdd0a8b src/main/java/org/mpi/openmind/repository/bo/Node.java
--- a/src/main/java/org/mpi/openmind/repository/bo/Node.java Tue Oct 25 18:42:05 2016 +0200
+++ b/src/main/java/org/mpi/openmind/repository/bo/Node.java Thu Oct 27 20:21:19 2016 +0200
@@ -400,4 +400,7 @@
return "!!!ENCODING-ERROR!!!";
}
+ public String getNodeType() {
+ return "NODE";
+ }
}
diff -r 69c233694e76 -r 2d669fdd0a8b src/main/java/org/mpi/openmind/repository/bo/Relation.java
--- a/src/main/java/org/mpi/openmind/repository/bo/Relation.java Tue Oct 25 18:42:05 2016 +0200
+++ b/src/main/java/org/mpi/openmind/repository/bo/Relation.java Thu Oct 27 20:21:19 2016 +0200
@@ -317,4 +317,10 @@
this.getOwnValue(),
this.getId());
}
+
+ @Override
+ public String getNodeType() {
+ return "RELATION";
+ }
+
}
diff -r 69c233694e76 -r 2d669fdd0a8b src/main/java/org/mpi/openmind/repository/utils/CsvNodeListReader.java
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/org/mpi/openmind/repository/utils/CsvNodeListReader.java Thu Oct 27 20:21:19 2016 +0200
@@ -0,0 +1,209 @@
+/**
+ *
+ */
+package org.mpi.openmind.repository.utils;
+
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.StringUtils;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVRecord;
+import org.mpi.openmind.repository.bo.Attribute;
+import org.mpi.openmind.repository.bo.Entity;
+import org.mpi.openmind.repository.bo.Node;
+import org.mpi.openmind.repository.bo.Relation;
+
+/**
+ * Class that reads a CSV file into a list of Nodes.
+ *
+ * @author casties
+ *
+ */
+public class CsvNodeListReader {
+
+ public static List readCsv(Reader input) throws Exception {
+ ArrayList nodeList = new ArrayList();
+ HashMap entMap = new HashMap();
+ HashMap relMap = new HashMap();
+ // parse all records from CSV file
+ for (CSVRecord csvRecord : CSVFormat.DEFAULT.withHeader().parse(input)) {
+ // will throw exception if node_type is missing
+ String node_type = csvRecord.get("node_type");
+ Node node = null;
+
+ if (node_type.equals("ENTITY")) {
+ //System.out.println("Entity: "+csvRecord);
+ Entity ent = new Entity();
+ // TODO: is lightweight ok if we want to add attributes?
+ ent.setLightweight(false);
+ if (csvRecord.isSet("id")) {
+ Long id = Long.decode(csvRecord.get("id"));
+ ent.setId(id);
+ entMap.put(id, ent);
+ }
+ nodeList.add(ent);
+ node = ent;
+
+ } else if (node_type.equals("ATTRIBUTE")) {
+ //System.out.println("Attribute: "+csvRecord);
+ Attribute att = new Attribute();
+ // throws exception if no source_id
+ Long source_id = Long.decode(csvRecord.get("source_id"));
+ att.setSourceId(source_id);
+ // save Attribute
+ nodeList.add(att);
+ // set optional properties
+ if (csvRecord.isSet("id")) {
+ Long id = Long.decode(csvRecord.get("id"));
+ att.setId(id);
+ }
+ if (csvRecord.isSet("source_modif")) {
+ Long mtime = Long.decode(csvRecord.get("source_modif"));
+ att.setSourceModif(mtime);
+ }
+ if (csvRecord.isSet("source_obj_class")) {
+ String oc = csvRecord.get("source_obj_class");
+ att.setSourceObjectClass(oc);
+ }
+ node = att;
+
+ } else if (node_type.equals("RELATION")) {
+ //System.out.println("Relation: "+csvRecord);
+ Relation rel = new Relation();
+ if (csvRecord.isSet("id")) {
+ Long id = Long.decode(csvRecord.get("id"));
+ rel.setId(id);
+ relMap.put(id, rel);
+ }
+ // throws exception if no source_id or target_id
+ Long source_id = Long.decode(csvRecord.get("source_id"));
+ rel.setSourceId(source_id);
+ Long target_id = Long.decode(csvRecord.get("target_id"));
+ rel.setTargetId(target_id);
+ // set optional properties
+ if (csvRecord.isSet("source_modif")) {
+ Long mtime = Long.decode(csvRecord.get("source_modif"));
+ rel.setSourceModif(mtime);
+ }
+ if (csvRecord.isSet("source_obj_class")) {
+ String oc = csvRecord.get("source_obj_class");
+ rel.setSourceObjectClass(oc);
+ }
+ if (csvRecord.isSet("target_modif")) {
+ Long mtime = Long.decode(csvRecord.get("target_modif"));
+ rel.setTargetModif(mtime);
+ }
+ if (csvRecord.isSet("target_obj_class")) {
+ String oc = csvRecord.get("target_obj_class");
+ rel.setTargetObjectClass(oc);
+ }
+ nodeList.add(rel);
+ node = rel;
+
+ } else {
+ // unknown node_type
+ throw new Exception("Unknown node_type "+node_type);
+ }
+
+ /*
+ * set common node properties
+ */
+ if (csvRecord.isSet("row_id")) {
+ Long row_id = Long.decode(csvRecord.get("row_id"));
+ // TODO: should we really set row_id?
+ node.setRowId(row_id);
+ }
+ if (csvRecord.isSet("object_class")) {
+ String oc = csvRecord.get("object_class");
+ node.setObjectClass(oc);
+ }
+ if (csvRecord.isSet("user")) {
+ String user = csvRecord.get("user");
+ node.setUser(user);
+ }
+ if (csvRecord.isSet("public")) {
+ boolean pub = csvRecord.get("public").equals("true");
+ node.setIsPublic(pub);
+ }
+ if (csvRecord.isSet("type")) {
+ String type = csvRecord.get("type");
+ node.setType(type);
+ }
+ if (csvRecord.isSet("version")) {
+ Long version = Long.decode(csvRecord.get("version"));
+ node.setVersion(version);
+ }
+ if (csvRecord.isSet("type")) {
+ String type = csvRecord.get("type");
+ node.setType(type);
+ }
+ if (csvRecord.isSet("modification_time")) {
+ Long mtime = Long.decode(csvRecord.get("modification_time"));
+ node.setModificationTime(mtime);
+ }
+ if (csvRecord.isSet("system_status")) {
+ String status = csvRecord.get("system_status");
+ node.setSystemStatus(status);
+ }
+ if (csvRecord.isSet("content_type")) {
+ String ct = csvRecord.get("content_type");
+ node.setContentType(ct);
+ }
+ if (csvRecord.isSet("own_value_b64")) {
+ // get base64-encoded own_value
+ String ov_b64 = csvRecord.get("own_value_b64");
+ // decode
+ byte[] ov_b = Base64.decodeBase64(ov_b64);
+ String ov = StringUtils.newStringUtf8(ov_b);
+ node.setOwnValue(ov);
+ }
+
+ }
+ /*
+ * re-read list and attach attributes and relations to entities
+ */
+ @SuppressWarnings("unchecked")
+ List oldNodeList = (List) nodeList.clone();
+ for (Node n : oldNodeList) {
+ if (n.getNodeType().equals("ATTRIBUTE")) {
+ Attribute att = (Attribute) n;
+ Long source_id = att.getSourceId();
+ if (entMap.containsKey(source_id)) {
+ Entity ent = entMap.get(source_id);
+ // add attribute to entity
+ ent.addAttribute(att);
+ // remove attribute from list
+ nodeList.remove(att);
+ } else if (relMap.containsKey(source_id)) {
+ Relation rel = relMap.get(source_id);
+ // add attribute to relation
+ rel.addAttribute(att);
+ // remove attribute from list
+ nodeList.remove(att);
+ }
+ } else if (n.getNodeType().equals("RELATION")) {
+ Relation rel = (Relation) n;
+ Long source_id = rel.getSourceId();
+ if (entMap.containsKey(source_id)) {
+ Entity ent = entMap.get(source_id);
+ // add attribute to entity
+ ent.addSourceRelation(rel);
+ }
+ Long target_id = rel.getTargetId();
+ if (entMap.containsKey(target_id)) {
+ Entity ent = entMap.get(target_id);
+ // add attribute to entity
+ ent.addTargetRelation(rel);
+ }
+ }
+
+ }
+
+ return nodeList;
+
+ }
+}