# HG changeset patch # User casties # Date 1477592479 -7200 # Node ID 2d669fdd0a8be652440289539f4dc92b25325f5e # Parent 69c233694e765adfa25b7c0a6989059862962e66 New CsvNodeListReader for import-merge. diff -r 69c233694e76 -r 2d669fdd0a8b pom.xml --- a/pom.xml Tue Oct 25 18:42:05 2016 +0200 +++ b/pom.xml Thu Oct 27 20:21:19 2016 +0200 @@ -163,7 +163,17 @@ 2.5 provided --> - + + + org.apache.commons + commons-csv + 1.4 + + + commons-codec + commons-codec + 1.10 + diff -r 69c233694e76 -r 2d669fdd0a8b src/main/java/org/mpi/openmind/repository/bo/Attribute.java --- a/src/main/java/org/mpi/openmind/repository/bo/Attribute.java Tue Oct 25 18:42:05 2016 +0200 +++ b/src/main/java/org/mpi/openmind/repository/bo/Attribute.java Thu Oct 27 20:21:19 2016 +0200 @@ -246,4 +246,9 @@ } return "!!!ENCODING-ERROR!!!"; } + + @Override + public String getNodeType() { + return "ATTRIBUTE"; + } } diff -r 69c233694e76 -r 2d669fdd0a8b src/main/java/org/mpi/openmind/repository/bo/Entity.java --- a/src/main/java/org/mpi/openmind/repository/bo/Entity.java Tue Oct 25 18:42:05 2016 +0200 +++ b/src/main/java/org/mpi/openmind/repository/bo/Entity.java Thu Oct 27 20:21:19 2016 +0200 @@ -1286,4 +1286,9 @@ } } + + @Override + public String getNodeType() { + return "ENTITY"; + } } diff -r 69c233694e76 -r 2d669fdd0a8b src/main/java/org/mpi/openmind/repository/bo/Node.java --- a/src/main/java/org/mpi/openmind/repository/bo/Node.java Tue Oct 25 18:42:05 2016 +0200 +++ b/src/main/java/org/mpi/openmind/repository/bo/Node.java Thu Oct 27 20:21:19 2016 +0200 @@ -400,4 +400,7 @@ return "!!!ENCODING-ERROR!!!"; } + public String getNodeType() { + return "NODE"; + } } diff -r 69c233694e76 -r 2d669fdd0a8b src/main/java/org/mpi/openmind/repository/bo/Relation.java --- a/src/main/java/org/mpi/openmind/repository/bo/Relation.java Tue Oct 25 18:42:05 2016 +0200 +++ b/src/main/java/org/mpi/openmind/repository/bo/Relation.java Thu Oct 27 20:21:19 2016 +0200 @@ -317,4 +317,10 @@ this.getOwnValue(), this.getId()); } + + @Override + public String getNodeType() { + return "RELATION"; + } + } diff -r 69c233694e76 -r 2d669fdd0a8b src/main/java/org/mpi/openmind/repository/utils/CsvNodeListReader.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/main/java/org/mpi/openmind/repository/utils/CsvNodeListReader.java Thu Oct 27 20:21:19 2016 +0200 @@ -0,0 +1,209 @@ +/** + * + */ +package org.mpi.openmind.repository.utils; + +import java.io.Reader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.StringUtils; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; +import org.mpi.openmind.repository.bo.Attribute; +import org.mpi.openmind.repository.bo.Entity; +import org.mpi.openmind.repository.bo.Node; +import org.mpi.openmind.repository.bo.Relation; + +/** + * Class that reads a CSV file into a list of Nodes. + * + * @author casties + * + */ +public class CsvNodeListReader { + + public static List readCsv(Reader input) throws Exception { + ArrayList nodeList = new ArrayList(); + HashMap entMap = new HashMap(); + HashMap relMap = new HashMap(); + // parse all records from CSV file + for (CSVRecord csvRecord : CSVFormat.DEFAULT.withHeader().parse(input)) { + // will throw exception if node_type is missing + String node_type = csvRecord.get("node_type"); + Node node = null; + + if (node_type.equals("ENTITY")) { + //System.out.println("Entity: "+csvRecord); + Entity ent = new Entity(); + // TODO: is lightweight ok if we want to add attributes? + ent.setLightweight(false); + if (csvRecord.isSet("id")) { + Long id = Long.decode(csvRecord.get("id")); + ent.setId(id); + entMap.put(id, ent); + } + nodeList.add(ent); + node = ent; + + } else if (node_type.equals("ATTRIBUTE")) { + //System.out.println("Attribute: "+csvRecord); + Attribute att = new Attribute(); + // throws exception if no source_id + Long source_id = Long.decode(csvRecord.get("source_id")); + att.setSourceId(source_id); + // save Attribute + nodeList.add(att); + // set optional properties + if (csvRecord.isSet("id")) { + Long id = Long.decode(csvRecord.get("id")); + att.setId(id); + } + if (csvRecord.isSet("source_modif")) { + Long mtime = Long.decode(csvRecord.get("source_modif")); + att.setSourceModif(mtime); + } + if (csvRecord.isSet("source_obj_class")) { + String oc = csvRecord.get("source_obj_class"); + att.setSourceObjectClass(oc); + } + node = att; + + } else if (node_type.equals("RELATION")) { + //System.out.println("Relation: "+csvRecord); + Relation rel = new Relation(); + if (csvRecord.isSet("id")) { + Long id = Long.decode(csvRecord.get("id")); + rel.setId(id); + relMap.put(id, rel); + } + // throws exception if no source_id or target_id + Long source_id = Long.decode(csvRecord.get("source_id")); + rel.setSourceId(source_id); + Long target_id = Long.decode(csvRecord.get("target_id")); + rel.setTargetId(target_id); + // set optional properties + if (csvRecord.isSet("source_modif")) { + Long mtime = Long.decode(csvRecord.get("source_modif")); + rel.setSourceModif(mtime); + } + if (csvRecord.isSet("source_obj_class")) { + String oc = csvRecord.get("source_obj_class"); + rel.setSourceObjectClass(oc); + } + if (csvRecord.isSet("target_modif")) { + Long mtime = Long.decode(csvRecord.get("target_modif")); + rel.setTargetModif(mtime); + } + if (csvRecord.isSet("target_obj_class")) { + String oc = csvRecord.get("target_obj_class"); + rel.setTargetObjectClass(oc); + } + nodeList.add(rel); + node = rel; + + } else { + // unknown node_type + throw new Exception("Unknown node_type "+node_type); + } + + /* + * set common node properties + */ + if (csvRecord.isSet("row_id")) { + Long row_id = Long.decode(csvRecord.get("row_id")); + // TODO: should we really set row_id? + node.setRowId(row_id); + } + if (csvRecord.isSet("object_class")) { + String oc = csvRecord.get("object_class"); + node.setObjectClass(oc); + } + if (csvRecord.isSet("user")) { + String user = csvRecord.get("user"); + node.setUser(user); + } + if (csvRecord.isSet("public")) { + boolean pub = csvRecord.get("public").equals("true"); + node.setIsPublic(pub); + } + if (csvRecord.isSet("type")) { + String type = csvRecord.get("type"); + node.setType(type); + } + if (csvRecord.isSet("version")) { + Long version = Long.decode(csvRecord.get("version")); + node.setVersion(version); + } + if (csvRecord.isSet("type")) { + String type = csvRecord.get("type"); + node.setType(type); + } + if (csvRecord.isSet("modification_time")) { + Long mtime = Long.decode(csvRecord.get("modification_time")); + node.setModificationTime(mtime); + } + if (csvRecord.isSet("system_status")) { + String status = csvRecord.get("system_status"); + node.setSystemStatus(status); + } + if (csvRecord.isSet("content_type")) { + String ct = csvRecord.get("content_type"); + node.setContentType(ct); + } + if (csvRecord.isSet("own_value_b64")) { + // get base64-encoded own_value + String ov_b64 = csvRecord.get("own_value_b64"); + // decode + byte[] ov_b = Base64.decodeBase64(ov_b64); + String ov = StringUtils.newStringUtf8(ov_b); + node.setOwnValue(ov); + } + + } + /* + * re-read list and attach attributes and relations to entities + */ + @SuppressWarnings("unchecked") + List oldNodeList = (List) nodeList.clone(); + for (Node n : oldNodeList) { + if (n.getNodeType().equals("ATTRIBUTE")) { + Attribute att = (Attribute) n; + Long source_id = att.getSourceId(); + if (entMap.containsKey(source_id)) { + Entity ent = entMap.get(source_id); + // add attribute to entity + ent.addAttribute(att); + // remove attribute from list + nodeList.remove(att); + } else if (relMap.containsKey(source_id)) { + Relation rel = relMap.get(source_id); + // add attribute to relation + rel.addAttribute(att); + // remove attribute from list + nodeList.remove(att); + } + } else if (n.getNodeType().equals("RELATION")) { + Relation rel = (Relation) n; + Long source_id = rel.getSourceId(); + if (entMap.containsKey(source_id)) { + Entity ent = entMap.get(source_id); + // add attribute to entity + ent.addSourceRelation(rel); + } + Long target_id = rel.getTargetId(); + if (entMap.containsKey(target_id)) { + Entity ent = entMap.get(target_id); + // add attribute to entity + ent.addTargetRelation(rel); + } + } + + } + + return nodeList; + + } +}