Mercurial > hg > openmind

changeset 29:5786aa6caeb3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
new XML export and test script.
author casties
date Mon, 22 Aug 2016 19:43:34 +0200
parents bdba8c108183
children 2f19cdf8e60b
files src/main/java/org/mpi/openmind/repository/utils/OM4StreamWriter.java src/main/java/org/mpi/openmind/repository/utils/XMLUtil.java src/main/java/org/mpi/openmind/scripts/IsmiXmlExport.java
diffstat 3 files changed, 321 insertions(+), 177 deletions(-) [+]
line wrap: on
line diff
--- a/src/main/java/org/mpi/openmind/repository/utils/OM4StreamWriter.java	Mon Aug 22 19:42:34 2016 +0200
+++ b/src/main/java/org/mpi/openmind/repository/utils/OM4StreamWriter.java	Mon Aug 22 19:43:34 2016 +0200
@@ -3,15 +3,13 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
-import java.text.DateFormat;
 import java.text.DecimalFormat;
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
 
-import javolution.xml.stream.XMLOutputFactory;
-import javolution.xml.stream.XMLStreamException;
-import javolution.xml.stream.XMLStreamWriter;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -20,184 +18,255 @@
 import org.mpi.openmind.repository.bo.Node;
 import org.mpi.openmind.repository.bo.Relation;
 import org.mpi.openmind.repository.services.PersistenceService;
-import org.mpi.openmind.scheduling.utils.Scheduling;
 
 public class OM4StreamWriter {
 
-	private static Logger logger = Logger.getLogger(OM4StreamWriter.class);
-	
-	private static int itemsPerPage = 500;
-	
-	public static void backupEntities(String fileName, PersistenceService ps){
-		generateEntities(fileName, ps, Node.TYPE_ABOX);
-	}
+    private static Logger logger = Logger.getLogger(OM4StreamWriter.class);
+
+    private static int itemsPerPage = 500;
+
+    /**
+     * Return the object's string representation or "null" if its null.
+     * 
+     * @param s
+     * @return
+     */
+    public static String defaultString(Object s) {
+        if (s == null) {
+            return "null";
+        } else {
+            return s.toString();
+        }
+    }
+    
+    
+    public static void backupEntities(String fileName, PersistenceService ps) {
+        writeEntsAndRels(fileName, ps, Node.TYPE_ABOX);
+    }
+
+    public static void backupDefinitions(String fileName, PersistenceService ps) {
+        writeEntsAndRels(fileName, ps, Node.TYPE_TBOX);
+    }
+
+    /**
+     * Writes all entities of the given type and their relations to the XML file at fileName.
+     * 
+     * @param fileName
+     * @param ps
+     * @param type
+     */
+    private static void writeEntsAndRels(String fileName, PersistenceService ps, String type) {
+        OutputStreamWriter out;
+        try {
+            FileOutputStream fileOut = new FileOutputStream(fileName);
+            out = new OutputStreamWriter(fileOut, "UTF-8");
+            XMLOutputFactory factory = XMLOutputFactory.newInstance();
+            XMLStreamWriter writer = factory.createXMLStreamWriter(out);
+
+            int entitiesCount = 0;
+
+            writer.writeStartDocument("UTF-8", "1.0");
+
+            if (type.equals(Node.TYPE_ABOX)) {
+                writer.writeStartElement(XMLUtil.OPENMIND_DATA);
+                writer.writeAttribute("version", "3.0");
+                entitiesCount = ps.getEntityCount(null).intValue();
+            } else {
+                writer.writeStartElement(XMLUtil.META_DATA);
+                writer.writeAttribute("version", "3.0");
+                entitiesCount = ps.getEntityCount(Node.TYPE_TBOX).intValue();
+            }
+
+            int numberOfPages = entitiesCount / itemsPerPage;
+            // debug: int numberOfPages = 1;
+            int counter = 0;
+            long start = System.currentTimeMillis();
+            DecimalFormat df = new DecimalFormat("#.##");
 
-	public static void backupDefinitions(String fileName, PersistenceService ps){
-		generateEntities(fileName, ps, Node.TYPE_TBOX);
-	}
-	
-	private static void generateEntities(String fileName, PersistenceService ps, String type) {
-		OutputStreamWriter out;
-		try {
-			FileOutputStream fileOut = new FileOutputStream (fileName);
-			out = new OutputStreamWriter (fileOut, "UTF-8");
-			
-			XMLOutputFactory factory = XMLOutputFactory.newInstance();
-		    factory.setProperty(XMLOutputFactory.INDENTATION, "\t");
+            List<Relation> relList = new ArrayList<Relation>();
+
+            /*
+             * write entities
+             */
+            writer.writeStartElement((type.equals(Node.TYPE_TBOX)) ? XMLUtil.DEFINITIONS : XMLUtil.ENTITIES);
+            writer.writeAttribute("number", Integer.toString(entitiesCount));
+            for (int currentPage = 0; currentPage <= numberOfPages; currentPage++) {
+                int startRecord = currentPage * itemsPerPage;
+                List<Entity> entities;
+
+                if (type.equals(Node.TYPE_ABOX)) {
+                    entities = ps.getEntityPage(null, startRecord, itemsPerPage);
+                } else {
+                    entities = ps.getEntityPage(Node.TYPE_TBOX, startRecord, itemsPerPage);
+                }
+
+                for (Entity ent : entities) {
+                    // write entities to XML
+                    writeEntity(ent, writer, ps);
+                    // add (source)relations to list
+                    relList.addAll(ent.getSourceRelations());
+                    
+                    counter++;
+                    /* if ((counter % 50) == 0) {
+                        logger.debug("*");
+                    } */
+                }
+                
+                long runtime = System.currentTimeMillis() - start;
+                double percent = ((double) counter / (double) entitiesCount) * 100.0;
+                logger.debug("(" + df.format(percent) + "%) \t[" + counter + "/" + entitiesCount + "]\t");
+                logger.debug("Speed[ents/s]: " + df.format((double) counter / ((double) runtime / 1000)));
+                writer.flush();
+            }
+            writer.writeEndElement();
+
+            /*
+             * write relations (from list)
+             */
+            writer.writeStartElement(XMLUtil.RELATIONS);
+            writer.writeAttribute("number", Integer.toString(relList.size()));
+            for (Relation rel : relList) {
+                writeRelation(rel, writer);
+            }
+            writer.writeEndElement();
+
+            // end file.
+            writer.writeEndElement();
+
+            writer.flush();
+            writer.close();
+
+            logger.info("END Stream Writer");
+        } catch (IOException e) {
+            e.printStackTrace();
+        } catch (XMLStreamException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
 
-			XMLStreamWriter xmlStreamWriter = factory.createXMLStreamWriter(out);
-			
-			int entitiesCount = 0;
-			
-			xmlStreamWriter.writeStartDocument("UTF-8", "1.0");
-			
-			if(type.equals(Node.TYPE_ABOX)){
-				xmlStreamWriter.writeStartElement(XMLUtil.OPENMIND_DATA);
-				entitiesCount = ps.getEntityCount(null).intValue();
-			}else{
-				xmlStreamWriter.writeStartElement(XMLUtil.META_DATA);
-				entitiesCount = ps.getEntityCount(Node.TYPE_TBOX).intValue();
-			}
-			
-			int numberOfPages = entitiesCount / itemsPerPage;
-			int counter = 0;
-			long start = System.currentTimeMillis();
-			DecimalFormat df = new DecimalFormat("#.##");
-			
-			List<Relation> relList = new ArrayList<Relation>();
-			
-			//(type.equals(Node.TYPE_TBOX)) ? "" : ""
-			
-			xmlStreamWriter.writeStartElement((type.equals(Node.TYPE_TBOX)) ? XMLUtil.DEFINITIONS : XMLUtil.ENTITIES);
-			for(int currentPage = 0; currentPage <= numberOfPages; currentPage++){
-				int startRecord = currentPage * itemsPerPage;
-				List<Entity> entities;
-				
-				if(type.equals(Node.TYPE_ABOX)){
-					entities = ps.getEntityPage(null, startRecord, itemsPerPage);
-				}else{
-					entities = ps.getEntityPage(Node.TYPE_TBOX, startRecord, itemsPerPage);
-				}
-				
-				for(Entity ent : entities){
-					insertEntity(ent, xmlStreamWriter, ps);
-					relList.addAll(ent.getSourceRelations());
-					counter++;
-					if ((counter % 50) == 0) {
-						System.out.print("*");
-					}
-				}
-				long diff = System.currentTimeMillis() - start;
-				double min = (double) diff / (double) (60 * 1000);
-				double percent = ((double) counter / (double) entitiesCount) * 100.0;
-				System.out.print("\n(" + df.format(percent) + "%) \t[" + counter + "/" + entitiesCount + "]\t");
-				logger.info("Speed[ents/min]: " + (double) counter / min);
-				xmlStreamWriter.flush();
-			}
-			xmlStreamWriter.writeEndElement();
-			
-			xmlStreamWriter.writeStartElement(XMLUtil.RELATIONS);
-			for(Relation rel : relList){
-				insertRelation(rel, xmlStreamWriter);
-			}
-			xmlStreamWriter.writeEndElement();
-			
-						
-			//end file.
-			xmlStreamWriter.writeEndElement();
-			
-			xmlStreamWriter.flush();
-			xmlStreamWriter.close();
-			
-			logger.info("END Stream Writer");
-		} catch (IOException e) {
-			e.printStackTrace();
-		} catch (XMLStreamException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
-	}
-	
-	private static void insertRelation(Relation rel, XMLStreamWriter writer) throws XMLStreamException{
-		writer.writeStartElement(XMLUtil.RELATION);
-		
-		if(rel.getId() != null)
-    		writer.writeAttribute(XMLUtil.ID, rel.getId().toString());
-        if(StringUtils.isNotEmpty(rel.getOwnValue()))
-        	writer.writeAttribute(XMLUtil.OWN_VALUE, rel.getOwnValue());
-        if(StringUtils.isNotEmpty(rel.getObjectClass()))
-        	writer.writeAttribute(XMLUtil.OBJECT_CLASS, rel.getObjectClass());
-        if(StringUtils.isNotEmpty(rel.getContentType()))
-        	writer.writeAttribute(XMLUtil.CONTENT_TYPE, rel.getContentType());
-        if(rel.getSourceId() != null)
-        	writer.writeAttribute(XMLUtil.RELATION_SOURCE_ID, rel.getSourceId().toString());
-        if(rel.getTargetId() != null)
-        	writer.writeAttribute(XMLUtil.RELATION_TARGET_ID, rel.getTargetId().toString());
-        if(rel.getVersion() != null)
-        	writer.writeAttribute(XMLUtil.VERSION, rel.getVersion().toString());
-        if(rel.getModificationTime() != null)
-        	writer.writeAttribute(XMLUtil.MODIFICATION_TIME, rel.getModificationTime().toString());
+    /**
+     * Write OpenMind relation to XML.
+     * 
+     * @param rel
+     * @param writer
+     * @throws XMLStreamException
+     */
+    private static void writeRelation(Relation rel, XMLStreamWriter writer) throws XMLStreamException {
+        writer.writeStartElement(XMLUtil.RELATION);
+
+        /*
+         * write XML-attributes
+         */
+        writer.writeAttribute(XMLUtil.OBJECT_CLASS, defaultString(rel.getObjectClass()));
+        writer.writeAttribute(XMLUtil.ID, defaultString(rel.getId()));
+        writer.writeAttribute(XMLUtil.ROW_ID, defaultString(rel.getRowId()));
+        if (StringUtils.isNotEmpty(rel.getContentType()))
+            writer.writeAttribute(XMLUtil.CONTENT_TYPE, rel.getContentType());
+        writer.writeAttribute(XMLUtil.RELATION_SOURCE_ID, defaultString(rel.getSourceId()));
+        writer.writeAttribute(XMLUtil.RELATION_TARGET_ID, defaultString(rel.getTargetId()));
+        writer.writeAttribute(XMLUtil.VERSION, defaultString(rel.getVersion()));
+        writer.writeAttribute(XMLUtil.MODIFICATION_TIME, defaultString(rel.getModificationTime()));
+        if (rel.getUser() != null)
+            writer.writeAttribute(XMLUtil.USER, rel.getUser());
+        if (rel.getIsPublic())
+            writer.writeAttribute(XMLUtil.PUBLIC, "true");
+
+        /*
+         * write OpenMind attributes of this relation as XML tags
+         */
+        if (rel.getAttributes().size() > 0) {
+            writer.writeStartElement(XMLUtil.ATTRIBUTES);
+            for (Attribute att : rel.getAttributes()) {
+                writeAttribute(att, writer);
+            }
+            writer.writeEndElement();
+        }
         
-		writer.writeEndElement();
-	}
-	
-	private static void insertEntity(Entity entity, XMLStreamWriter writer, PersistenceService ps) throws XMLStreamException{
-		
-		writer.writeStartElement((entity.getType().equals(Node.TYPE_TBOX)) ? XMLUtil.DEFINITION : XMLUtil.ENTITY);
-		
-		if(entity.isLightweight()){
-			entity = ps.getEntityContent(entity);
-		}
+        /*
+         *  write own value as content
+         */
+        if (StringUtils.isNotEmpty(rel.getOwnValue()))
+            writer.writeCharacters(rel.getOwnValue());
+
+        writer.writeEndElement();
+    }
+
+    /**
+     * Write OpenMind entity to XML.
+     * 
+     * @param entity
+     * @param writer
+     * @param ps
+     * @throws XMLStreamException
+     */
+    private static void writeEntity(Entity entity, XMLStreamWriter writer, PersistenceService ps)
+            throws XMLStreamException {
 
-		if(StringUtils.isNotEmpty(entity.getObjectClass()))
-        	writer.writeAttribute(XMLUtil.OBJECT_CLASS, entity.getObjectClass());
-        if(entity.getId() != null)
-    		writer.writeAttribute(XMLUtil.ID, entity.getId().toString());
-        if(StringUtils.isNotEmpty(entity.getOwnValue()))
-    		writer.writeAttribute(XMLUtil.OWN_VALUE, entity.getOwnValue());
-        if(StringUtils.isNotEmpty(entity.getContentType()))
-        	writer.writeAttribute(XMLUtil.CONTENT_TYPE, entity.getContentType());
-        if(entity.getVersion() != null)
-        	writer.writeAttribute(XMLUtil.VERSION, entity.getVersion().toString());
-        if(entity.getModificationTime() != null)
-        	writer.writeAttribute(XMLUtil.MODIFICATION_TIME, entity.getModificationTime().toString());
+        writer.writeStartElement((entity.getType().equals(Node.TYPE_TBOX)) ? XMLUtil.DEFINITION : XMLUtil.ENTITY);
+
+        if (entity.isLightweight()) {
+            entity = ps.getEntityContent(entity);
+        }
+
+        /*
+         * write XML attributes
+         */
+        writer.writeAttribute(XMLUtil.OBJECT_CLASS, defaultString(entity.getObjectClass()));
+        writer.writeAttribute(XMLUtil.ID, defaultString(entity.getId()));
+        writer.writeAttribute(XMLUtil.ROW_ID, defaultString(entity.getRowId()));
+        if (StringUtils.isNotEmpty(entity.getContentType()))
+            writer.writeAttribute(XMLUtil.CONTENT_TYPE, entity.getContentType());
+        writer.writeAttribute(XMLUtil.VERSION, defaultString(entity.getVersion()));
+        writer.writeAttribute(XMLUtil.MODIFICATION_TIME, defaultString(entity.getModificationTime()));
+        if (entity.getUser() != null)
+            writer.writeAttribute(XMLUtil.USER, entity.getUser());
+        if (entity.getIsPublic())
+            writer.writeAttribute(XMLUtil.PUBLIC, "true");
 
-		
-		if(entity.getAttributes().size() > 0){
-			writer.writeStartElement(XMLUtil.ATTRIBUTES);
-			for(Attribute att : entity.getAttributes()){
-				insertEntity(att, writer);
-			}	
-			writer.writeEndElement();
-		}
-		
-		writer.writeEndElement();
-	}
-	
-	private static void insertEntity(Attribute att, XMLStreamWriter writer) throws XMLStreamException{
-		writer.writeStartElement(XMLUtil.ATTRIBUTE);
+        /*
+         * write OpenMind attributes of this entity as XML tags
+         */
+        if (entity.getAttributes().size() > 0) {
+            writer.writeStartElement(XMLUtil.ATTRIBUTES);
+            for (Attribute att : entity.getAttributes()) {
+                writeAttribute(att, writer);
+            }
+            writer.writeEndElement();
+        }
+
+        /*
+         * write own value
+         */
+        if (StringUtils.isNotEmpty(entity.getOwnValue()))
+            writer.writeCharacters(entity.getOwnValue());
+
+        writer.writeEndElement();
+    }
+
+    private static void writeAttribute(Attribute att, XMLStreamWriter writer) throws XMLStreamException {
+        writer.writeStartElement(XMLUtil.ATTRIBUTE);
+
+        /*
+         * write XML attributes
+         */
+        writer.writeAttribute(XMLUtil.ATTRIBUTE_NAME, defaultString(att.getName()));
+        writer.writeAttribute(XMLUtil.ID, defaultString(att.getId()));
+        writer.writeAttribute(XMLUtil.ROW_ID, defaultString(att.getRowId()));
+        writer.writeAttribute(XMLUtil.CONTENT_TYPE, defaultString(att.getContentType()));
+        writer.writeAttribute(XMLUtil.VERSION, defaultString(att.getVersion()));
+        writer.writeAttribute(XMLUtil.MODIFICATION_TIME, defaultString(att.getModificationTime()));
+        if (att.getUser() != null)
+            writer.writeAttribute(XMLUtil.USER, att.getUser());
+        if (att.getIsPublic())
+            writer.writeAttribute(XMLUtil.PUBLIC, "true");
+
+        /*
+         * write value as content
+         */
+        if (StringUtils.isNotEmpty(att.getValue()))
+            writer.writeCharacters(att.getValue());
         
-        if(StringUtils.isNotEmpty(att.getName()))
-        	writer.writeAttribute(XMLUtil.ATTRIBUTE_NAME, att.getName());
-		if(att.getId() != null)
-    		writer.writeAttribute(XMLUtil.ID, att.getId().toString());
-        if(StringUtils.isNotEmpty(att.getValue()))
-        	writer.writeAttribute(XMLUtil.ATTRIBUTE_VALUE, att.getValue());
-        if(StringUtils.isNotEmpty(att.getContentType()))
-        	writer.writeAttribute(XMLUtil.CONTENT_TYPE, att.getContentType());
-        if(att.getVersion() != null)
-        	writer.writeAttribute(XMLUtil.VERSION, att.getVersion().toString());
-        if(att.getModificationTime() != null)
-        	writer.writeAttribute(XMLUtil.MODIFICATION_TIME, att.getModificationTime().toString());
-        
-		writer.writeEndElement();
-	}
-	
-	
-	
-	public static void main(String[] args) {
-		//ServiceRegistry srvReg = new ServiceRegistry();
-		//OM4StreamWriter.backupEntities("/Users/jurzua/ent.xml", srvReg.getPersistenceService());
-		logger.info(DateFormat.getInstance().format(new Date()));
-	}
+        writer.writeEndElement();
+    }
 }
--- a/src/main/java/org/mpi/openmind/repository/utils/XMLUtil.java	Mon Aug 22 19:42:34 2016 +0200
+++ b/src/main/java/org/mpi/openmind/repository/utils/XMLUtil.java	Mon Aug 22 19:43:34 2016 +0200
@@ -58,9 +58,12 @@
     public static String RELATION_ID = "id";
     public static String VIEW = "view";
     public static String VIEWS = "views";
-    public static String MODIFICATION_TIME = "modification-time";
+    public static String MODIFICATION_TIME = "mtime";
     public static String VERSION = "version";
     public static String ID = "id";
+    public static String ROW_ID = "row-id";
+    public static String USER = "user";
+    public static String PUBLIC = "public";
     public static String CONCEPTS = "concepts";
     public static String CONCEPT = "concept";
     public static String ASSERTIONS = "assertions";
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/org/mpi/openmind/scripts/IsmiXmlExport.java	Mon Aug 22 19:43:34 2016 +0200
@@ -0,0 +1,72 @@
+package org.mpi.openmind.scripts;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.mpi.openmind.cache.WrapperService;
+import org.mpi.openmind.configuration.ConfigurationService;
+import org.mpi.openmind.repository.services.PersistenceService;
+import org.mpi.openmind.repository.services.ServiceRegistry;
+import org.mpi.openmind.repository.utils.OM4StreamWriter;
+
+
+public class IsmiXmlExport {
+	
+    
+    
+	private static Logger logger;
+    private static ConfigurationService configurationService;
+    private static PersistenceService persistenceService;
+
+
+    static {
+        
+        ConsoleAppender console = new ConsoleAppender(); // create appender
+        // configure the appender
+        String PATTERN = "%d [%p|%c|%C{1}] %m%n";
+        console.setLayout(new PatternLayout(PATTERN));
+        console.setThreshold(Level.INFO);
+        console.activateOptions();
+        // add appender to any Logger (here is root)
+        //Logger.getRootLogger().addAppender(console);
+        
+        logger = Logger.getLogger(OM4StreamWriter.class);
+        logger.setLevel(Level.DEBUG);
+    }
+
+	public static void execute(){
+		ServiceRegistry services = new ServiceRegistry();
+        configurationService = services.getConfigurationService();
+        persistenceService = services.getPS();
+		exportXml(services.getWrapper());
+	}
+	
+	private static void exportXml(WrapperService omService){
+	    // get path from configuration service
+        String schedulingPath = configurationService.getSchedulingPath();
+        if(schedulingPath.charAt(schedulingPath.length() - 1) != '/'){
+            schedulingPath += "/";
+        }
+        // add date
+        Date now = new Date();
+        DateFormat formatter = new SimpleDateFormat("yyMMdd-HHmm");
+        String defFn = "openmind-defs-" + formatter.format(now) + ".xml";
+        String entFn = "openmind-data-" + formatter.format(now) + ".xml";
+        // export XML
+        logger.info("Backup Definitions as: " + defFn);
+        OM4StreamWriter.backupDefinitions(schedulingPath + defFn, persistenceService);
+        logger.info("Backup Entities as: " + entFn);
+        OM4StreamWriter.backupEntities(schedulingPath + entFn, persistenceService);
+	}
+	
+	
+	public static void main(String[] args){
+		execute();
+		System.exit(0);
+	}
+}