view src/main/java/org/mpi/openmind/repository/utils/OM4StreamWriter.java @ 31:7d8ebe8ac8a2

create reader and check script for XML dumps.
author casties
date Wed, 24 Aug 2016 19:12:24 +0200
parents 5786aa6caeb3
children e0be7c0030f5
line wrap: on
line source

package org.mpi.openmind.repository.utils;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;

import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.mpi.openmind.repository.bo.Attribute;
import org.mpi.openmind.repository.bo.Entity;
import org.mpi.openmind.repository.bo.Node;
import org.mpi.openmind.repository.bo.Relation;
import org.mpi.openmind.repository.services.PersistenceService;

/**
 * Export all entities and relations and definitions to XML.
 * 
 * Saves (real) entities and relations (i.e. assertions) and definitions 
 * (i.e. definition entities and relations) in separate files.
 * 
 * @author jurzua, casties
 *
 */
public class OM4StreamWriter {

    private static Logger logger = Logger.getLogger(OM4StreamWriter.class);

    private static int itemsPerPage = 500;

    /**
     * Return the object's string representation or "null" if its null.
     * 
     * @param s
     * @return
     */
    private static String defaultString(Object s) {
        if (s == null) {
            return "null";
        } else {
            return s.toString();
        }
    }
    
    
    public static void backupEntities(String fileName, PersistenceService ps) {
        writeEntsAndRels(fileName, ps, Node.TYPE_ABOX);
    }

    public static void backupDefinitions(String fileName, PersistenceService ps) {
        writeEntsAndRels(fileName, ps, Node.TYPE_TBOX);
    }

    /**
     * Writes all entities of the given type and their relations to the XML file at fileName.
     * 
     * @param fileName
     * @param ps
     * @param type
     */
    private static void writeEntsAndRels(String fileName, PersistenceService ps, String type) {
        OutputStreamWriter out;
        try {
            FileOutputStream fileOut = new FileOutputStream(fileName);
            out = new OutputStreamWriter(fileOut, "UTF-8");
            XMLOutputFactory factory = XMLOutputFactory.newInstance();
            XMLStreamWriter writer = factory.createXMLStreamWriter(out);

            int entitiesCount = 0;

            writer.writeStartDocument("UTF-8", "1.0");

            if (type.equals(Node.TYPE_ABOX)) {
                writer.writeStartElement(XMLUtil.OPENMIND_DATA);
                writer.writeAttribute("version", "4.3");
                entitiesCount = ps.getEntityCount(null).intValue();
            } else {
                writer.writeStartElement(XMLUtil.META_DATA);
                writer.writeAttribute("version", "4.3");
                entitiesCount = ps.getEntityCount(Node.TYPE_TBOX).intValue();
            }

            int numberOfPages = entitiesCount / itemsPerPage;
            // debug: int numberOfPages = 1;
            int counter = 0;
            long start = System.currentTimeMillis();
            DecimalFormat df = new DecimalFormat("#.##");

            List<Relation> relList = new ArrayList<Relation>();

            /*
             * write entities
             */
            writer.writeStartElement((type.equals(Node.TYPE_TBOX)) ? XMLUtil.DEFINITIONS : XMLUtil.ENTITIES);
            writer.writeAttribute("number", Integer.toString(entitiesCount));
            for (int currentPage = 0; currentPage <= numberOfPages; currentPage++) {
                int startRecord = currentPage * itemsPerPage;
                List<Entity> entities;

                if (type.equals(Node.TYPE_ABOX)) {
                    entities = ps.getEntityPage(null, startRecord, itemsPerPage);
                } else {
                    entities = ps.getEntityPage(Node.TYPE_TBOX, startRecord, itemsPerPage);
                }

                for (Entity ent : entities) {
                    // write entities to XML
                    writeEntity(ent, writer, ps);
                    // add (source)relations to list
                    relList.addAll(ent.getSourceRelations());
                    
                    counter++;
                    /* if ((counter % 50) == 0) {
                        logger.debug("*");
                    } */
                }
                
                long runtime = System.currentTimeMillis() - start;
                double percent = ((double) counter / (double) entitiesCount) * 100.0;
                logger.debug("(" + df.format(percent) + "%) \t[" + counter + "/" + entitiesCount + "]\t");
                logger.debug("Speed[ents/s]: " + df.format((double) counter / ((double) runtime / 1000)));
                writer.flush();
            }
            writer.writeEndElement();

            /*
             * write relations (from list)
             */
            writer.writeStartElement(XMLUtil.RELATIONS);
            writer.writeAttribute("number", Integer.toString(relList.size()));
            for (Relation rel : relList) {
                writeRelation(rel, writer);
            }
            writer.writeEndElement();

            // end file.
            writer.writeEndElement();

            writer.flush();
            writer.close();

            logger.info("END Stream Writer");
        } catch (IOException e) {
            e.printStackTrace();
        } catch (XMLStreamException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

    /**
     * Write OpenMind relation to XML.
     * 
     * @param rel
     * @param writer
     * @throws XMLStreamException
     */
    private static void writeRelation(Relation rel, XMLStreamWriter writer) throws XMLStreamException {
        writer.writeStartElement(XMLUtil.RELATION);

        /*
         * write XML-attributes
         */
        writer.writeAttribute(XMLUtil.OBJECT_CLASS, defaultString(rel.getObjectClass()));
        writer.writeAttribute(XMLUtil.ID, defaultString(rel.getId()));
        writer.writeAttribute(XMLUtil.ROW_ID, defaultString(rel.getRowId()));
        if (StringUtils.isNotEmpty(rel.getContentType()))
            writer.writeAttribute(XMLUtil.CONTENT_TYPE, rel.getContentType());
        writer.writeAttribute(XMLUtil.RELATION_SOURCE_ID, defaultString(rel.getSourceId()));
        writer.writeAttribute(XMLUtil.RELATION_TARGET_ID, defaultString(rel.getTargetId()));
        writer.writeAttribute(XMLUtil.VERSION, defaultString(rel.getVersion()));
        writer.writeAttribute(XMLUtil.MODIFICATION_TIME, defaultString(rel.getModificationTime()));
        if (rel.getUser() != null)
            writer.writeAttribute(XMLUtil.USER, rel.getUser());
        if (rel.getIsPublic())
            writer.writeAttribute(XMLUtil.PUBLIC, "true");

        /*
         * write OpenMind attributes of this relation as XML tags
         */
        if (rel.getAttributes().size() > 0) {
            writer.writeStartElement(XMLUtil.ATTRIBUTES);
            for (Attribute att : rel.getAttributes()) {
                writeAttribute(att, writer);
            }
            writer.writeEndElement();
        }
        
        /*
         *  write own value as content
         */
        if (StringUtils.isNotEmpty(rel.getOwnValue()))
            writer.writeCharacters(rel.getOwnValue());

        writer.writeEndElement();
    }

    /**
     * Write OpenMind entity to XML.
     * 
     * @param entity
     * @param writer
     * @param ps
     * @throws XMLStreamException
     */
    private static void writeEntity(Entity entity, XMLStreamWriter writer, PersistenceService ps)
            throws XMLStreamException {

        writer.writeStartElement((entity.getType().equals(Node.TYPE_TBOX)) ? XMLUtil.DEFINITION : XMLUtil.ENTITY);

        if (entity.isLightweight()) {
            entity = ps.getEntityContent(entity);
        }

        /*
         * write XML attributes
         */
        writer.writeAttribute(XMLUtil.OBJECT_CLASS, defaultString(entity.getObjectClass()));
        writer.writeAttribute(XMLUtil.ID, defaultString(entity.getId()));
        writer.writeAttribute(XMLUtil.ROW_ID, defaultString(entity.getRowId()));
        if (StringUtils.isNotEmpty(entity.getContentType()))
            writer.writeAttribute(XMLUtil.CONTENT_TYPE, entity.getContentType());
        writer.writeAttribute(XMLUtil.VERSION, defaultString(entity.getVersion()));
        writer.writeAttribute(XMLUtil.MODIFICATION_TIME, defaultString(entity.getModificationTime()));
        if (entity.getUser() != null)
            writer.writeAttribute(XMLUtil.USER, entity.getUser());
        if (entity.getIsPublic())
            writer.writeAttribute(XMLUtil.PUBLIC, "true");

        /*
         * write OpenMind attributes of this entity as XML tags
         */
        if (entity.getAttributes().size() > 0) {
            writer.writeStartElement(XMLUtil.ATTRIBUTES);
            for (Attribute att : entity.getAttributes()) {
                writeAttribute(att, writer);
            }
            writer.writeEndElement();
        }

        /*
         * write own value
         */
        if (StringUtils.isNotEmpty(entity.getOwnValue()))
            writer.writeCharacters(entity.getOwnValue());

        writer.writeEndElement();
    }

    private static void writeAttribute(Attribute att, XMLStreamWriter writer) throws XMLStreamException {
        writer.writeStartElement(XMLUtil.ATTRIBUTE);

        /*
         * write XML attributes
         */
        writer.writeAttribute(XMLUtil.ATTRIBUTE_NAME, defaultString(att.getName()));
        writer.writeAttribute(XMLUtil.ID, defaultString(att.getId()));
        writer.writeAttribute(XMLUtil.ROW_ID, defaultString(att.getRowId()));
        writer.writeAttribute(XMLUtil.CONTENT_TYPE, defaultString(att.getContentType()));
        writer.writeAttribute(XMLUtil.VERSION, defaultString(att.getVersion()));
        writer.writeAttribute(XMLUtil.MODIFICATION_TIME, defaultString(att.getModificationTime()));
        if (att.getUser() != null)
            writer.writeAttribute(XMLUtil.USER, att.getUser());
        if (att.getIsPublic())
            writer.writeAttribute(XMLUtil.PUBLIC, "true");

        /*
         * write value as content
         */
        if (StringUtils.isNotEmpty(att.getValue()))
            writer.writeCharacters(att.getValue());
        
        writer.writeEndElement();
    }
}