Mercurial > hg > openmind

changeset 65:74cd973f6ece

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
more work on stale entity update problem. cache compare function works now.
author casties
date Fri, 27 Jan 2017 19:55:33 +0100
parents 83b209aa4226
children 3e4b05a6cb47
files .hgignore src/main/java/org/mpi/openmind/cache/AbstractCacheService.java src/main/java/org/mpi/openmind/cache/CacheService.java src/main/java/org/mpi/openmind/cache/WrapperService.java src/main/java/org/mpi/openmind/repository/bo/Attribute.java
diffstat 5 files changed, 324 insertions(+), 14 deletions(-) [+]
line wrap: on
line diff
--- a/.hgignore	Thu Jan 26 19:37:25 2017 +0100
+++ b/.hgignore	Fri Jan 27 19:55:33 2017 +0100
@@ -18,4 +18,6 @@
 syntax: regexp
 ^src/main/resources/log4j\.properties$
 syntax: regexp
-^src/main/resources/openmind\.properties$
\ No newline at end of file
+^src/main/resources/openmind\.properties$
+syntax: regexp
+^bin$
\ No newline at end of file
--- a/src/main/java/org/mpi/openmind/cache/AbstractCacheService.java	Thu Jan 26 19:37:25 2017 +0100
+++ b/src/main/java/org/mpi/openmind/cache/AbstractCacheService.java	Fri Jan 27 19:55:33 2017 +0100
@@ -27,7 +27,7 @@
 	
 	private PersistenceService persistenceService;
 	
-	private DuplexMap<Entity, String, Long> entMap;
+	protected DuplexMap<Entity, String, Long> entMap;
 	private Map<String, Boolean> entLoadedByOC;
 	
 	public AbstractCacheService(){
@@ -178,7 +178,15 @@
 	}
 	
 	
-	public boolean isEntityCurrent(Entity ent) {
+	/**
+	 * Check if the given Entity is up-to-date with the cache.
+	 * 
+	 * Compares accessTime only.
+	 * 
+	 * @param ent
+	 * @return
+	 */
+	public boolean isLWEntityCurrent(Entity ent) {
 		Entity cacheEnt = this.entMap.getValuesByOwnKey(ent.getId());
 		if (cacheEnt == null) {
 			// not in cache
@@ -197,7 +205,7 @@
 	//******************************************
 	
 	//AttKey(OC, attName), SrcId
-	private TripleMap<Attribute, AttKey, Long, Long> attMap;
+	protected TripleMap<Attribute, AttKey, Long, Long> attMap;
 	private Map<Long, Boolean> attLoadedBySrcId;
 	private Map<AttKey, Boolean> attLoadedByOCAndName;
 	
@@ -243,7 +251,7 @@
 	}
 	
 	/**
-	 * Return a list of (cloned) Attributes of the Entity.
+	 * Return a list of (cloned) Attributes with the given source id.
 	 * 
 	 * Loads Attributes from the database into the cache.
 	 * 
@@ -258,25 +266,32 @@
 		return list;
 	}
 	
-	public List<Attribute> getAttsBySrcIdReadOnly(Long srcId){
-		//logger.info("getAttsBySrcIdReadOnly srcId=" + srcId);
+	/**
+	 * Return a list of (non-cloned) Attributes with the given source id from the cache.
+	 * 
+	 * Loads Attributes from the database into the cache.
+	 * 
+	 * @param srcId
+	 * @return
+	 */
+	public List<Attribute> getAttsBySrcIdReadOnly(Long srcId) {
 		if(attLoadedBySrcId.get(srcId) == null || attLoadedBySrcId.get(srcId) == false){
 			synchronized (attMap) {
-				if(attLoadedBySrcId.get(srcId) == null){
-					//logger.info("getAttsBySrcIdReadOnly fromDB entId=" + srcId);
+				if(attLoadedBySrcId.get(srcId) == null) {
 					attLoadedBySrcId.put(srcId, false);
-					List<Attribute> list = 
-						getPs().getAllAttributes(srcId, -1);
-					for(Attribute att : list){
+					// load from db
+					List<Attribute> list = getPs().getAllAttributes(srcId, -1);
+					for (Attribute att : list) {
 						// update access timestamp
 						att.setAccessTime(System.currentTimeMillis());
+						// put in cache
 						this.attMap.put(att.getKey(), att);
 					}
 					attLoadedBySrcId.put(srcId, true);
-					//logger.info("STARTING getAttsBySrcIdReadOnly srcId=" + srcId + " - size=" + list.size());
 				}
 			}
 		}
+		// return attributes from cache
 		return this.attMap.getValuesByBKey(srcId);
 	}
 	
@@ -304,11 +319,36 @@
 		return this.attMap.getValuesByAKey(ocNameKey);
 	}
 	
+
+	/**
+	 * Check if the given Attribute is up-to-date with the cache.
+	 * 
+	 * Compares accessTime only.
+	 * 
+	 * @param att
+	 * @return
+	 */
+	public boolean isAttCurrent(Attribute att) {
+		Attribute cacheAtt = this.attMap.get(att.getKey());
+		if (cacheAtt == null) {
+			// not in cache
+			return true;
+		}
+		long ct = cacheAtt.getAccessTime();
+		long et = att.getAccessTime();
+		if (ct == 0 || et == 0 || et > ct) {
+			logger.error("Weird cache access times! old="+Long.toString(ct)+" new="+Long.toString(et));
+		}
+		return (ct == et);
+	}
+	
+
+	
 	//******************************************
 	//************Relations*********************
 	//******************************************
 	//RelKey<>, src, tar, relId
-	private PentaMap<Relation, RelKey, Long, Long, String, Long> relMap;
+	protected PentaMap<Relation, RelKey, Long, Long, String, Long> relMap;
 	private Map<Long, Boolean> relLoadedBySrcId;
 	private Map<Long, Boolean> relLoadedByTarId;
 	private Map<String, Boolean> relLoadedByName;
@@ -384,6 +424,14 @@
         return relMap.getValuesByBKey(srcId);
     }
 	
+	/**
+	 * Return Relations with matching target id from cache.
+     * 
+     * Loads Relations from db and stores in cache if not in cache.
+     * 
+	 * @param tarId
+	 * @return
+	 */
 	public List<Relation> getRelsByTarId(Long tarId){
 		if(relLoadedByTarId.get(tarId) == null || relLoadedByTarId.get(tarId) == false){
 			synchronized (relMap) {
@@ -441,6 +489,29 @@
 		return relMap.getValuesByDKey(name);
 	}
 	
+	/**
+	 * Check if the given Relation is up-to-date with the cache.
+	 * 
+	 * Compares accessTime only.
+	 * 
+	 * @param rel
+	 * @return
+	 */
+	public boolean isRelCurrent(Relation rel) {
+		Relation cacheRel = this.relMap.get(rel.getKey());
+		if (cacheRel == null) {
+			// not in cache
+			return true;
+		}
+		long ct = cacheRel.getAccessTime();
+		long et = rel.getAccessTime();
+		if (ct == 0 || et == 0 || et > ct) {
+			logger.error("Weird cache access times! old="+Long.toString(ct)+" new="+Long.toString(et));
+		}
+		return (ct == et);
+	}
+	
+
 	public PersistenceService getPs() {
 		return persistenceService;
 	}
--- a/src/main/java/org/mpi/openmind/cache/CacheService.java	Thu Jan 26 19:37:25 2017 +0100
+++ b/src/main/java/org/mpi/openmind/cache/CacheService.java	Fri Jan 27 19:55:33 2017 +0100
@@ -7,6 +7,7 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.mpi.openmind.repository.bo.Attribute;
 import org.mpi.openmind.repository.bo.Entity;
@@ -372,6 +373,233 @@
 		}
 	}
 	
+
+	/**
+	 * Returns if this Entity and its Attributes and Relations are up-to-date with the cache.
+	 * 
+	 * Checks accessTimes only.
+	 * 
+	 * @param ent
+	 * @return
+	 */
+	public boolean isEntityCurrent(Entity ent) {
+		// check entity
+		if (!isLWEntityCurrent(ent)) {
+			return false;
+		}
+		
+		if (ent.isLightweight()) {
+			logger.error("Can not check attributes and relations of lightweight entity!");
+			return false;
+		}
+		Long entId = ent.getId();
+		
+		// check attached attributes
+		List<Attribute> entAtts = ent.getAttributes();
+		List<Attribute> cacheAtts = attMap.getValuesByBKey(entId);
+		for (Attribute att : entAtts) {
+			if (!isAttCurrent(att)) {
+				return false;
+			} else {
+				if (!cacheAtts.remove(att)) {
+					return false;
+				}
+			}
+		}		
+		// check additional attributes in cache
+		if (!cacheAtts.isEmpty()) {
+			for (Attribute ca : cacheAtts) {
+				// ignore empty attributes
+				if (!StringUtils.isEmpty(ca.getOwnValue())) {
+					return false;
+				}
+			}
+		}
+		
+		// check attached source relations
+		List<Relation> entSrcRels = ent.getSourceRelations();
+		List<Relation> cacheSrcRels = relMap.getValuesByBKey(entId);
+		for (Relation rel: entSrcRels) {
+			if (!isRelCurrent(rel)) {
+				return false;
+			} else {
+				if (!cacheSrcRels.remove(rel)) {
+					return false;
+				}
+			}
+		}
+		if (!cacheSrcRels.isEmpty()) {
+			return false;
+		}
+		// check attached target relations
+		List<Relation> entTarRels = ent.getTargetRelations();
+		List<Relation> cacheTarRels = relMap.getValuesByCKey(entId);
+		for (Relation rel: entTarRels) {
+			if (!isRelCurrent(rel)) {
+				return false;
+			} else {
+				if (!cacheTarRels.remove(rel)) {
+					return false;
+				}
+			}
+		}
+		if (!cacheTarRels.isEmpty()) {
+			return false;
+		}	
+		
+		return true;
+	}
+
+	
+	public class EntityDiff {
+		public List<Attribute> addedAttributes = new ArrayList<Attribute>();
+		public List<Attribute> removedAttributes = new ArrayList<Attribute>();
+		public List<Attribute> modifiedAttributes = new ArrayList<Attribute>();
+		public List<Relation> addedSrcRels = new ArrayList<Relation>();
+		public List<Relation> removedSrcRels = new ArrayList<Relation>();
+		public List<Relation> modifiedSrcRels = new ArrayList<Relation>();
+		public List<Relation> addedTarRels = new ArrayList<Relation>();
+		public List<Relation> removedTarRels = new ArrayList<Relation>();
+		public List<Relation> modifiedTarRels = new ArrayList<Relation>();
+	}
+
+	/**
+	 * Returns an EntityDiff with the differences between the given Entity and the current cache.
+	 * 
+	 * Ignores empty Attributes. Compares only content if acceptNewIDs.
+	 * 
+	 * @param ent
+	 * @return
+	 * @throws Exception
+	 */
+	public EntityDiff diffEntityCache(Entity ent, boolean acceptNewIDs) throws Exception {
+		if (ent.isLightweight()) {
+			throw new Exception("Can not diff attributes and relations of lightweight entity!");
+		}
+		Long entId = ent.getId();
+		EntityDiff diff = new EntityDiff();
+		
+		/*
+		 * check attributes
+		 */
+		List<Attribute> entAtts = ent.getAttributes();
+		List<Attribute> cacheAtts = attMap.getValuesByBKey(entId);
+		// check attached attributes and compare to cached
+		for (Attribute att : entAtts) {
+			if (!isAttCurrent(att)) {
+				diff.modifiedAttributes.add(att);
+			}
+			if (!cacheAtts.remove(att)) {
+				boolean found = false;
+				for (Attribute ca : cacheAtts) {
+					if (ca.getId() == att.getId()) {
+						cacheAtts.remove(ca);
+						found = true;
+						break;
+					} else if (acceptNewIDs && ca.equalsContent(att)) {
+						// same content is good enough
+						cacheAtts.remove(ca);
+						found = true;
+						break;
+					}
+				}
+				if (!found) {
+					diff.addedAttributes.add(att);
+				}
+			}
+		}
+		if (!cacheAtts.isEmpty()) {
+			// more cached attributes
+			for (Attribute ca : cacheAtts) {
+				// ignore empty attributes
+				if (!StringUtils.isEmpty(ca.getOwnValue())) {
+					diff.removedAttributes.add(ca);
+				}
+			}
+		}
+		
+		/*
+		 * source relations
+		 */
+		List<Relation> entSrcRels = ent.getSourceRelations();
+		List<Relation> cacheSrcRels = relMap.getValuesByBKey(entId);
+		// check attached source relations and compare to cached
+		for (Relation rel : entSrcRels) {
+			if (!isRelCurrent(rel)) {
+				diff.modifiedSrcRels.add(rel);
+			}
+			if (!cacheSrcRels.remove(rel)) {
+				boolean found = false;
+				for (Relation cr : cacheSrcRels) {
+					if (cr.getId() == rel.getId()) {
+						cacheSrcRels.remove(cr);
+						found = true;
+						break;
+					} else if (acceptNewIDs && cr.equalsContent(rel)) {
+						// same content is good enough
+						cacheSrcRels.remove(cr);
+						found = true;
+						break;
+					}
+				}
+				if (!found) {
+					diff.addedSrcRels.add(rel);
+				}
+			}
+		}
+		if (!cacheSrcRels.isEmpty()) {
+			for (Relation rel : cacheSrcRels) {
+				diff.removedSrcRels.add(rel);
+			}
+		}
+		
+		/*
+		 * target relations
+		 */
+		List<Relation> entTarRels = ent.getTargetRelations();
+		List<Relation> cacheTarRels = relMap.getValuesByCKey(entId);
+		// check attached target relations and compare to cached
+		for (Relation rel : entTarRels) {
+			if (!isRelCurrent(rel)) {
+				diff.modifiedTarRels.add(rel);
+			}
+			if (!cacheTarRels.remove(rel)) {
+				boolean found = false;
+				for (Relation cr : cacheTarRels) {
+					if (cr.getId() == rel.getId()) {
+						cacheTarRels.remove(cr);
+						found = true;
+						break;
+					} else if (acceptNewIDs && cr.equalsContent(rel)) {
+						// same content is good enough
+						cacheTarRels.remove(cr);
+						found = true;
+						break;
+					}
+				}
+				if (!found) {
+					diff.addedTarRels.add(rel);
+				}
+			}
+		}
+		if (!cacheTarRels.isEmpty()) {
+			for (Relation rel : cacheTarRels) {
+				diff.removedTarRels.add(rel);
+			}
+		}
+		
+		// return null if diff is empty
+		if (diff.addedAttributes.isEmpty() && diff.removedAttributes.isEmpty() && diff.modifiedAttributes.isEmpty()
+				&& diff.addedSrcRels.isEmpty() && diff.removedSrcRels.isEmpty() && diff.modifiedSrcRels.isEmpty()
+				&& diff.addedTarRels.isEmpty() && diff.removedTarRels.isEmpty() && diff.modifiedTarRels.isEmpty()) {
+			return null;
+		}
+		return diff;
+	}
+	
+
+	
+	
 	
 	public List<ViewerAttribute> getViewerAttributes(Long page){
 		return getViewerAttMap().getValuesByAKey(page);
--- a/src/main/java/org/mpi/openmind/cache/WrapperService.java	Thu Jan 26 19:37:25 2017 +0100
+++ b/src/main/java/org/mpi/openmind/cache/WrapperService.java	Fri Jan 27 19:55:33 2017 +0100
@@ -11,6 +11,7 @@
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
+import org.mpi.openmind.cache.CacheService.EntityDiff;
 import org.mpi.openmind.repository.bo.Attribute;
 import org.mpi.openmind.repository.bo.Entity;
 import org.mpi.openmind.repository.bo.Node;
@@ -666,6 +667,8 @@
 		// check if entity is up to date with cache
 		if (!cache.isEntityCurrent(entity)) {
 			logger.error("Entity to save not up to date with cache!");
+			EntityDiff diff = cache.diffEntityCache(entity, true);
+			logger.debug(diff);
 		}
 		
 		// save in database
--- a/src/main/java/org/mpi/openmind/repository/bo/Attribute.java	Thu Jan 26 19:37:25 2017 +0100
+++ b/src/main/java/org/mpi/openmind/repository/bo/Attribute.java	Fri Jan 27 19:55:33 2017 +0100
@@ -178,6 +178,12 @@
 		this.possibleValues = possibleValues;
 	}
 
+	/**
+	 * Returns if the other Attribute has the same ownValue and name.
+	 * 
+	 * @param other
+	 * @return
+	 */
 	public boolean equalsContent(Attribute other){
 		if(other == null)
 			return false;