Mercurial > hg > openmind

changeset 108:bdd1c3fc0897

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Much faster DivaImportHttp that puts all entities in a Map first. Also uses ownValue instead of name attribute.
author Robert Casties <casties@mpiwg-berlin.mpg.de>
date Tue, 09 Jul 2019 07:30:50 +0200
parents 742347ef8410
children 8013b12cecf7
files src/main/java/org/mpi/openmind/scripts/DivaImportHttp.java
diffstat 1 files changed, 15 insertions(+), 13 deletions(-) [+]
line wrap: on
line diff
--- a/src/main/java/org/mpi/openmind/scripts/DivaImportHttp.java	Mon Jul 08 20:20:07 2019 +0200
+++ b/src/main/java/org/mpi/openmind/scripts/DivaImportHttp.java	Tue Jul 09 07:30:50 2019 +0200
@@ -2,6 +2,7 @@
 
 import java.io.InputStreamReader;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -22,7 +23,6 @@
 import org.mpi.openmind.repository.bo.Entity;
 import org.mpi.openmind.repository.bo.Node;
 import org.mpi.openmind.repository.services.ServiceRegistry;
-import org.mpi.openmind.repository.services.utils.AttributeFilter;
 
 
 /**
@@ -83,21 +83,23 @@
 		        // parse JSON directory index of manifest files
 		        JSONArray files = new JSONArray(jsonReader);
 		        int numFiles = files.length();
-	            List<Entity> list = new ArrayList<Entity>();
+		        // get all DIGITALIZATION entities
+		        System.out.println("Loading all DIGITALIZATIONs");
+		        List<Entity> digiList = omService.getEntitiesByDef(DIGITALIZATION);
+		        // unpack in Map by name/ownValue
+		        Map<String, Entity> digiMap = new HashMap<String, Entity>(digiList.size());
+		        for (Entity digi : digiList) {
+		        	String name = digi.getOwnValue();
+		        	digiMap.put(name, digi);
+		        }
+	            List<Entity> saveList = new ArrayList<Entity>();
 	            // go through all filenames in the list
 		        for (int i = 0; i < numFiles; ++i) {
 		        	JSONObject file = files.getJSONObject(i);
 		            String filename = file.getString("name");
                     System.out.println("check: "+filename);
-                    // create filter to search OpenMind
                     String digiName = filename.replace(".json", "");
-		            List<AttributeFilter> filters = new ArrayList<AttributeFilter>();
-		            AttributeFilter filter = new AttributeFilter("name", digiName, "DIGITALIZATION");
-		            filter.setExactMatch(true);
-					filters.add(filter);
-		            // get matching DIGITALIZATIONs
-                    Map<Entity, Attribute> res = omService.searchEntityByAttributeFilter(filters, -1);
-                    if (res.size() > 0) {
+                    if (digiMap.containsKey(digiName)) {
                         //System.out.println("  exists: "+res);
                     } else {
                         // no existing DIGITALIZATION - create new Entity
@@ -107,15 +109,15 @@
                         digi.addAttribute(new Attribute("name", "text", filename));
                         digi.addAttribute(new Attribute("num_files", "text", "100"));
                         // add to list
-                        list.add(digi);
+                        saveList.add(digi);
                     }
 		        }
 		        // ensure http entity is fully consumed
 		        EntityUtils.consume(htent);
 		        // persist OpenMind entities
-		        omService.saveEntityList(list, userName);
+		        omService.saveEntityList(saveList, userName);
                 System.out.println("Found " + numFiles + " manifests");
-	            System.out.println("Created " + list.size() + " DIGITALIZATIONs");
+	            System.out.println("Created " + saveList.size() + " DIGITALIZATIONs");
 	            System.out.println("END");
 		        
 		    } finally {