Changeset 2118

Timestamp:

Nov 18, 2009, 8:46:32 AM (16 years ago)

Author:

dwinter

Message:

Neue Version, die Zwischenergebnisse in temporäre Files schreibt, anstelle diese im Speicher zu halten.

Location:

trunk/src/de/mpiwg/itgroup/fulltext/ocropus

Files:

• OCRFulltextSearchXML.java (modified) (6 diffs)
• SearchAndShowThread.java (modified) (6 diffs)
• SearchAndShowThreadMem.java (added)

trunk/src/de/mpiwg/itgroup/fulltext/ocropus/OCRFulltextSearchXML.java

@@ -5 +5 @@
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileReader;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -57 +58 @@
         private HashMap<String,OCRSearchThread> searchThreads = null;
 
-        static private HashMap<String, SearchAndShowThread> generalThreads;
+        static private HashMap<String, Object> generalThreads;
 
         //static private HashMap<String,WrapperThread> wrapperThreads;
@@ -172 +173 @@
                 if (generalThreads==null)
                 {
-                        generalThreads = new HashMap<String,SearchAndShowThread>();
-                }
+                        generalThreads = new HashMap<String,Object>();
+                }
+                
+                if (notEnoughMemory()){
+                        generalThreads.put(ticket,null);
+                        return ticket;
+                }
+                
                 ArrayList<String> ll = new ArrayList<String>();
                 System.out.println("languages:"+languages);
@@ -213 +220 @@
                 if (generalThreads==null)
                 {
-                        generalThreads = new HashMap<String,SearchAndShowThread>();
+                        generalThreads = new HashMap<String,Object>();
+                }
+                
+                if (notEnoughMemory()){
+                        generalThreads.put(ticket,null);
+                        return ticket;
                 }
                 
@@ -230 +242 @@
                 
                 
-                System.err.println("THREAD:"+generalThreads);   
+                System.err.println("THREADS:"+generalThreads.size());   
                 return ticket;
                 
         }
         
+        //test ob noch genug heapSpace fuer neuen thread, annahme mindestens 20 000 000 Bytes.
+        
+        private boolean notEnoughMemory() {
+                long fm = Runtime.getRuntime().freeMemory();
+                if (fm < 20000000){
+                        System.err.println("not enough memory:"+String.valueOf(fm));
+                        return true;
+                }
+                        
+                
+                return false;
+                
+        }
+
         public String getSearchResult(String ticket){
                 System.out.println("myTicket:"+ticket);
                 System.out.println(this.generalThreads);
-                SearchAndShowThread st = generalThreads.get(ticket);
+                
+                Object stObj = generalThreads.get(ticket);
+                SearchAndShowThread st=null;
+                
+                if (SearchAndShowThread.class.isInstance(stObj))
+                        st = (SearchAndShowThread)generalThreads.get(ticket);
+                
+                else
+                        return "<error value=\"not enough memory\" valueNum=\"1\"/>";
+                        
                 if (st.returnValue==null){
                         //System..println("ret: WAIT");
@@ -245 +280 @@
                 }
                 else {
-                        String txt= st.returnValue;
+                        //String txt= st.returnValue;
+                        File retFile = st.returnValue;
+                        byte[] buffer = new byte[(int) retFile.length()];
+                        FileInputStream f;
+                        try {
+                                f = new FileInputStream(retFile);
+                                f.read(buffer);
+                                f.close();
+                        } catch (FileNotFoundException e) {
+                                // TODO Auto-generated catch block
+                                System.err.println("getSearchResult: Cannot find temporary file with results:"+retFile.getPath());
+                        }  catch (IOException e) {
+                                System.err.println("getSearchResult: IO error:"+retFile.getPath());
+                                e.printStackTrace();
+                        }
+                        retFile.delete();
+                        
+
                         generalThreads.remove(ticket);
                         //System.err.println("ret:"+txt);
-                        return txt;
+                        return new String(buffer);
+                        //return txt;
                         
                 }

trunk/src/de/mpiwg/itgroup/fulltext/ocropus/SearchAndShowThread.java

@@ -1 +1 @@
 package de.mpiwg.itgroup.fulltext.ocropus;
 
+import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -15 +17 @@
         private HashMap<String, Hits> results;
 
-        public String returnValue = null;
+        public File returnValue = null;
         public String currentLanguage = null;
         public int counter = 0;
@@ -87 +89 @@
         }
 
-        private String wrapResultsInXMLShort(HashMap<String, Hits> results) throws CorruptIndexException, IOException
+        private File wrapResultsInXMLShort(HashMap<String, Hits> results) throws CorruptIndexException, IOException
         {
+                long mem0 = Runtime.getRuntime().freeMemory();
+                long mem1;
+                
+                File tempFile = File.createTempFile("temp", ".tmp" );
+                
+                FileWriter fw = new FileWriter(tempFile);
+                
+                //String ret="<searchresult encoding=\"utf-8\">";
+                fw.write("<searchresult encoding=\"utf-8\">");
+                for (String lang: results.keySet())
+                        //gehe durch alle Sprachen
+                {
+                        currentLanguage=lang;
+                        
+                        HashMap<String, ArrayList<String>> linesInPath = new HashMap<String,ArrayList<String>>();
+                        HashMap<String, ArrayList<String>> boxesInPath = new HashMap<String,ArrayList<String>>();
+                        HashMap<String, ArrayList<String>> dimInPath = new HashMap<String,ArrayList<String>>();
+                        HashMap<String, String> dcMetaDataInPath = new HashMap<String,String>();
+                        fw.write("<results lang=\""+lang+"\"><resultsShort>");
+                        analyseResults(results, lang, linesInPath, boxesInPath, dimInPath,dcMetaDataInPath);
+                        //String mdString = dcMetaDataInPath.toString();
+                        //System.out.println("mdString:"+mdString.getBytes("utf-8"));
+                        fw.write("<dcMetaDatas>"+dcMetaDataInPath.toString()+"</dcMetaDatas>");
+                        
+                        //System.out.println("wrapResultsInXMLShort: Analised the results:"+lang);
+                        //setze das jetzt in XML
+                                        
+                                fw.write("<lines>"+pack(linesInPath)+"</lines>");
+                                fw.write("<boxes>"+pack(boxesInPath)+"</boxes>");
+                                fw.write("<dims>"+pack(dimInPath)+"</dims>");                           
+                        
+                        System.out.println("wrapResultsInXMLShort: Wrapped the results:"+lang
+                        );
+                        fw.write("</resultsShort></results>");
+                }
+                System.out.println("wrapResultsInXMLShort: Wrapped the results");
+                mem1 = Runtime.getRuntime().freeMemory();
+                Runtime.getRuntime().gc();
+                System.out.println("free memory changed:"+String.valueOf(mem1-mem0));
+                System.out.println("free memory current:"+String.valueOf(mem1));
+                fw.write("</searchresult>");
+                fw.close();
+                return tempFile;
+        }
+
+        private String wrapResultsInXMLShortMem(HashMap<String, Hits> results) throws CorruptIndexException, IOException
+        {
+                long mem0 = Runtime.getRuntime().totalMemory() -
+              Runtime.getRuntime().freeMemory();
+
+        
                 String ret="<searchresult encoding=\"utf-8\">";
                 for (String lang: results.keySet())
@@ -117 +170 @@
                 }
                 System.out.println("wrapResultsInXMLShort: Wrapped the results");
+                long mem1 = Runtime.getRuntime().totalMemory() -
+              Runtime.getRuntime().freeMemory();
+
+                System.out.println("memory:"+String.valueOf(mem1-mem0));
                 return ret+"</searchresult>";
         }
@@ -133 +190 @@
                         // TODO Auto-generated catch block
                         e1.printStackTrace();
-                        returnValue ="";
+                        returnValue =null;
                 } catch (IOException e1) {
                         // TODO Auto-generated catch block
                         e1.printStackTrace();
-                        returnValue ="";
+                        returnValue =null;
                 }
                 try {
@@ -143 +200 @@
                 } catch (CorruptIndexException e) {
                         e.printStackTrace();
-                        returnValue ="";
+                        returnValue =null;
                 } catch (IOException e) {
                         // TODO Auto-generated catch block
                         e.printStackTrace();
-                        returnValue ="";
+                        returnValue =null;
                         
                 }