Index: /trunk/src/de/mpiwg/itgroup/fulltext/ocropus/OCRFulltextSearchXML.java =================================================================== --- /trunk/src/de/mpiwg/itgroup/fulltext/ocropus/OCRFulltextSearchXML.java (revision 2116) +++ /trunk/src/de/mpiwg/itgroup/fulltext/ocropus/OCRFulltextSearchXML.java (revision 2118) @@ -5,4 +5,5 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; @@ -57,5 +58,5 @@ private HashMap searchThreads = null; - static private HashMap generalThreads; + static private HashMap generalThreads; //static private HashMap wrapperThreads; @@ -172,6 +173,12 @@ if (generalThreads==null) { - generalThreads = new HashMap(); - } + generalThreads = new HashMap(); + } + + if (notEnoughMemory()){ + generalThreads.put(ticket,null); + return ticket; + } + ArrayList ll = new ArrayList(); System.out.println("languages:"+languages); @@ -213,5 +220,10 @@ if (generalThreads==null) { - generalThreads = new HashMap(); + generalThreads = new HashMap(); + } + + if (notEnoughMemory()){ + generalThreads.put(ticket,null); + return ticket; } @@ -230,13 +242,36 @@ - System.err.println("THREAD:"+generalThreads); + System.err.println("THREADS:"+generalThreads.size()); return ticket; } + //test ob noch genug heapSpace fuer neuen thread, annahme mindestens 20 000 000 Bytes. + + private boolean notEnoughMemory() { + long fm = Runtime.getRuntime().freeMemory(); + if (fm < 20000000){ + System.err.println("not enough memory:"+String.valueOf(fm)); + return true; + } + + + return false; + + } + public String getSearchResult(String ticket){ System.out.println("myTicket:"+ticket); System.out.println(this.generalThreads); - SearchAndShowThread st = generalThreads.get(ticket); + + Object stObj = generalThreads.get(ticket); + SearchAndShowThread st=null; + + if (SearchAndShowThread.class.isInstance(stObj)) + st = (SearchAndShowThread)generalThreads.get(ticket); + + else + return ""; + if (st.returnValue==null){ //System..println("ret: WAIT"); @@ -245,8 +280,26 @@ } else { - String txt= st.returnValue; + //String txt= st.returnValue; + File retFile = st.returnValue; + byte[] buffer = new byte[(int) retFile.length()]; + FileInputStream f; + try { + f = new FileInputStream(retFile); + f.read(buffer); + f.close(); + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + System.err.println("getSearchResult: Cannot find temporary file with results:"+retFile.getPath()); + } catch (IOException e) { + System.err.println("getSearchResult: IO error:"+retFile.getPath()); + e.printStackTrace(); + } + retFile.delete(); + + generalThreads.remove(ticket); //System.err.println("ret:"+txt); - return txt; + return new String(buffer); + //return txt; } Index: /trunk/src/de/mpiwg/itgroup/fulltext/ocropus/SearchAndShowThread.java =================================================================== --- /trunk/src/de/mpiwg/itgroup/fulltext/ocropus/SearchAndShowThread.java (revision 2116) +++ /trunk/src/de/mpiwg/itgroup/fulltext/ocropus/SearchAndShowThread.java (revision 2118) @@ -1,4 +1,6 @@ package de.mpiwg.itgroup.fulltext.ocropus; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; @@ -15,5 +17,5 @@ private HashMap results; - public String returnValue = null; + public File returnValue = null; public String currentLanguage = null; public int counter = 0; @@ -87,6 +89,57 @@ } - private String wrapResultsInXMLShort(HashMap results) throws CorruptIndexException, IOException + private File wrapResultsInXMLShort(HashMap results) throws CorruptIndexException, IOException { + long mem0 = Runtime.getRuntime().freeMemory(); + long mem1; + + File tempFile = File.createTempFile("temp", ".tmp" ); + + FileWriter fw = new FileWriter(tempFile); + + //String ret=""; + fw.write(""); + for (String lang: results.keySet()) + //gehe durch alle Sprachen + { + currentLanguage=lang; + + HashMap> linesInPath = new HashMap>(); + HashMap> boxesInPath = new HashMap>(); + HashMap> dimInPath = new HashMap>(); + HashMap dcMetaDataInPath = new HashMap(); + fw.write(""); + analyseResults(results, lang, linesInPath, boxesInPath, dimInPath,dcMetaDataInPath); + //String mdString = dcMetaDataInPath.toString(); + //System.out.println("mdString:"+mdString.getBytes("utf-8")); + fw.write(""+dcMetaDataInPath.toString()+""); + + //System.out.println("wrapResultsInXMLShort: Analised the results:"+lang); + //setze das jetzt in XML + + fw.write(""+pack(linesInPath)+""); + fw.write(""+pack(boxesInPath)+""); + fw.write(""+pack(dimInPath)+""); + + System.out.println("wrapResultsInXMLShort: Wrapped the results:"+lang + ); + fw.write(""); + } + System.out.println("wrapResultsInXMLShort: Wrapped the results"); + mem1 = Runtime.getRuntime().freeMemory(); + Runtime.getRuntime().gc(); + System.out.println("free memory changed:"+String.valueOf(mem1-mem0)); + System.out.println("free memory current:"+String.valueOf(mem1)); + fw.write(""); + fw.close(); + return tempFile; + } + + private String wrapResultsInXMLShortMem(HashMap results) throws CorruptIndexException, IOException + { + long mem0 = Runtime.getRuntime().totalMemory() - + Runtime.getRuntime().freeMemory(); + + String ret=""; for (String lang: results.keySet()) @@ -117,4 +170,8 @@ } System.out.println("wrapResultsInXMLShort: Wrapped the results"); + long mem1 = Runtime.getRuntime().totalMemory() - + Runtime.getRuntime().freeMemory(); + + System.out.println("memory:"+String.valueOf(mem1-mem0)); return ret+""; } @@ -133,9 +190,9 @@ // TODO Auto-generated catch block e1.printStackTrace(); - returnValue =""; + returnValue =null; } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); - returnValue =""; + returnValue =null; } try { @@ -143,9 +200,9 @@ } catch (CorruptIndexException e) { e.printStackTrace(); - returnValue =""; + returnValue =null; } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); - returnValue =""; + returnValue =null; } Index: /trunk/src/de/mpiwg/itgroup/fulltext/ocropus/SearchAndShowThreadMem.java =================================================================== --- /trunk/src/de/mpiwg/itgroup/fulltext/ocropus/SearchAndShowThreadMem.java (revision 2118) +++ /trunk/src/de/mpiwg/itgroup/fulltext/ocropus/SearchAndShowThreadMem.java (revision 2118) @@ -0,0 +1,213 @@ +package de.mpiwg.itgroup.fulltext.ocropus; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.search.Hit; +import org.apache.lucene.search.Hits; + +public class SearchAndShowThreadMem implements Runnable { + + private HashMap results; + + public String returnValue = null; + public String currentLanguage = null; + public int counter = 0; + private OCRFulltextSearch searcher; + private String mdString; + private String searchString; + + private ArrayList languages; + + public SearchAndShowThreadMem(OCRFulltextSearch searcher, String mdString, String searchString, ArrayList languages){ + this.searcher = searcher; + this.mdString=mdString; + this.searchString=searchString; + if (languages.size()==0) + languages=null; + this.languages=languages; + } + public SearchAndShowThreadMem(OCRFulltextSearch searcher, String searchString) { + this.searcher = searcher; + this.mdString=null; + this.searchString=searchString; + } + public SearchAndShowThreadMem(OCRFulltextSearch searcher, + String searchString, ArrayList ll) { + + this.searcher = searcher; + this.mdString=null; + this.searchString=searchString; + if (ll.size()==0) + languages=null; + this.languages=ll; + + } + private void analyseResults(HashMap results, String lang, + HashMap> linesInPath, + HashMap> boxesInPath, + HashMap> dimInPath, + HashMap dcMetaDataInPath) + throws CorruptIndexException, IOException { + Iterator hitIterator = results.get(lang).iterator(); + System.out.println("starting analysis"); + + counter = 0; + while (hitIterator.hasNext()){ + counter+=1; + //jetzt durch alle Treffer und baue fuer jeden Pfad drei Arrays fuer Zeilennummer des Teffers, die Box, und die Seitengroesse enthalten + Hit hit = hitIterator.next(); + String path = hit.get("path"); + if (!linesInPath.keySet().contains(path)) + { + linesInPath.put(path, new ArrayList()); + boxesInPath.put(path, new ArrayList()); + dimInPath.put(path, new ArrayList()); + } + String ln = hit.get("lineNumber"); + linesInPath.get(path).add(ln); + String box = hit.get("bbox"); + try { // falls dcMetaData feld existiert fŸlle dies ansonsten tue nichts. + String md = hit.get("dcMetaData"); + dcMetaDataInPath.put(path,md); + } catch (RuntimeException e) { + + } + + + boxesInPath.get(path).add(box); + String pageDimension = hit.get("pageDimension"); + dimInPath.get(path).add(pageDimension); + } + System.out.println("starting analysis: done"); + } + + private File wrapResultsInXMLShort(HashMap results) throws CorruptIndexException, IOException + { + long mem0 = Runtime.getRuntime().totalMemory() - + Runtime.getRuntime().freeMemory(); + + File tempFile = File.createTempFile("temp", ".tmp" ); + + FileWriter fw = new FileWriter(tempFile); + + //String ret=""; + fw.write(""); + for (String lang: results.keySet()) + //gehe durch alle Sprachen + { + currentLanguage=lang; + + HashMap> linesInPath = new HashMap>(); + HashMap> boxesInPath = new HashMap>(); + HashMap> dimInPath = new HashMap>(); + HashMap dcMetaDataInPath = new HashMap(); + fw.write(""); + analyseResults(results, lang, linesInPath, boxesInPath, dimInPath,dcMetaDataInPath); + //String mdString = dcMetaDataInPath.toString(); + //System.out.println("mdString:"+mdString.getBytes("utf-8")); + fw.write(""+dcMetaDataInPath.toString()+""); + + //System.out.println("wrapResultsInXMLShort: Analised the results:"+lang); + //setze das jetzt in XML + + fw.write(""+pack(linesInPath)+""); + fw.write(""+pack(boxesInPath)+""); + fw.write(""+pack(dimInPath)+""); + + System.out.println("wrapResultsInXMLShort: Wrapped the results:"+lang + ); + fw.write(""); + } + System.out.println("wrapResultsInXMLShort: Wrapped the results"); + long mem1 = Runtime.getRuntime().totalMemory() - + Runtime.getRuntime().freeMemory(); + + System.out.println("memory:"+String.valueOf(mem1-mem0)); + fw.write(""); + fw.close(); + return tempFile; + } + + private String wrapResultsInXMLShortMem(HashMap results) throws CorruptIndexException, IOException + { + long mem0 = Runtime.getRuntime().totalMemory() - + Runtime.getRuntime().freeMemory(); + + + String ret=""; + for (String lang: results.keySet()) + //gehe durch alle Sprachen + { + currentLanguage=lang; + + HashMap> linesInPath = new HashMap>(); + HashMap> boxesInPath = new HashMap>(); + HashMap> dimInPath = new HashMap>(); + HashMap dcMetaDataInPath = new HashMap(); + ret+=""; + analyseResults(results, lang, linesInPath, boxesInPath, dimInPath,dcMetaDataInPath); + //String mdString = dcMetaDataInPath.toString(); + //System.out.println("mdString:"+mdString.getBytes("utf-8")); + ret+=""+dcMetaDataInPath.toString()+""; + + //System.out.println("wrapResultsInXMLShort: Analised the results:"+lang); + //setze das jetzt in XML + + ret+= ""+pack(linesInPath)+""; + ret+= ""+pack(boxesInPath)+""; + ret+= ""+pack(dimInPath)+""; + + System.out.println("wrapResultsInXMLShort: Wrapped the results:"+lang + ); + ret+=""; + } + System.out.println("wrapResultsInXMLShort: Wrapped the results"); + long mem1 = Runtime.getRuntime().totalMemory() - + Runtime.getRuntime().freeMemory(); + + System.out.println("memory:"+String.valueOf(mem1-mem0)); + return ret+""; + } + + private String pack(HashMap> linesInPath) { + + return linesInPath.toString(); + } + public void run() { + try { + if (languages!=null) + if (languages.size()==0) // wenn keine Sprache ausgewaehlt, dann Ubergebe null == alle Sprachen + languages=null; + results = searcher.searchMD(searchString,mdString,languages); + } catch (ParseException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + returnValue =""; + } catch (IOException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + returnValue =""; + } + try { + returnValue = wrapResultsInXMLShort(results); + } catch (CorruptIndexException e) { + e.printStackTrace(); + returnValue =""; + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + returnValue =""; + + } + + + } + +}