Mercurial > hg > mpdl-group

comparison software/mpdl-services/mpiwg-mpdl-lt/src/de/mpg/mpiwg/berlin/mpdl/lt/morph/app/MorphologyCache.java @ 23:e845310098ba

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
diverse Korrekturen
author Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de>
date Tue, 27 Nov 2012 12:35:19 +0100
parents 7d6d969b10cf
children
comparison
equal deleted inserted replaced
22:6a45a982c333 23:e845310098ba
23 private static Logger LOGGER = Logger.getLogger(MorphologyCache.class.getName()); 23 private static Logger LOGGER = Logger.getLogger(MorphologyCache.class.getName());
24 private static String DATA_DIR = Constants.getInstance().getDataDir(); 24 private static String DATA_DIR = Constants.getInstance().getDataDir();
25 private static String DB_DIR_DONATUS = DATA_DIR + "/dataBerkeleyDB/donatus"; 25 private static String DB_DIR_DONATUS = DATA_DIR + "/dataBerkeleyDB/donatus";
26 public static int QUERY_MODE = 0; 26 public static int QUERY_MODE = 0;
27 public static int DOCUMENT_MODE = 1; 27 public static int DOCUMENT_MODE = 1;
28 private static long MIN_RAM = 500000000;
28 private static int MAX_HASHTABLE_SIZE = Constants.MORPHOLOGY_CACHE_SIZE; 29 private static int MAX_HASHTABLE_SIZE = Constants.MORPHOLOGY_CACHE_SIZE;
30 private Date touchTimer;
29 protected int mode = QUERY_MODE; 31 protected int mode = QUERY_MODE;
30 private Hashtable<String, Hashtable<String, Lemma>> forms = new Hashtable<String, Hashtable<String, Lemma>>(); // cache of forms: hashKey is formName 32 private Hashtable<String, Hashtable<String, Lemma>> forms = new Hashtable<String, Hashtable<String, Lemma>>(); // cache of forms: hashKey is formName
31 private Hashtable<String, Lemma> lemmas = new Hashtable<String, Lemma>(); // cache of lemmas: hashKey is lemmaName 33 private Hashtable<String, Lemma> lemmas = new Hashtable<String, Lemma>(); // cache of lemmas: hashKey is lemmaName
32 private DBMorphHandler dbMorphHandlerStatic; // handles static morph data (BerkeleyDB) 34 private DBMorphHandler dbMorphHandler; // handles morph data (BerkeleyDB)
33 private Date beginOfOperation; 35 private Date beginOfOperation;
34 private Date endOfOperation; 36 private Date endOfOperation;
35 37
36 public static MorphologyCache getInstance() throws ApplicationException { 38 public static MorphologyCache getInstance() throws ApplicationException {
37 if (instance == null) { 39 if (instance == null) {
40 } 42 }
41 return instance; 43 return instance;
42 } 44 }
43 45
44 private void init() throws ApplicationException { 46 private void init() throws ApplicationException {
47 long maxMemory = Runtime.getRuntime().maxMemory();
48 if (maxMemory < MIN_RAM) {
49 String message = "Morphology cache: at least " + MIN_RAM + " is needed as heap space: please start java with parameter -Xmx with more than this value)";
50 LOGGER.severe(message);
51 throw new ApplicationException(message);
52 }
53 touchTimer = new Date();
45 instance.beginOperation(); 54 instance.beginOperation();
46 dbMorphHandlerStatic = new DBMorphHandler(DB_DIR_DONATUS); 55 dbMorphHandler = new DBMorphHandler(DB_DIR_DONATUS);
47 dbMorphHandlerStatic.start(); 56 dbMorphHandler.startReadOnly();
48 dbMorphHandlerStatic.openDatabases(); 57 dbMorphHandler.openDatabases();
49 instance.endOperation(); 58 instance.endOperation();
50 Double elapsedTime = new Util().getSecondWithMillisecondsBetween(instance.beginOfOperation, instance.endOfOperation); 59 Double elapsedTime = new Util().getSecondWithMillisecondsBetween(instance.beginOfOperation, instance.endOfOperation);
51 LOGGER.info("Morphology db cache: opened (needed " + elapsedTime + " seconds)"); 60 LOGGER.info("Morphology cache: morphology db opened read only (needed " + elapsedTime + " seconds, heap space: " + maxMemory + " bytes)");
52 } 61 }
53 62
54 public int getMode() { 63 public int getMode() {
55 return mode; 64 return mode;
56 } 65 }
58 public void setMode(int newMode) { 67 public void setMode(int newMode) {
59 this.mode = newMode; 68 this.mode = newMode;
60 } 69 }
61 70
62 public void end() throws ApplicationException { 71 public void end() throws ApplicationException {
63 dbMorphHandlerStatic.closeDatabases(); 72 dbMorphHandler.closeDatabases();
64 LOGGER.info("Morphology db cache: closed"); 73 LOGGER.info("Morphology cache: db closed");
65 } 74 forms = null;
66 75 lemmas = null;
67 /* 76 dbMorphHandler = null;
68 public ArrayList<Lemma> getLemmasByFormName(String lang, String formName, boolean normalize) throws ApplicationException { 77 instance = null;
69 return getLemmasByFormName(lang, formName, normalize, Normalizer.DISPLAY); 78 }
70 } 79
71 */
72
73 public ArrayList<Lemma> getLemmasByFormName(String lang, String formNameArg, int normMode) throws ApplicationException { 80 public ArrayList<Lemma> getLemmasByFormName(String lang, String formNameArg, int normMode) throws ApplicationException {
74 String language = Language.getInstance().getLanguageId(lang); 81 String language = Language.getInstance().getLanguageId(lang);
75 ArrayList<Lemma> retFormLemmas = null; 82 ArrayList<Lemma> retFormLemmas = null;
76 String formName = formNameArg; 83 String formName = formNameArg;
77 Normalizer normalizer = new Normalizer(language); 84 Normalizer normalizer = new Normalizer(language);
82 Hashtable<String, Lemma> formLemmasHashtable = forms.get(key); 89 Hashtable<String, Lemma> formLemmasHashtable = forms.get(key);
83 if (formLemmasHashtable == null) { 90 if (formLemmasHashtable == null) {
84 ArrayList<Lemma> dbFormLemmas = readLemmasByFormName(language, formName); 91 ArrayList<Lemma> dbFormLemmas = readLemmasByFormName(language, formName);
85 // put lemmas into local cache 92 // put lemmas into local cache
86 int localHashTableSize = forms.size(); 93 int localHashTableSize = forms.size();
87 if (localHashTableSize >= MAX_HASHTABLE_SIZE) { 94 Date now = new Date();
88 clearCache(); 95 if (now.getTime() - touchTimer.getTime() > 900000) { // is true each 0,25 hours: then free memory is fetched (needs some time)
96 touchTimer = new Date();
97 long freeMemory = Runtime.getRuntime().freeMemory();
98 LOGGER.info(touchTimer + ": Morphology cache: free memory in heap space: " + freeMemory + " bytes");
99 if (freeMemory < MIN_RAM || localHashTableSize >= MAX_HASHTABLE_SIZE) { // if freeMemory is less then MIN_RAM then clear cache to get some new memory
100 clearCache();
101 freeMemory = Runtime.getRuntime().freeMemory();
102 LOGGER.info(touchTimer + ": Morphology cache: cache cleared, free memory in heap space: " + freeMemory + " bytes");
103 }
89 } 104 }
90 if (dbFormLemmas != null && ! dbFormLemmas.isEmpty()) { 105 if (dbFormLemmas != null && ! dbFormLemmas.isEmpty()) {
91 formLemmasHashtable = new Hashtable<String, Lemma>(); 106 formLemmasHashtable = new Hashtable<String, Lemma>();
92 for (int i=0; i<dbFormLemmas.size(); i++) { 107 for (int i=0; i<dbFormLemmas.size(); i++) {
93 Lemma lemma = dbFormLemmas.get(i); 108 Lemma lemma = dbFormLemmas.get(i);
269 lemmas = new Hashtable<String, Lemma>(); 284 lemmas = new Hashtable<String, Lemma>();
270 } 285 }
271 286
272 private ArrayList<Lemma> readLemmasByFormName(String lang, String formName) throws ApplicationException { 287 private ArrayList<Lemma> readLemmasByFormName(String lang, String formName) throws ApplicationException {
273 String language = Language.getInstance().getLanguageId(lang); 288 String language = Language.getInstance().getLanguageId(lang);
274 ArrayList<Lemma> lemmasStatic = dbMorphHandlerStatic.readLemmas(language, formName); 289 ArrayList<Lemma> lemmasStatic = dbMorphHandler.readLemmas(language, formName);
275 return lemmasStatic; 290 return lemmasStatic;
276 } 291 }
277 292
278 private ArrayList<Form> readFormsByLemmaName(String lang, String lemmaName) throws ApplicationException { 293 private ArrayList<Form> readFormsByLemmaName(String lang, String lemmaName) throws ApplicationException {
279 String language = Language.getInstance().getLanguageId(lang); 294 String language = Language.getInstance().getLanguageId(lang);
280 ArrayList<Form> formsStatic = dbMorphHandlerStatic.readForms(language, lemmaName); 295 ArrayList<Form> formsStatic = dbMorphHandler.readForms(language, lemmaName);
281 return formsStatic; 296 return formsStatic;
282 } 297 }
283 298
284 private ArrayList<String> getVariantsFromLuceneQuery(String queryString) { 299 private ArrayList<String> getVariantsFromLuceneQuery(String queryString) {
285 LuceneUtil luceneUtil = LuceneUtil.getInstance(); 300 LuceneUtil luceneUtil = LuceneUtil.getInstance();