mpdl-group: software/mpdl-services/mpiwg-mpdl-lt/src/de/mpg/mpiwg/berlin/mpdl/lt/dict/db/DBLexWriter.java annotate

author	Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de>
date	Wed, 09 Nov 2011 15:32:05 +0100
parents
children

rev	line source
19 4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	1 package de.mpg.mpiwg.berlin.mpdl.lt.dict.db;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	2
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	3 import java.io.BufferedOutputStream;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	4 import java.io.BufferedReader;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	5 import java.io.File;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	6 import java.io.FileNotFoundException;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	7 import java.io.FileOutputStream;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	8 import java.io.FileReader;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	9 import java.io.IOException;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	10 import java.io.Reader;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	11 import java.io.StringReader;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	12 import java.io.UnsupportedEncodingException;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	13 import java.util.ArrayList;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	14 import java.util.Date;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	15 import java.util.HashMap;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	16 import java.util.Iterator;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	17 import java.util.List;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	18 import java.util.regex.Matcher;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	19 import java.util.regex.Pattern;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	20
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	21 import org.xml.sax.InputSource;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	22 import org.xml.sax.SAXException;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	23 import org.xml.sax.XMLReader;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	24
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	25 import com.sleepycat.je.Cursor;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	26 import com.sleepycat.je.Database;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	27 import com.sleepycat.je.DatabaseEntry;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	28 import com.sleepycat.je.DatabaseException;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	29 import com.sleepycat.je.LockMode;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	30 import com.sleepycat.je.OperationStatus;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	31 import com.sleepycat.je.util.DbLoad;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	32 import com.sun.org.apache.xerces.internal.parsers.SAXParser;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	33
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	34 import de.mpg.mpiwg.berlin.mpdl.util.StringUtils;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	35 import de.mpg.mpiwg.berlin.mpdl.exception.ApplicationException;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	36 import de.mpg.mpiwg.berlin.mpdl.lt.dict.app.Lexica;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	37 import de.mpg.mpiwg.berlin.mpdl.lt.dict.app.Lexicon;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	38 import de.mpg.mpiwg.berlin.mpdl.lt.dict.app.LexiconEntry;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	39 import de.mpg.mpiwg.berlin.mpdl.lt.general.Constants;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	40 import de.mpg.mpiwg.berlin.mpdl.lt.text.transcode.Transcoder;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	41
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	42 public class DBLexWriter {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	43 private static DBLexWriter instance;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	44 private static String DATA_DIR = Constants.getInstance().getDataDir();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	45 private static String DATA_FILES_DIR_LEXICA = DATA_DIR + "/dataFiles/pollux";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	46 private static String DB_DIR_LEXICA = DATA_DIR + "/dataBerkeleyDB/pollux";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	47 private DbEnvLex dbEnvLexica;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	48 private Date beginOfOperation;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	49 private Date endOfOperation;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	50
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	51 public static DBLexWriter getInstance() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	52 if (instance == null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	53 instance = new DBLexWriter();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	54 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	55 return instance;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	56 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	57
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	58 public static void main(String[] args) throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	59 getInstance();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	60 instance.beginOperation();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	61 System.out.print("Start ...");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	62 // instance.initReadOnly();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	63 instance.initReadWrite();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	64 // instance.readSampleData();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	65 // instance.testTranscoder();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	66 // instance.printSizeOfAllLexicons();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	67 instance.writeLexiconsToFiles();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	68 // instance.loadPolluxDbDumpsToDb();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	69 // instance.copyAndRepairAndTranscodeDumps();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	70 instance.end();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	71 instance.endOperation();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	72 // Double elapsedTime = new Util().getSecondWithMillisecondsBetween(instance.beginOfOperation, instance.endOfOperation);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	73 System.out.println("End.");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	74 // System.out.println("Needed time: " + elapsedTime + " seconds");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	75 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	76
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	77 private void initReadWrite() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	78 dbEnvLexica = new DbEnvLex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	79 dbEnvLexica.setDataDir(DB_DIR_LEXICA);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	80 dbEnvLexica.initReadWrite();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	81 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	82
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	83 private void initReadOnly() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	84 dbEnvLexica = new DbEnvLex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	85 dbEnvLexica.setDataDir(DB_DIR_LEXICA);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	86 dbEnvLexica.initReadOnly();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	87 ArrayList<Lexicon> lexicons = Lexica.getInstance().getLocalLexicons();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	88 for (int i=0; i<lexicons.size(); i++) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	89 Lexicon lexicon = lexicons.get(i);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	90 String lexiconName = lexicon.getName();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	91 dbEnvLexica.openDatabase(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	92 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	93 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	94
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	95 private void loadPolluxDbDumpsToDb() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	96 ArrayList<Lexicon> lexicons = Lexica.getInstance().getLocalLexicons();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	97 for (int i=0; i<lexicons.size(); i++) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	98 Lexicon lexicon = lexicons.get(i);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	99 String lexiconName = lexicon.getName();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	100 loadDbDumpToDb(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	101 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	102 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	103
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	104 private void loadDbDumpToDb(String lexiconName) throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	105 String dumpFileName = DATA_FILES_DIR_LEXICA + "/" + lexiconName + ".dump";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	106 String dbName = lexiconName + "Dump.db";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	107 try {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	108 BufferedReader bufferedReader = new BufferedReader(new FileReader(dumpFileName));
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	109 DbLoad loader = new DbLoad();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	110 loader.setEnv(dbEnvLexica.getEnv());
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	111 loader.setDbName(dbName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	112 loader.setInputReader(bufferedReader);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	113 loader.setIgnoreUnknownConfig(true);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	114 loader.load();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	115 bufferedReader.close();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	116 } catch (FileNotFoundException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	117 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	118 } catch (IOException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	119 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	120 } catch (DatabaseException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	121 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	122 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	123 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	124
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	125 private void readSampleData() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	126 try {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	127 List<String> dbNames = dbEnvLexica.getEnv().getDatabaseNames();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	128 String l1 = readEntry("autenrieth", "au)to/s");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	129 String l2 = readEntry("ls", "laudabilis");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	130 String l3 = readEntry("lsjUnicode", "ἄδρεπτος");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	131 String l4 = readEntry("salmoneUnicode", "ءرش");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	132 System.out.println("Autenrieth: autos: " + l1);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	133 System.out.println("Lewis & Short: Laudabilis: " + l2);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	134 System.out.println("LSJ: ἄδρεπτος: " + l3);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	135 System.out.println("Salmone: طب: " + l4);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	136 printSampleEntries("salmoneUnicode", 10);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	137 printSampleEntries("lsjUnicode", 1000);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	138 } catch (DatabaseException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	139 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	140 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	141 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	142
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	143 private void end() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	144 ArrayList<Lexicon> lexicons = Lexica.getInstance().getLocalLexicons();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	145 for (int i=0; i<lexicons.size(); i++) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	146 Lexicon lexicon = lexicons.get(i);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	147 String lexiconName = lexicon.getName();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	148 dbEnvLexica.closeDatabase(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	149 dbEnvLexica.closeDatabase(lexiconName + "Dump");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	150 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	151 dbEnvLexica.close();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	152 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	153
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	154 private String readEntry(String lexiconName, String formName) throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	155 String retString = null;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	156 try {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	157 String keyStr = formName;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	158 DatabaseEntry dbEntryKey = new DatabaseEntry(keyStr.getBytes("utf-8"));
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	159 Database lexDB = dbEnvLexica.getLexiconDB(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	160 Cursor cursor = lexDB.openCursor(null, null);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	161 DatabaseEntry foundValue = new DatabaseEntry();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	162 OperationStatus operationStatus = cursor.getSearchKey(dbEntryKey, foundValue, LockMode.DEFAULT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	163 if (operationStatus == OperationStatus.SUCCESS) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	164 byte[] foundValueBytes = foundValue.getData();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	165 retString = new String(foundValueBytes, "utf-8");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	166 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	167 cursor.close();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	168 } catch (DatabaseException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	169 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	170 } catch (UnsupportedEncodingException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	171 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	172 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	173 return retString;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	174 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	175
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	176 private void printSizeOfAllLexiconsTemp() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	177 String lexiconName = "lsj";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	178 int[] sizes = getSizes(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	179 System.out.println(lexiconName + ": " + sizes[0] + " records (" + sizes[1] + " of them are not xml valid)");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	180 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	181
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	182 private void printSizeOfAllLexicons() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	183 ArrayList<Lexicon> lexicons = Lexica.getInstance().getLocalLexicons();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	184 for (int i=0; i<lexicons.size(); i++) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	185 Lexicon lexicon = lexicons.get(i);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	186 String lexiconName = lexicon.getName();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	187 int[] sizes = getSizes(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	188 System.out.println(lexiconName + ": " + sizes[0] + " records (" + sizes[1] + " of them are not xml valid)");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	189 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	190 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	191
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	192 private int[] getSizes(String lexiconName) throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	193 int size = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	194 int sizeXmlNotValidEntries = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	195 try {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	196 dbEnvLexica.openDatabase(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	197 Database lexDB = dbEnvLexica.getLexiconDB(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	198 Cursor cursor = lexDB.openCursor(null, null);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	199 DatabaseEntry dbEntryKey = new DatabaseEntry();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	200 DatabaseEntry dbEntryValue = new DatabaseEntry();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	201 OperationStatus operationStatus = cursor.getFirst(dbEntryKey, dbEntryValue, LockMode.DEFAULT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	202 while (operationStatus == OperationStatus.SUCCESS) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	203 operationStatus = cursor.getNext(dbEntryKey, dbEntryValue, LockMode.DEFAULT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	204 byte[] dbEntryKeyBytes = dbEntryKey.getData();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	205 String dbEntryKeyStr = new String(dbEntryKeyBytes, "utf-8");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	206 byte[] dbEntryValueBytes = dbEntryValue.getData();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	207 String dbEntryValueStr = new String(dbEntryValueBytes, "utf-8");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	208 int begin = dbEntryValueStr.indexOf("<repaired-entry>");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	209 int end = dbEntryValueStr.indexOf("</repaired-entry>");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	210 dbEntryValueStr = dbEntryValueStr.substring(begin, end) + "</repaired-entry>";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	211 LexiconEntry dbLexEntry = new LexiconEntry(lexiconName, dbEntryKeyStr, dbEntryValueStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	212 LexiconEntry xmlLexiconEntry = xmlParse(dbLexEntry);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	213 if (! xmlLexiconEntry.isXmlValid()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	214 sizeXmlNotValidEntries ++;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	215 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	216 size++;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	217 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	218 cursor.close();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	219 } catch (DatabaseException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	220 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	221 } catch (UnsupportedEncodingException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	222 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	223 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	224 int[] sizes = new int[2];
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	225 sizes[0] = size;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	226 sizes[1] = sizeXmlNotValidEntries;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	227 return sizes;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	228 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	229
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	230 private void copyAndRepairAndTranscodeDumps() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	231 try {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	232 ArrayList<Lexicon> lexicons = Lexica.getInstance().getLocalLexicons();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	233 for (int i=0; i<lexicons.size(); i++) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	234 Lexicon lexicon = lexicons.get(i);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	235 String lexiconName = lexicon.getName();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	236 HashMap<String, DatabaseEntry> lexDumpHashMap = getWholeLexiconHashMap(lexiconName + "Dump");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	237 dbEnvLexica.openDatabase(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	238 Database lexDB = dbEnvLexica.getLexiconDB(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	239 Iterator<String> lexDumpIter = lexDumpHashMap.keySet().iterator();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	240 while (lexDumpIter.hasNext()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	241 String lexDumpKeyStr = lexDumpIter.next();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	242 DatabaseEntry lexDumpValue = lexDumpHashMap.get(lexDumpKeyStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	243 byte[] lexDumpValueBytes = lexDumpValue.getData();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	244 String lexDumpValueStr = new String(lexDumpValueBytes, "utf-8");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	245 String newLexValueStr = new String(lexDumpValueBytes, "utf-8");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	246 // repair lsj
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	247 if (lexiconName.equals("lsj")) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	248 newLexValueStr = newLexValueStr.replaceAll("<br>", "<br/>");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	249 newLexValueStr = newLexValueStr.replaceAll("<p>", "<p/>");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	250 String elementNameGreek = "G";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	251 newLexValueStr = deleteNestedTags(elementNameGreek, newLexValueStr); // delete tags <G> and </G> inside <G>
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	252 newLexValueStr = newLexValueStr.replaceAll("lang=greek", "lang=\"greek\"");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	253 boolean senseContained = newLexValueStr.matches(".<sense.>.*");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	254 boolean endSenseContained = newLexValueStr.matches(".</sense>.");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	255 if (senseContained && ! endSenseContained)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	256 newLexValueStr = newLexValueStr.replaceAll("<sense .*?>", "");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	257 else if (!senseContained && endSenseContained)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	258 newLexValueStr = newLexValueStr.replaceAll("</sense>", "");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	259 boolean refContained = newLexValueStr.matches(".<ref.>.*");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	260 boolean endRefContained = newLexValueStr.matches(".</ref>.");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	261 if (refContained && ! endRefContained)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	262 newLexValueStr = newLexValueStr.replaceAll("<ref .*?>", "");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	263 else if (!refContained && endRefContained)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	264 newLexValueStr = newLexValueStr.replaceAll("</ref>", "");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	265 /*
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	266 boolean itypeContained = newLexValueStr.matches(".<itype.>.*");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	267 boolean endItypeContained = newLexValueStr.matches(".</itype>.");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	268 if (itypeContained && ! endItypeContained)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	269 newLexValueStr = newLexValueStr.replaceAll("<itype .*?>", "");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	270 else if (!itypeContained && endItypeContained)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	271 newLexValueStr = newLexValueStr.replaceAll("</itype>", "");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	272 */
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	273 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	274 // repair cooper
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	275 if (lexiconName.equals("cooper")) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	276 newLexValueStr = newLexValueStr.replaceAll("<PB>", ""); // TODO hack
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	277 newLexValueStr = newLexValueStr.replaceAll("<p>", "<p/>"); // TODO hack
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	278 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	279 // repair baretti
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	280 if (lexiconName.equals("baretti")) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	281 newLexValueStr = newLexValueStr.replaceAll("<li>", "<li/>"); // TODO hack
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	282 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	283 // repair for all lexicons
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	284 newLexValueStr = newLexValueStr.replaceAll("type=style", "type=\"style\"");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	285 newLexValueStr = newLexValueStr.replaceAll("type=dom", "type=\"dom\"");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	286 newLexValueStr = newLexValueStr.replaceAll("<\\*>", "");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	287 newLexValueStr = newLexValueStr.replaceAll("<p />", "<p/>");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	288 LexiconEntry newLexEntryTemp = new LexiconEntry(lexiconName, lexDumpKeyStr, newLexValueStr); // lexDumpKeyStr is not transcoded yet but it will not be used in further in the code
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	289 LexiconEntry newLexEntry = xmlParseAndRepair(newLexEntryTemp);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	290 String xmlValidString = "<xml-valid>true</xml-valid>";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	291 if (! newLexEntry.isXmlValid()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	292 xmlValidString = "<xml-valid>false</xml-valid>";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	293 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	294 newLexValueStr = newLexEntry.getContent();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	295 // transcode the Betacode lexicon entries to Unicode (key and value)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	296 if (lexicon.isBetacodeLexicon()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	297 Transcoder transcoder = Transcoder.getInstance();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	298 lexDumpKeyStr = transcoder.transcodeFromBetaCode2Unicode(lexDumpKeyStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	299 String elementName = "G";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	300 if (newLexEntry.isXmlValid()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	301 newLexValueStr = transcodeByElementName("fromBetacode2Unicode", elementName, newLexValueStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	302 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	303 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	304 // transcode the Buckwalter entries to Unicode (key and value)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	305 if (lexicon.isBuckwalterLexicon()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	306 Transcoder transcoder = Transcoder.getInstance();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	307 lexDumpKeyStr = transcoder.transcodeFromBuckwalter2Unicode(lexDumpKeyStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	308 String elementName = "AR";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	309 if (newLexEntry.isXmlValid()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	310 newLexValueStr = transcodeByElementName("fromBuckwalter2Unicode", elementName, newLexValueStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	311 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	312 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	313 // put the entry into database
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	314 newLexValueStr = "<content>" + xmlValidString + "<original-entry>" + lexDumpValueStr + "</original-entry>" + "<repaired-entry>" + newLexValueStr + "</repaired-entry>" + "</content>";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	315 DatabaseEntry newLexDumpKey = new DatabaseEntry(lexDumpKeyStr.getBytes("utf-8"));
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	316 DatabaseEntry newLexValue = new DatabaseEntry(newLexValueStr.getBytes("utf-8"));
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	317 lexDB.put(null, newLexDumpKey, newLexValue);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	318 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	319 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	320 } catch (DatabaseException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	321 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	322 } catch (UnsupportedEncodingException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	323 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	324 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	325 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	326
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	327 private void printSampleEntries(String lexiconName, int count) throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	328 try {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	329 int counter = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	330 dbEnvLexica.openDatabase(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	331 Database lexDB = dbEnvLexica.getLexiconDB(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	332 Cursor cursor = lexDB.openCursor(null, null);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	333 DatabaseEntry dbEntryKey = new DatabaseEntry();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	334 DatabaseEntry dbEntryValue = new DatabaseEntry();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	335 OperationStatus operationStatus = cursor.getFirst(dbEntryKey, dbEntryValue, LockMode.DEFAULT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	336 while (operationStatus == OperationStatus.SUCCESS && counter < count) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	337 int size = dbEntryKey.getSize();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	338 if (size > 0) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	339 byte[] dbEntryKeyBytes = dbEntryKey.getData();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	340 String dbEntryKeyStr = new String(dbEntryKeyBytes, "utf-8");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	341 System.out.println(lexiconName + ": key: " + dbEntryKeyStr + " value size: " + dbEntryValue.getSize());
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	342 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	343 operationStatus = cursor.getNext(dbEntryKey, dbEntryValue, LockMode.DEFAULT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	344 counter++;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	345 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	346 cursor.close();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	347 } catch (DatabaseException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	348 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	349 } catch (UnsupportedEncodingException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	350 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	351 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	352 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	353
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	354 private void testTranscoder() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	355 String testStr = "<G>hfhf fdfd<G>ei)mi/</G> (<tr>sum</tr>), Aeol. <G>e)/mmi</G> hfhfh </G><author>Sapph.</author>2.15, <author>Theoc.</author>20.32; Cret. <G>h)mi/</G> <title>GDI</title> 4959a; <per>2</per><number>sg.</number> <G>ei)=</G>, Ep. and Ion. <cit><G>ei)s</G> <author>Od.</author>17.388</cit>, al., Aeol. <G>e)/ssi</G>, Ep. and Dor. <cit><G>e)ssi/</G> <author>Il.</author>1.176</cit>, <author>Pi.</author>";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	356 String testStr2 = "aaaaa <G>1111a <G>2222a</G> <G>3333a</G> 1111a</G> aaaaa bbbbb <G>1111b <G>2222b</G> <G>3333b</G> 1111b</G> bbbbb ";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	357 String testStr3 = "<G>e)pano/rqwsin e)/xein</G>, opp <G>a)ni/aton ei)=nai hi</G>3. 1165 b18. --<G>e)panorqw/seis kai boh/qeiai rb</G>5. 1383 a20.";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	358 String testStr4 = "<G>suni^hmi</G> <author>Ar.</author><title>Av.</title>946 (s. v.l.), <author>Strato Com.</author>1.3: with variation of quantity, <G>plei=ston ou)=lon i(/ei <G>[i^]</G>, i)/oulon i(/ei [i_</G>] <title>Carm.Pop.</title> 1.]:&#x2014" +
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	359 ";<br><tr>release, let go</tr>, <cit><G>h(=ka ..po/das kai\\ xei=re fe/resqai</G> <author>Od.</author>12.442</cit>; <G>h(=ke fe/resqai</G> <tr>let</tr> him float" +
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	360 "off, <author>Il.</author>21.120; <tr>let fall</tr>, <G>ka\\d de\\ ka/rhtos h(=ke ko/mas</G> <tr>made</tr> his locks <tr>flow</tr> down from his head, <author>Od.<" +
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	361 "/author>6.231; [<cit><G>e)qei/ras] i(/ei lo/fon a)mfi/</G> .... ggg";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	362 String testStr5 = "plei=ston ou)=lon i(/ei ";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	363 String testStr6 = "a as< as as: )a *s ss ";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	364 Transcoder t = Transcoder.getInstance();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	365 String transcoded = t.transcodeFromBetaCode2Unicode(testStr4);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	366 transcoded = t.transcodeFromBetaCode2Unicode(testStr5);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	367 transcoded = t.transcodeFromBetaCode2Unicode(testStr6);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	368
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	369 String arabTestStr1 = "^nutaf";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	370 String arabTestStr2 = "min";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	371 String arabTestStr3 = "Aal-Hiyal (^qAla ^>arisTwTAlys) yataEaj~aba Aal-nAs minhA <im~A fy Aal->a$yA' Aal~aty taEriDu TabEAF fa-mim~A lA yuElamu Eil~atuhu wa-<im~A fy Aal->a$yA' Aal-muxAlifap li-l-TabE fa-mim~A yuEmalu bi-Aal-SinAEap li-manfaEap Aal-nAs li->an~a Aal-TabyEap tulzimu >abadAF jihap wAHidap wa->am~A manAfiE Aal-nAs fa-<in~ahA taxtalifu <ixtilAfAF kavyrAF.";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	372 transcoded = t.transcodeFromBuckwalter2Unicode(arabTestStr1);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	373 transcoded = t.transcodeFromBuckwalter2Unicode(arabTestStr2);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	374 transcoded = t.transcodeFromBuckwalter2Unicode(arabTestStr3);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	375
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	376 // String deletedNestedTags = deleteNestedTags("G", testStr4);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	377 // String regExpr = "(<G>.?)<G>(.?)</G>(.?)<G>(.?)</G>(.*?</G>)";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	378 String regExpr = "(<G>.?)<G>(.)(</G>){1,}(.*?</G>)";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	379 // String regExpr = "(<G>.?)<G>(.?)</G>(.?)<G>(.?)</G>(.*?</G>)";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	380 String replaceStr = testStr2.replaceAll(regExpr, "$1$2$4");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	381 // String replaceStr2 = testStr2.replaceAll("<G>(.)<G>(.)</G>(.)<G>(.)</G>(.*)</G>", "<G>$2$3$4$5</G>");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	382 regExpr = "<G>.?(<G>.?</G>){1,}.*?</G>";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	383 regExpr = "(<G>.?)<G>(.?)</G>(.?){1,}(.?</G>)";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	384 // String regExpr = "[a-zA-Z0-9]+?\\[.+?\\]/" + "\|" + "[a-zA-Z0-9]+?/" + "\|" + "[a-zA-Z0-9]+?\\[.+\\]$" + "\|" + "[a-zA-Z0-9]+?$"; // pathName example: "/archimedes[@xmlns:xlink eq "http://www.w3.org/1999/xlink"]/text/body/chap/p[@type eq "main"]/s/foreign[@lang eq "en"]"
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	385 Pattern p = Pattern.compile(regExpr, Pattern.CASE_INSENSITIVE \| Pattern.MULTILINE); // both flags enabled
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	386 Matcher m = p.matcher(testStr2);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	387 while (m.find()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	388 int msBeginPos = m.start();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	389 int msEndPos = m.end();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	390 String matchStr = testStr2.substring(msBeginPos, msEndPos);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	391 String bla = "";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	392 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	393
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	394 String retStr = transcodeByElementName("fromBetacode2Unicode", "G", testStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	395 retStr = transcodeByElementName("fromBetacode2Unicode", "G", "bla");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	396 retStr = transcodeByElementName("fromBetacode2Unicode", "G", "");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	397 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	398
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	399 private String transcodeByElementName(String transcodeDirection, String elementName, String inputStr) throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	400 if (inputStr == null \|\| elementName == null)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	401 return null;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	402 String elemBeginTag = "<" + elementName + ">";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	403 String elemEndTag = "</" + elementName + ">";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	404 Transcoder transcoder = Transcoder.getInstance();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	405 String outputStr = "";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	406 int begin = inputStr.indexOf(elemBeginTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	407 int end = inputStr.indexOf(elemEndTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	408 while (begin != -1 && end != -1 && begin < end) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	409 String before = inputStr.substring(0, begin);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	410 String origStr = inputStr.substring(begin + elemBeginTag.length(), end);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	411 origStr = StringUtils.deleteSpecialXmlEntities(origStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	412 String transcodedStr = origStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	413 if (transcodeDirection.equals("fromBetacode2Unicode"))
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	414 transcodedStr = transcoder.transcodeFromBetaCode2Unicode(origStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	415 else if (transcodeDirection.equals("fromBuckwalter2Unicode"))
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	416 transcodedStr = transcoder.transcodeFromBuckwalter2Unicode(origStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	417 outputStr = outputStr + before + new String(elemBeginTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	418 outputStr = outputStr + transcodedStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	419 outputStr = outputStr + new String(elemEndTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	420 inputStr = inputStr.substring(end + elemEndTag.length());
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	421 begin = inputStr.indexOf(elemBeginTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	422 end = inputStr.indexOf(elemEndTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	423 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	424 outputStr = outputStr + inputStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	425 return outputStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	426 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	427
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	428 private String deleteNestedTags(String elementName, String inputStr) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	429 String inputStrTmp = new String(inputStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	430 String elemBeginTag = "<" + elementName + ">";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	431 String elemEndTag = "</" + elementName + ">";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	432 String outputStr = "";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	433 int begin = inputStrTmp.indexOf(elemBeginTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	434 int end = inputStrTmp.indexOf(elemEndTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	435 while (begin != -1 && end != -1) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	436 end = getIndexClosedTag(begin, elementName, inputStrTmp);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	437 String before = inputStrTmp.substring(0, begin);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	438 String origStr = null;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	439 if (end == -1) // if no end tag could be found
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	440 origStr = inputStrTmp.substring(begin + elemBeginTag.length(), inputStrTmp.length());
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	441 else
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	442 origStr = inputStrTmp.substring(begin + elemBeginTag.length(), end);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	443 origStr = origStr.replaceAll(elemBeginTag, "");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	444 origStr = origStr.replaceAll(elemEndTag, "");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	445 outputStr = outputStr + before + new String(elemBeginTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	446 outputStr = outputStr + origStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	447 outputStr = outputStr + new String(elemEndTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	448 inputStrTmp = inputStrTmp.substring(end + elemEndTag.length());
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	449 begin = inputStrTmp.indexOf(elemBeginTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	450 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	451 outputStr = outputStr + inputStrTmp;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	452 return outputStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	453 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	454
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	455 private int getIndexClosedTag(int begin, String elementName, String inputStr) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	456 int beginTmp = begin;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	457 int retIndex = -1;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	458 String elemBeginTag = "<" + elementName + ">";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	459 String elemEndTag = "</" + elementName + ">";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	460 int indexEndTag = inputStr.indexOf(elemEndTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	461 while (indexEndTag != -1) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	462 String betweenTmpStr = inputStr.substring(beginTmp + elemBeginTag.length(), indexEndTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	463 int indexBeginTag = betweenTmpStr.indexOf(elemBeginTag);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	464 if (indexBeginTag != -1) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	465 beginTmp = indexEndTag;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	466 } else {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	467 return indexEndTag;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	468 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	469 indexEndTag = inputStr.indexOf(elemEndTag, indexEndTag + elemEndTag.length());
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	470 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	471 return retIndex;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	472 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	473
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	474 private HashMap<String, DatabaseEntry> getWholeLexiconHashMap(String lexiconName) throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	475 HashMap<String, DatabaseEntry> lexHashMap = new HashMap<String, DatabaseEntry>();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	476 try {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	477 dbEnvLexica.openDatabase(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	478 Database lexDB = dbEnvLexica.getLexiconDB(lexiconName);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	479 Cursor cursor = lexDB.openCursor(null, null);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	480 DatabaseEntry dbEntryKey = new DatabaseEntry();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	481 DatabaseEntry dbEntryValue = new DatabaseEntry();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	482 OperationStatus operationStatus = cursor.getFirst(dbEntryKey, dbEntryValue, LockMode.DEFAULT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	483 while (operationStatus == OperationStatus.SUCCESS) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	484 int size = dbEntryKey.getSize();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	485 if (size > 0) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	486 byte[] dbEntryKeyBytes = dbEntryKey.getData();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	487 String dbEntryKeyStr = new String(dbEntryKeyBytes, "utf-8");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	488 DatabaseEntry newDbEntryValue = new DatabaseEntry(dbEntryValue.getData());
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	489 lexHashMap.put(dbEntryKeyStr, newDbEntryValue);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	490 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	491 operationStatus = cursor.getNext(dbEntryKey, dbEntryValue, LockMode.DEFAULT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	492 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	493 cursor.close();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	494 } catch (DatabaseException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	495 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	496 } catch (UnsupportedEncodingException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	497 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	498 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	499 return lexHashMap;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	500 }

19

4a3641ae14d2 Erstellung