mpdl-group: software/mpdl-services/mpiwg-mpdl-lt/src/de/mpg/mpiwg/berlin/mpdl/lt/text/norm/Normalizer.java annotate

author	Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de>
date	Wed, 09 Nov 2011 15:32:05 +0100
parents
children	7d6d969b10cf

rev	line source
19 4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	1 package de.mpg.mpiwg.berlin.mpdl.lt.text.norm;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	2
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	3 import java.io.IOException;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	4 import java.io.StringReader;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	5 import java.util.ArrayList;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	6
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	7 import de.mpg.mpiwg.berlin.mpdl.exception.ApplicationException;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	8 import de.mpg.mpiwg.berlin.mpdl.lt.general.Language;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	9 import de.mpg.mpiwg.berlin.mpdl.lt.text.norm.lang.MpdlNormalizerLexAR;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	10 import de.mpg.mpiwg.berlin.mpdl.lt.text.norm.lang.MpdlNormalizerLexDE;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	11 import de.mpg.mpiwg.berlin.mpdl.lt.text.norm.lang.MpdlNormalizerLexEL;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	12 import de.mpg.mpiwg.berlin.mpdl.lt.text.norm.lang.MpdlNormalizerLexEN;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	13 import de.mpg.mpiwg.berlin.mpdl.lt.text.norm.lang.MpdlNormalizerLexFR;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	14 import de.mpg.mpiwg.berlin.mpdl.lt.text.norm.lang.MpdlNormalizerLexIT;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	15 import de.mpg.mpiwg.berlin.mpdl.lt.text.norm.lang.MpdlNormalizerLexLA;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	16 import de.mpg.mpiwg.berlin.mpdl.lt.text.norm.lang.MpdlNormalizerLexNL;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	17 import de.mpg.mpiwg.berlin.mpdl.lt.text.norm.lang.MpdlNormalizerLexZH;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	18 import de.mpg.mpiwg.berlin.mpdl.lt.text.reg.Regularization;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	19 import de.mpg.mpiwg.berlin.mpdl.lt.text.reg.RegularizationManager;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	20
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	21 public class Normalizer {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	22 public static int DISPLAY = 1; // normalization in DISPLAY mode
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	23 public static int DICTIONARY = 2; // normalization in DICTIONARY mode
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	24 public static int SEARCH = 3; // normalization in SEARCH mode; never used so far in indexing because it does not support the morph. lexicons such as CELEX (e.g. eingeschränkt would not be stemmed to eingeschraenkt)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	25 private int normMode = DISPLAY; // Default e.g. for indexing and querying
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	26 private String[] normFunctions = {"norm"}; // default is to use the norm function
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	27 private String language;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	28 private int[] offsets;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	29
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	30 public Normalizer(String[] normFunctions, String lang) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	31 this.normFunctions = normFunctions;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	32 String language = Language.getInstance().getLanguageId(lang);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	33 this.language = language;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	34 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	35
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	36 public Normalizer(String language) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	37 this.language = language;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	38 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	39
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	40 public String getLanguage() {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	41 return language;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	42 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	43
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	44 public void setNormMode(int normMode) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	45 this.normMode = normMode;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	46 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	47
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	48 /**
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	49 * Applies the normalization rules in <code>language</code> to
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	50 * <code>s</code>, without offset tracking.
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	51 *
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	52 * @param s source string
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	53 * @return normalized string
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	54 */
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	55 public String normalize(String s) throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	56 String normStr = s;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	57 if (useSpecialNormFunction())
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	58 normStr = removeSpecialNWDMarks(normStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	59 if (useRegFunction()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	60 // try to regularize the string to the norm form over predefined regularizations
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	61 RegularizationManager regManager = RegularizationManager.getInstance();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	62 ArrayList<Regularization> regs = regManager.findRegsByOrig(language, s);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	63 if (regs != null && regs.size() > 0) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	64 Regularization reg = regs.get(0); // only one: the first one
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	65 String regNormStr = reg.getNorm();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	66 normStr = regNormStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	67 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	68 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	69 if (useNormFunction()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	70 // normalize the string by string replacements
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	71 if (normMode == DICTIONARY) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	72 normStr = normalize(normStr, DICTIONARY);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	73 } else if (normMode == DISPLAY) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	74 normStr = normalize(normStr, DISPLAY);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	75 } else if (normMode == SEARCH) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	76 normStr = normalize(normStr, SEARCH);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	77 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	78 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	79 if (useSpecialNormFunction())
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	80 normStr = insertSpecialNWDMarks(normStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	81 return normStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	82 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	83
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	84 private boolean useRegFunction() {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	85 boolean useReg = false;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	86 for (int i=0; i< normFunctions.length; i++) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	87 String function = normFunctions[i];
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	88 if (function.equals("reg"))
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	89 return true;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	90 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	91 return useReg;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	92 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	93
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	94 private boolean useNormFunction() {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	95 boolean useNorm = false;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	96 for (int i=0; i< normFunctions.length; i++) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	97 String function = normFunctions[i];
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	98 if (function.equals("norm") \|\| function.equals("specialNorm"))
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	99 return true;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	100 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	101 return useNorm;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	102 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	103
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	104 private boolean useSpecialNormFunction() {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	105 boolean useNorm = false;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	106 for (int i=0; i< normFunctions.length; i++) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	107 String function = normFunctions[i];
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	108 if (function.equals("specialNorm"))
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	109 return true;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	110 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	111 return useNorm;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	112 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	113
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	114 private String normalize(String s, int mode) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	115 String inputStr = s;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	116 StringReader strReader = new StringReader(inputStr + "\n");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	117 String retStr = "";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	118 String token = "";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	119 try {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	120 if (Language.getInstance().isLatin(language)) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	121 MpdlNormalizerLexLA mpdlNormalizerLex = new MpdlNormalizerLexLA(strReader);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	122 if (mode == DISPLAY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	123 mpdlNormalizerLex.yybegin(MpdlNormalizerLexLA.DISP);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	124 else if (mode == DICTIONARY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	125 mpdlNormalizerLex.yybegin(MpdlNormalizerLexLA.DICT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	126 else if (mode == SEARCH)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	127 mpdlNormalizerLex.yybegin(MpdlNormalizerLexLA.SEARCH);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	128 while (token != null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	129 token = mpdlNormalizerLex.yylex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	130 if (token != null)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	131 retStr += token;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	132 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	133 } else if (Language.getInstance().isArabic(language)) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	134 MpdlNormalizerLexAR mpdlNormalizerLex = new MpdlNormalizerLexAR(strReader);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	135 if (mode == DISPLAY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	136 mpdlNormalizerLex.yybegin(MpdlNormalizerLexAR.DISP);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	137 else if (mode == DICTIONARY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	138 mpdlNormalizerLex.yybegin(MpdlNormalizerLexAR.DICT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	139 else if (mode == SEARCH)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	140 mpdlNormalizerLex.yybegin(MpdlNormalizerLexAR.SEARCH);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	141 while (token != null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	142 token = mpdlNormalizerLex.yylex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	143 if (token != null)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	144 retStr += token;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	145 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	146 } else if (Language.getInstance().isGerman(language)) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	147 MpdlNormalizerLexDE mpdlNormalizerLex = new MpdlNormalizerLexDE(strReader);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	148 if (mode == DISPLAY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	149 mpdlNormalizerLex.yybegin(MpdlNormalizerLexDE.DISP);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	150 else if (mode == DICTIONARY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	151 mpdlNormalizerLex.yybegin(MpdlNormalizerLexDE.CELEX);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	152 else if (mode == SEARCH)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	153 mpdlNormalizerLex.yybegin(MpdlNormalizerLexDE.SEARCH);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	154 while (token != null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	155 token = mpdlNormalizerLex.yylex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	156 if (token != null)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	157 retStr += token;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	158 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	159 } else if (Language.getInstance().isGreek(language)) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	160 MpdlNormalizerLexEL mpdlNormalizerLex = new MpdlNormalizerLexEL(strReader);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	161 if (mode == DISPLAY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	162 mpdlNormalizerLex.yybegin(MpdlNormalizerLexEL.DISP);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	163 else if (mode == DICTIONARY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	164 mpdlNormalizerLex.yybegin(MpdlNormalizerLexEL.SIGMA);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	165 else if (mode == SEARCH)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	166 mpdlNormalizerLex.yybegin(MpdlNormalizerLexEL.SEARCH);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	167 while (token != null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	168 token = mpdlNormalizerLex.yylex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	169 if (token != null)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	170 retStr += token;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	171 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	172 } else if (Language.getInstance().isEnglish(language)) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	173 MpdlNormalizerLexEN mpdlNormalizerLex = new MpdlNormalizerLexEN(strReader);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	174 if (mode == DISPLAY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	175 mpdlNormalizerLex.yybegin(MpdlNormalizerLexEN.DISP);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	176 else if (mode == DICTIONARY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	177 mpdlNormalizerLex.yybegin(MpdlNormalizerLexEN.DICT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	178 else if (mode == SEARCH)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	179 mpdlNormalizerLex.yybegin(MpdlNormalizerLexEN.SEARCH);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	180 while (token != null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	181 token = mpdlNormalizerLex.yylex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	182 if (token != null)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	183 retStr += token;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	184 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	185 } else if (Language.getInstance().isFrench(language)) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	186 MpdlNormalizerLexFR mpdlNormalizerLex = new MpdlNormalizerLexFR(strReader);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	187 if (mode == DISPLAY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	188 mpdlNormalizerLex.yybegin(MpdlNormalizerLexFR.DISP);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	189 else if (mode == DICTIONARY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	190 mpdlNormalizerLex.yybegin(MpdlNormalizerLexFR.DICT_ASCII);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	191 else if (mode == SEARCH)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	192 mpdlNormalizerLex.yybegin(MpdlNormalizerLexFR.SEARCH);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	193 while (token != null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	194 token = mpdlNormalizerLex.yylex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	195 if (token != null)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	196 retStr += token;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	197 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	198 } else if (Language.getInstance().isItalian(language)) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	199 MpdlNormalizerLexIT mpdlNormalizerLex = new MpdlNormalizerLexIT(strReader);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	200 if (mode == DISPLAY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	201 mpdlNormalizerLex.yybegin(MpdlNormalizerLexIT.DISP);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	202 else if (mode == DICTIONARY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	203 mpdlNormalizerLex.yybegin(MpdlNormalizerLexIT.DICT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	204 else if (mode == SEARCH)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	205 mpdlNormalizerLex.yybegin(MpdlNormalizerLexIT.SEARCH);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	206 while (token != null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	207 token = mpdlNormalizerLex.yylex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	208 if (token != null)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	209 retStr += token;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	210 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	211 } else if (Language.getInstance().isDutch(language)) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	212 MpdlNormalizerLexNL mpdlNormalizerLex = new MpdlNormalizerLexNL(strReader);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	213 if (mode == DISPLAY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	214 mpdlNormalizerLex.yybegin(MpdlNormalizerLexNL.DISP);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	215 else if (mode == DICTIONARY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	216 mpdlNormalizerLex.yybegin(MpdlNormalizerLexNL.DICT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	217 else if (mode == SEARCH)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	218 mpdlNormalizerLex.yybegin(MpdlNormalizerLexNL.SEARCH);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	219 while (token != null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	220 token = mpdlNormalizerLex.yylex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	221 if (token != null)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	222 retStr += token;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	223 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	224 } else if (Language.getInstance().isChinese(language)) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	225 MpdlNormalizerLexZH mpdlNormalizerLex = new MpdlNormalizerLexZH(strReader);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	226 if (mode == DISPLAY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	227 mpdlNormalizerLex.yybegin(MpdlNormalizerLexZH.DISP);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	228 else if (mode == DICTIONARY)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	229 mpdlNormalizerLex.yybegin(MpdlNormalizerLexZH.DICT);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	230 else if (mode == SEARCH)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	231 mpdlNormalizerLex.yybegin(MpdlNormalizerLexZH.SEARCH);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	232 while (token != null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	233 token = mpdlNormalizerLex.yylex();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	234 if (token != null)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	235 retStr += token;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	236 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	237 } else {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	238 retStr = s; // return the string unchanged
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	239 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	240 } catch (IOException e ) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	241 // nothing cause IOException is not needed for a StringReader
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	242 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	243 return retStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	244 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	245
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	246
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	247 // used only in XmlTokenizerContentHandler // TODO make it better
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	248 private String removeSpecialNWDMarks(String inputString) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	249 String COMPLEX_ELEMENT_NWD_MARK = new Character('\u2424').toString(); // not word delimiting element
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	250 String cleanedWord = inputString;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	251 boolean startsWithNWDMark = cleanedWord.startsWith(COMPLEX_ELEMENT_NWD_MARK);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	252 if (startsWithNWDMark)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	253 cleanedWord = cleanedWord.substring(1);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	254 int countNWDMarks = cleanedWord.length() - cleanedWord.replaceAll(COMPLEX_ELEMENT_NWD_MARK, "").length();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	255 if (countNWDMarks > 1)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	256 cleanedWord = cleanedWord.replaceAll(COMPLEX_ELEMENT_NWD_MARK + "+", COMPLEX_ELEMENT_NWD_MARK);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	257 // boolean notHyphenPlusNWD = cleanedWord.matches(".[^-]+" + COMPLEX_ELEMENT_NWD_MARK + "+."); // e.g. "praebi ta"
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	258 // if (notHyphenPlusNWD)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	259 // cleanedWord = cleanedWord.replaceAll("([^-]+)" + COMPLEX_ELEMENT_NWD_MARK + "+", "$1-" + COMPLEX_ELEMENT_NWD_MARK); // e.g. "praebi ta" is replaced by "praebi- ta"
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	260 cleanedWord = cleanedWord.replaceAll(COMPLEX_ELEMENT_NWD_MARK, " ");
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	261 return cleanedWord;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	262 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	263
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	264 private String insertSpecialNWDMarks(String inputString) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	265 String COMPLEX_ELEMENT_NWD_MARK = new Character('\u2424').toString(); // not word delimiting element
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	266 String retStr = inputString;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	267 boolean startsWithNWDMark = retStr.startsWith(COMPLEX_ELEMENT_NWD_MARK);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	268 int countNWDMarks = retStr.length() - retStr.replaceAll(COMPLEX_ELEMENT_NWD_MARK, "").length();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	269 retStr = retStr.replaceAll(" ", COMPLEX_ELEMENT_NWD_MARK);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	270 // if (notHyphenPlusNWD)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	271 // normalizedWordStr = normalizedWordStr.replaceAll("-" + COMPLEX_ELEMENT_NWD_MARK, COMPLEX_ELEMENT_NWD_MARK); // e.g. "praebi- ta" is replaced by "praebi ta"
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	272 if (countNWDMarks > 1) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	273 String nwdStr = "";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	274 for (int i=0; i<countNWDMarks; i++)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	275 nwdStr += COMPLEX_ELEMENT_NWD_MARK;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	276 retStr = retStr.replaceAll(COMPLEX_ELEMENT_NWD_MARK, nwdStr);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	277 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	278 if (startsWithNWDMark)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	279 retStr = COMPLEX_ELEMENT_NWD_MARK + retStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	280 return retStr;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	281 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	282
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	283 /**
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	284 * Old code from Arboreal (Malcolm Hyman)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	285 * Applies the normalization rules in <code>language</code> to
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	286 * <code>s</code>, with offset tracking.<p>
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	287 *
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	288 * <strong>WARNING:</strong>
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	289 * Arboreal will not work properly if a normalization substitution
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	290 * replaces a source character with more than two target characters!
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	291 * This is simply a BUG, and should be fixed. Fortunately, however,
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	292 * one does not often need such a replacement.<p>
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	293 *
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	294 * @param s source string
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	295 * @param offsets character offset table
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	296 * @return normalized string
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	297 */
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	298 private String normalize4Lexica(String s, int[] offsets) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	299 this.offsets = offsets;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	300 if (language.equals("la") \|\| language.equals("lat")) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	301 StringBuffer buf = new StringBuffer();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	302 int n = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	303 for (int i = 0; i < s.length(); i++) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	304 char c = s.charAt(i);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	305 String replace = new String();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	306 switch (c) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	307 case 'j': replace = "i"; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	308 case 'v': replace = "u"; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	309 /*
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	310 * Linguistic note: /u/ and /v/ are rarely phonemic
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	311 * in Latin, as in alui 's/he nourished' vs.
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	312 * alvi 'of a belly', volui 's/he wished' or 'it rolled'
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	313 * vs. volvi 'to be rolled', (in)seruit 's/he joined
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	314 * together' vs. (in)servit 's/he serves'.
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	315 */
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	316 case 'q':
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	317 if ((i < s.length() - 1) && (s.charAt(i + 1) == ';'))
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	318 replace = "qu";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	319 else
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	320 replace = "q";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	321 break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	322 case ';':
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	323 if ((i > 0) && (s.charAt(i - 1) == 'q'))
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	324 replace = "e";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	325 else if ((i == 0) \|\| ! Character.isLetter(s.charAt(i - 1)))
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	326 replace = ";";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	327 else
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	328 replace = "";
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	329 break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	330 case '\u0300': replace = ""; break; // COMBINING GRAVE ACCENT
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	331 case '\u0301': replace = ""; break; // COMBINING ACCUTE ACCENT
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	332 case '\u0302': replace = ""; break; // COMBINING CIRCUMFLEX ACCENT
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	333
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	334 case '\u00c0': replace = "A"; break; // LATIN CAPITAL LETTER A GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	335 case '\u00c1': replace = "A"; break; // LATIN CAPITAL LETTER A ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	336 case '\u00c2': replace = "A"; break; // LATIN CAPITAL LETTER A CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	337 case '\u00c4': replace = "A"; break; // LATIN CAPITAL LETTER A DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	338 case '\u00c6': replace = "Ae"; break; // LATIN CAPITAL LETTER A E
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	339 case '\u00c7': replace = "C"; break; // LATIN CAPITAL LETTER C CEDILLA
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	340 case '\u00c8': replace = "E"; break; // LATIN CAPITAL LETTER E GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	341 case '\u00c9': replace = "E"; break; // LATIN CAPITAL LETTER E ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	342 case '\u00ca': replace = "E"; break; // LATIN CAPITAL LETTER E CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	343 case '\u00cb': replace = "E"; break; // LATIN CAPITAL LETTER E DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	344 case '\u00cc': replace = "I"; break; // LATIN CAPITAL LETTER I GRAVE;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	345 case '\u00cd': replace = "I"; break; // LATIN CAPITAL LETTER I ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	346 case '\u00ce': replace = "I"; break; // LATIN CAPITAL LETTER I CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	347 case '\u00cf': replace = "I"; break; // LATIN CAPITAL LETTER I DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	348 case '\u00d2': replace = "O"; break; // LATIN CAPITAL LETTER O GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	349 case '\u00d3': replace = "O"; break; // LATIN CAPITAL LETTER O ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	350 case '\u00d4': replace = "O"; break; // LATIN CAPITAL LETTER O CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	351 case '\u00d6': replace = "O"; break; // LATIN CAPITAL LETTER O DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	352 case '\u00d9': replace = "U"; break; // LATIN CAPITAL LETTER U GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	353 case '\u00da': replace = "U"; break; // LATIN CAPITAL LETTER U ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	354 case '\u00db': replace = "U"; break; // LATIN CAPITAL LETTER U CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	355 case '\u00dc': replace = "U"; break; // LATIN CAPITAL LETTER U DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	356 case '\u00e0': replace = "a"; break; // LATIN SMALL LETTER A GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	357 case '\u00e1': replace = "a"; break; // LATIN SMALL LETTER A ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	358 case '\u00e2': replace = "a"; break; // LATIN SMALL LETTER A CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	359 case '\u00e4': replace = "a"; break; // LATIN SMALL LETTER A DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	360 case '\u00e6': replace = "ae"; break; // LATIN SMALL LETTER A E
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	361 case '\u00e7': replace = "c"; break; // LATIN SMALL LETTER C CEDILLA
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	362 case '\u00e8': replace = "e"; break; // LATIN SMALL LETTER E GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	363 case '\u00e9': replace = "e"; break; // LATIN SMALL LETTER E ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	364 case '\u00ea': replace = "e"; break; // LATIN SMALL LETTER E CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	365 case '\u00eb': replace = "e"; break; // LATIN SMALL LETTER E DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	366 case '\u00ec': replace = "i"; break; // LATIN SMALL LETTER I GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	367 case '\u00ed': replace = "i"; break; // LATIN SMALL LETTER I ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	368 case '\u00ee': replace = "i"; break; // LATIN SMALL LETTER I CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	369 case '\u00ef': replace = "i"; break; // LATIN SMALL LETTER I DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	370 case '\u00f2': replace = "o"; break; // LATIN SMALL LETTER O GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	371 case '\u00f3': replace = "o"; break; // LATIN SMALL LETTER O ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	372 case '\u00f4': replace = "o"; break; // LATIN SMALL LETTER O CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	373 case '\u00f6': replace = "o"; break; // LATIN SMALL LETTER O DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	374 case '\u00f9': replace = "u"; break; // LATIN SMALL LETTER U GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	375 case '\u00fa': replace = "u"; break; // LATIN SMALL LETTER U ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	376 case '\u00fb': replace = "u"; break; // LATIN SMALL LETTER U CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	377 case '\u00fc': replace = "u"; break; // LATIN SMALL LETTER U DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	378 case '\u0100': replace = "A"; break; // LATIN CAPITAL LETTER A MACRON
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	379 case '\u0101': replace = "a"; break; // LATIN SMALL LETTER A MACRON
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	380 case '\u0102': replace = "A"; break; // LATIN CAPITAL LETTER A BREVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	381 case '\u0103': replace = "a"; break; // LATIN SMALL LETTER A BREVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	382 case '\u0112': replace = "E"; break; // LATIN CAPITAL LETTER E MACRON
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	383 case '\u0113': replace = "e"; break; // LATIN SMALL LETTER E MACRON
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	384 case '\u0114': replace = "E"; break; // LATIN CAPITAL LETTER E BREVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	385 case '\u0115': replace = "e"; break; // LATIN SMALL LETTER E BREVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	386 case '\u0118': replace = "Ae"; break; // LATIN CAPITAL LETTER E OGONEK
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	387 case '\u0119': replace = "ae"; break; // LATIN SMALL LETTER E OGONEK
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	388 case '\u012a': replace = "I"; break; // LATIN CAPITAL LETTER I MACRON
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	389 case '\u012b': replace = "i"; break; // LATIN SMALL LETTER I MACRON
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	390 case '\u012c': replace = "I"; break; // LATIN CAPITAL LETTER I BREVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	391 case '\u012d': replace = "i"; break; // LATIN SMALL LETTER I BREVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	392 case '\u014c': replace = "O"; break; // LATIN CAPITAL LETTER O MACRON
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	393 case '\u014d': replace = "o"; break; // LATIN SMALL LETTER O MACRON
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	394 case '\u014e': replace = "O"; break; // LATIN CAPITAL LETTER O BREVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	395 case '\u014f': replace = "o"; break; // LATIN SMALL LETTER O BREVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	396 case '\u0152': replace = "Oe"; break; // LATIN CAPITAL LETTER O E
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	397 case '\u0153': replace = "oe"; break; // LATIN SMALL LETTER O E
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	398 case '\u016a': replace = "U"; break; // LATIN CAPITAL LETTER U MACRON
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	399 case '\u016b': replace = "u"; break; // LATIN SMALL LETTER U MACRON
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	400 case '\u016c': replace = "U"; break; // LATIN CAPITAL LETTER U BREVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	401 case '\u016d': replace = "u"; break; // LATIN SMALL LETTER U BREVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	402 case '\u017f': replace = "s"; break; // LATIN SMALL LETTER LONG S
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	403 case '\u00df': replace = "ss"; break; // LATIN SMALL LETTER SHARP S
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	404 case '\u00ad': break; // soft hyphen
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	405 // new in MPDL project by J. Willenborg
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	406 case '\u1e14': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	407 case '\u1e15': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	408 case '\u1e16': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	409 case '\u1e17': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	410 case '\u1e18': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	411 case '\u1e19': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	412 case '\u1e1a': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	413 case '\u1e1b': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	414 case '\u1e1c': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	415 case '\u1e1d': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	416 case '\u1eb8': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	417 case '\u1eb9': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	418 case '\u1eba': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	419 case '\u1ebb': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	420 case '\u1ebc': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	421 case '\u1ebd': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	422 case '\u1ebe': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	423 case '\u1ebf': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	424 case '\u1ec0': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	425 case '\u1ec1': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	426 case '\u1ec2': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	427 case '\u1ec3': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	428 case '\u1ec4': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	429 case '\u1ec5': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	430 case '\u1ec6': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	431 case '\u1ec7': replace = "e"; break; // LATIN ... LETTER E WITH ...
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	432 // by Malcolm
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	433 case '\u2329': break; // BRA
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	434 case '\u232a': break; // KET
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	435 default: replace += c; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	436 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	437 buf.append(replace);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	438 // update offsets if replacement is a different length
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	439 if (offsets != null) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	440 int r = replace.length();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	441 if (r == 0)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	442 this.offsets = arrayKill(this.offsets, i - n);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	443 else if (r == 2)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	444 this.offsets = arrayInsert(this.offsets, i - n + 1, this.offsets[i - n], r - 1);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	445 n += 1 - r;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	446 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	447 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	448 return buf.toString();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	449 } else if (language.equals("it")) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	450 // new Mpdl code: added by J. Willenborg: some of Malcolms code did not work without errors so it has to be taken away, also all latin stuff is imported
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	451 StringBuffer buf = new StringBuffer();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	452 int n = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	453 for (int i = 0; i < s.length(); i++) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	454 char c = s.charAt(i);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	455 String replace = new String();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	456 switch (c) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	457 case '\u00c0': replace = "A"; break; // LATIN CAPITAL LETTER A GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	458 case '\u00c1': replace = "A"; break; // LATIN CAPITAL LETTER A ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	459 case '\u00c2': replace = "A"; break; // LATIN CAPITAL LETTER A CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	460 case '\u00c4': replace = "A"; break; // LATIN CAPITAL LETTER A DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	461 case '\u00c6': replace = "Ae"; break; // LATIN CAPITAL LETTER A E
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	462 case '\u00c7': replace = "C"; break; // LATIN CAPITAL LETTER C CEDILLA
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	463 case '\u00c8': replace = "E"; break; // LATIN CAPITAL LETTER E GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	464 case '\u00c9': replace = "E"; break; // LATIN CAPITAL LETTER E ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	465 case '\u00ca': replace = "E"; break; // LATIN CAPITAL LETTER E CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	466 case '\u00cb': replace = "E"; break; // LATIN CAPITAL LETTER E DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	467 case '\u00cc': replace = "I"; break; // LATIN CAPITAL LETTER I GRAVE;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	468 case '\u00cd': replace = "I"; break; // LATIN CAPITAL LETTER I ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	469 case '\u00ce': replace = "I"; break; // LATIN CAPITAL LETTER I CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	470 case '\u00cf': replace = "I"; break; // LATIN CAPITAL LETTER I DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	471 case '\u00d2': replace = "O"; break; // LATIN CAPITAL LETTER O GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	472 case '\u00d3': replace = "O"; break; // LATIN CAPITAL LETTER O ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	473 case '\u00d4': replace = "O"; break; // LATIN CAPITAL LETTER O CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	474 case '\u00d6': replace = "O"; break; // LATIN CAPITAL LETTER O DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	475 case '\u00d9': replace = "U"; break; // LATIN CAPITAL LETTER U GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	476 case '\u00da': replace = "U"; break; // LATIN CAPITAL LETTER U ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	477 case '\u00db': replace = "U"; break; // LATIN CAPITAL LETTER U CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	478 case '\u00dc': replace = "U"; break; // LATIN CAPITAL LETTER U DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	479 case '\u00e0': replace = "a"; break; // LATIN SMALL LETTER A GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	480 case '\u00e1': replace = "a"; break; // LATIN SMALL LETTER A ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	481 case '\u00e2': replace = "a"; break; // LATIN SMALL LETTER A CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	482 case '\u00e4': replace = "a"; break; // LATIN SMALL LETTER A DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	483 case '\u00e6': replace = "ae"; break; // LATIN SMALL LETTER A E
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	484 case '\u00e7': replace = "c"; break; // LATIN SMALL LETTER C CEDILLA
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	485 case '\u00e8': replace = "e"; break; // LATIN SMALL LETTER E GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	486 case '\u00e9': replace = "e"; break; // LATIN SMALL LETTER E ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	487 case '\u00ea': replace = "e"; break; // LATIN SMALL LETTER E CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	488 case '\u00eb': replace = "e"; break; // LATIN SMALL LETTER E DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	489 case '\u00ec': replace = "i"; break; // LATIN SMALL LETTER I GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	490 case '\u00ed': replace = "i"; break; // LATIN SMALL LETTER I ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	491 case '\u00ee': replace = "i"; break; // LATIN SMALL LETTER I CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	492 case '\u00ef': replace = "i"; break; // LATIN SMALL LETTER I DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	493 case '\u00f2': replace = "o"; break; // LATIN SMALL LETTER O GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	494 case '\u00f3': replace = "o"; break; // LATIN SMALL LETTER O ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	495 case '\u00f4': replace = "o"; break; // LATIN SMALL LETTER O CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	496 case '\u00f6': replace = "o"; break; // LATIN SMALL LETTER O DIAERESIS
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	497 case '\u00f9': replace = "u"; break; // LATIN SMALL LETTER U GRAVE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	498 case '\u00fa': replace = "u"; break; // LATIN SMALL LETTER U ACUTE
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	499 case '\u00fb': replace = "u"; break; // LATIN SMALL LETTER U CIRCUMFLEX
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	500 case '\u00fc': replace = "u"; break; // LATIN SMALL LETTER U DIAERESIS

19

4a3641ae14d2 Erstellung