Context Navigation

Back to Ticket #50

Ticket #50: MpdlNormalizer.java

File MpdlNormalizer.java, 52.4 KB (added by jwillenborg, 13 years ago)

Line
1	package de.mpg.mpiwg.berlin.mpdl.lt.analyzer;
2
3	import java.io.BufferedReader;
4	import java.io.IOException;
5	import java.io.InputStreamReader;
6	import java.io.UnsupportedEncodingException;
7	import java.util.ArrayList;
8
9	import de.mpg.mpiwg.berlin.mpdl.exception.ApplicationException;
10	import de.mpg.mpiwg.berlin.mpdl.lt.doc.regularization.Regularization;
11	import de.mpg.mpiwg.berlin.mpdl.lt.doc.regularization.RegularizationManager;
12	import de.mpg.mpiwg.berlin.mpdl.lt.general.Language;
13
14	public class MpdlNormalizer {
15	static final private String IT_VOWELS = "AEIOUaeiou" +
16	"\u00c6\u00e6" + // AE ligatures
17	"\u0152\u0153"; // OE ligatures
18	static final private String IT_CONS = "BCDFGHKLMNPQRSTVWXZ" +
19	"bcdfghklmnpqrstvwxz" +
20	"\u017f\u00df"; // long/sharp S
21	private String[] normFunctionsToUse = {"reg", "norm"}; // default is to use all of these normalization functions
22	private String language;
23	private int[] offsets;
24
25	public MpdlNormalizer(String[] normFunctionsToUse, String lang) {
26	this.normFunctionsToUse = normFunctionsToUse;
27	String language = Language.getInstance().getLanguageId(lang);
28	this.language = language;
29	}
30
31	public MpdlNormalizer(String language) {
32	this.language = language;
33	}
34
35	/**
36	* Applies the normalization rules in <code>language</code> to
37	* <code>s</code>, without offset tracking.
38	*
39	* @param s source string
40	* @return normalized string
41	*/
42	public String normalize(String s) throws ApplicationException {
43	String normStr = s;
44	if (useRegFunction()) {
45	// try to regularize the string to the norm form over predefined regularizations
46	RegularizationManager regManager = RegularizationManager.getInstance();
47	ArrayList<Regularization> regs = regManager.findRegsByOrig(language, s);
48	if (regs != null && regs.size() > 0) {
49	Regularization reg = regs.get(0); // only one: the first one
50	String regNormStr = reg.getNorm();
51	normStr = regNormStr;
52	}
53	}
54	if (useNormFunction()) {
55	// normalize the string by string replace
56	normStr = normalize(normStr, null);
57	}
58	return normStr;
59	}
60
61	private boolean useRegFunction() {
62	boolean useReg = false;
63	for (int i=0; i< normFunctionsToUse.length; i++) {
64	String function = normFunctionsToUse[i];
65	if (function.equals("reg"))
66	return true;
67	}
68	return useReg;
69	}
70
71	private boolean useNormFunction() {
72	boolean useNorm = false;
73	for (int i=0; i< normFunctionsToUse.length; i++) {
74	String function = normFunctionsToUse[i];
75	if (function.equals("norm"))
76	return true;
77	}
78	return useNorm;
79	}
80
81	/**
82	* Applies the normalization rules in <code>language</code> to
83	* <code>s</code>, with offset tracking.<p>
84	*
85	* <strong>WARNING:</strong>
86	* Arboreal will not work properly if a normalization substitution
87	* replaces a source character with more than two target characters!
88	* This is simply a BUG, and should be fixed. Fortunately, however,
89	* one does not often need such a replacement.<p>
90	*
91	* @param s source string
92	* @param offsets character offset table
93	* @return normalized string
94	*/
95	public String normalize(String s, int[] offsets) {
96	this.offsets = offsets;
97	if (language.equals("la") \|\| language.equals("lat")) {
98	StringBuffer buf = new StringBuffer();
99	int n = 0;
100	for (int i = 0; i < s.length(); i++) {
101	char c = s.charAt(i);
102	String replace = new String();
103	switch (c) {
104	case 'j': replace = "i"; break;
105	case 'v': replace = "u"; break;
106	/*
107	* Linguistic note: /u/ and /v/ are rarely phonemic
108	* in Latin, as in alui 's/he nourished' vs.
109	* alvi 'of a belly', volui 's/he wished' or 'it rolled'
110	* vs. volvi 'to be rolled', (in)seruit 's/he joined
111	* together' vs. (in)servit 's/he serves'.
112	*/
113	case 'q':
114	if ((i < s.length() - 1) && (s.charAt(i + 1) == ';'))
115	replace = "qu";
116	else
117	replace = "q";
118	break;
119	case ';':
120	if ((i > 0) && (s.charAt(i - 1) == 'q'))
121	replace = "e";
122	else if ((i == 0) \|\| ! Character.isLetter(s.charAt(i - 1)))
123	replace = ";";
124	else
125	replace = "";
126	break;
127	case '\u0300': replace = ""; break; // COMBINING GRAVE ACCENT
128	case '\u0301': replace = ""; break; // COMBINING ACCUTE ACCENT
129	case '\u0302': replace = ""; break; // COMBINING CIRCUMFLEX ACCENT
130
131	case '\u00c0': replace = "A"; break; // LATIN CAPITAL LETTER A GRAVE
132	case '\u00c1': replace = "A"; break; // LATIN CAPITAL LETTER A ACUTE
133	case '\u00c2': replace = "A"; break; // LATIN CAPITAL LETTER A CIRCUMFLEX
134	case '\u00c4': replace = "A"; break; // LATIN CAPITAL LETTER A DIAERESIS
135	case '\u00c6': replace = "Ae"; break; // LATIN CAPITAL LETTER A E
136	case '\u00c7': replace = "C"; break; // LATIN CAPITAL LETTER C CEDILLA
137	case '\u00c8': replace = "E"; break; // LATIN CAPITAL LETTER E GRAVE
138	case '\u00c9': replace = "E"; break; // LATIN CAPITAL LETTER E ACUTE
139	case '\u00ca': replace = "E"; break; // LATIN CAPITAL LETTER E CIRCUMFLEX
140	case '\u00cb': replace = "E"; break; // LATIN CAPITAL LETTER E DIAERESIS
141	case '\u00cc': replace = "I"; break; // LATIN CAPITAL LETTER I GRAVE;
142	case '\u00cd': replace = "I"; break; // LATIN CAPITAL LETTER I ACUTE
143	case '\u00ce': replace = "I"; break; // LATIN CAPITAL LETTER I CIRCUMFLEX
144	case '\u00cf': replace = "I"; break; // LATIN CAPITAL LETTER I DIAERESIS
145	case '\u00d2': replace = "O"; break; // LATIN CAPITAL LETTER O GRAVE
146	case '\u00d3': replace = "O"; break; // LATIN CAPITAL LETTER O ACUTE
147	case '\u00d4': replace = "O"; break; // LATIN CAPITAL LETTER O CIRCUMFLEX
148	case '\u00d6': replace = "O"; break; // LATIN CAPITAL LETTER O DIAERESIS
149	case '\u00d9': replace = "U"; break; // LATIN CAPITAL LETTER U GRAVE
150	case '\u00da': replace = "U"; break; // LATIN CAPITAL LETTER U ACUTE
151	case '\u00db': replace = "U"; break; // LATIN CAPITAL LETTER U CIRCUMFLEX
152	case '\u00dc': replace = "U"; break; // LATIN CAPITAL LETTER U DIAERESIS
153	case '\u00e0': replace = "a"; break; // LATIN SMALL LETTER A GRAVE
154	case '\u00e1': replace = "a"; break; // LATIN SMALL LETTER A ACUTE
155	case '\u00e2': replace = "a"; break; // LATIN SMALL LETTER A CIRCUMFLEX
156	case '\u00e4': replace = "a"; break; // LATIN SMALL LETTER A DIAERESIS
157	case '\u00e6': replace = "ae"; break; // LATIN SMALL LETTER A E
158	case '\u00e7': replace = "c"; break; // LATIN SMALL LETTER C CEDILLA
159	case '\u00e8': replace = "e"; break; // LATIN SMALL LETTER E GRAVE
160	case '\u00e9': replace = "e"; break; // LATIN SMALL LETTER E ACUTE
161	case '\u00ea': replace = "e"; break; // LATIN SMALL LETTER E CIRCUMFLEX
162	case '\u00eb': replace = "e"; break; // LATIN SMALL LETTER E DIAERESIS
163	case '\u00ec': replace = "i"; break; // LATIN SMALL LETTER I GRAVE
164	case '\u00ed': replace = "i"; break; // LATIN SMALL LETTER I ACUTE
165	case '\u00ee': replace = "i"; break; // LATIN SMALL LETTER I CIRCUMFLEX
166	case '\u00ef': replace = "i"; break; // LATIN SMALL LETTER I DIAERESIS
167	case '\u00f2': replace = "o"; break; // LATIN SMALL LETTER O GRAVE
168	case '\u00f3': replace = "o"; break; // LATIN SMALL LETTER O ACUTE
169	case '\u00f4': replace = "o"; break; // LATIN SMALL LETTER O CIRCUMFLEX
170	case '\u00f6': replace = "o"; break; // LATIN SMALL LETTER O DIAERESIS
171	case '\u00f9': replace = "u"; break; // LATIN SMALL LETTER U GRAVE
172	case '\u00fa': replace = "u"; break; // LATIN SMALL LETTER U ACUTE
173	case '\u00fb': replace = "u"; break; // LATIN SMALL LETTER U CIRCUMFLEX
174	case '\u00fc': replace = "u"; break; // LATIN SMALL LETTER U DIAERESIS
175	case '\u0100': replace = "A"; break; // LATIN CAPITAL LETTER A MACRON
176	case '\u0101': replace = "a"; break; // LATIN SMALL LETTER A MACRON
177	case '\u0102': replace = "A"; break; // LATIN CAPITAL LETTER A BREVE
178	case '\u0103': replace = "a"; break; // LATIN SMALL LETTER A BREVE
179	case '\u0112': replace = "E"; break; // LATIN CAPITAL LETTER E MACRON
180	case '\u0113': replace = "e"; break; // LATIN SMALL LETTER E MACRON
181	case '\u0114': replace = "E"; break; // LATIN CAPITAL LETTER E BREVE
182	case '\u0115': replace = "e"; break; // LATIN SMALL LETTER E BREVE
183	case '\u0118': replace = "Ae"; break; // LATIN CAPITAL LETTER E OGONEK
184	case '\u0119': replace = "ae"; break; // LATIN SMALL LETTER E OGONEK
185	case '\u012a': replace = "I"; break; // LATIN CAPITAL LETTER I MACRON
186	case '\u012b': replace = "i"; break; // LATIN SMALL LETTER I MACRON
187	case '\u012c': replace = "I"; break; // LATIN CAPITAL LETTER I BREVE
188	case '\u012d': replace = "i"; break; // LATIN SMALL LETTER I BREVE
189	case '\u014c': replace = "O"; break; // LATIN CAPITAL LETTER O MACRON
190	case '\u014d': replace = "o"; break; // LATIN SMALL LETTER O MACRON
191	case '\u014e': replace = "O"; break; // LATIN CAPITAL LETTER O BREVE
192	case '\u014f': replace = "o"; break; // LATIN SMALL LETTER O BREVE
193	case '\u0152': replace = "Oe"; break; // LATIN CAPITAL LETTER O E
194	case '\u0153': replace = "oe"; break; // LATIN SMALL LETTER O E
195	case '\u016a': replace = "U"; break; // LATIN CAPITAL LETTER U MACRON
196	case '\u016b': replace = "u"; break; // LATIN SMALL LETTER U MACRON
197	case '\u016c': replace = "U"; break; // LATIN CAPITAL LETTER U BREVE
198	case '\u016d': replace = "u"; break; // LATIN SMALL LETTER U BREVE
199	case '\u017f': replace = "s"; break; // LATIN SMALL LETTER LONG S
200	case '\u00df': replace = "ss"; break; // LATIN SMALL LETTER SHARP S
201	case '\u00ad': break; // soft hyphen
202	// new in MPDL project by J. Willenborg
203	case '\u1e14': replace = "e"; break; // LATIN ... LETTER E WITH ...
204	case '\u1e15': replace = "e"; break; // LATIN ... LETTER E WITH ...
205	case '\u1e16': replace = "e"; break; // LATIN ... LETTER E WITH ...
206	case '\u1e17': replace = "e"; break; // LATIN ... LETTER E WITH ...
207	case '\u1e18': replace = "e"; break; // LATIN ... LETTER E WITH ...
208	case '\u1e19': replace = "e"; break; // LATIN ... LETTER E WITH ...
209	case '\u1e1a': replace = "e"; break; // LATIN ... LETTER E WITH ...
210	case '\u1e1b': replace = "e"; break; // LATIN ... LETTER E WITH ...
211	case '\u1e1c': replace = "e"; break; // LATIN ... LETTER E WITH ...
212	case '\u1e1d': replace = "e"; break; // LATIN ... LETTER E WITH ...
213	case '\u1eb8': replace = "e"; break; // LATIN ... LETTER E WITH ...
214	case '\u1eb9': replace = "e"; break; // LATIN ... LETTER E WITH ...
215	case '\u1eba': replace = "e"; break; // LATIN ... LETTER E WITH ...
216	case '\u1ebb': replace = "e"; break; // LATIN ... LETTER E WITH ...
217	case '\u1ebc': replace = "e"; break; // LATIN ... LETTER E WITH ...
218	case '\u1ebd': replace = "e"; break; // LATIN ... LETTER E WITH ...
219	case '\u1ebe': replace = "e"; break; // LATIN ... LETTER E WITH ...
220	case '\u1ebf': replace = "e"; break; // LATIN ... LETTER E WITH ...
221	case '\u1ec0': replace = "e"; break; // LATIN ... LETTER E WITH ...
222	case '\u1ec1': replace = "e"; break; // LATIN ... LETTER E WITH ...
223	case '\u1ec2': replace = "e"; break; // LATIN ... LETTER E WITH ...
224	case '\u1ec3': replace = "e"; break; // LATIN ... LETTER E WITH ...
225	case '\u1ec4': replace = "e"; break; // LATIN ... LETTER E WITH ...
226	case '\u1ec5': replace = "e"; break; // LATIN ... LETTER E WITH ...
227	case '\u1ec6': replace = "e"; break; // LATIN ... LETTER E WITH ...
228	case '\u1ec7': replace = "e"; break; // LATIN ... LETTER E WITH ...
229	// by Malcolm
230	case '\u2329': break; // BRA
231	case '\u232a': break; // KET
232	default: replace += c; break;
233	}
234	buf.append(replace);
235	// update offsets if replacement is a different length
236	if (offsets != null) {
237	int r = replace.length();
238	if (r == 0)
239	this.offsets = arrayKill(this.offsets, i - n);
240	else if (r == 2)
241	this.offsets = arrayInsert(this.offsets, i - n + 1, this.offsets[i - n], r - 1);
242	n += 1 - r;
243	}
244	}
245	return buf.toString();
246	} else if (language.equals("it")) {
247	// new Mpdl code: added by J. Willenborg: some of Malcolms code did not work without errors so it has to be taken away, also all latin stuff is imported
248	StringBuffer buf = new StringBuffer();
249	int n = 0;
250	for (int i = 0; i < s.length(); i++) {
251	char c = s.charAt(i);
252	String replace = new String();
253	switch (c) {
254	case '\u00c0': replace = "A"; break; // LATIN CAPITAL LETTER A GRAVE
255	case '\u00c1': replace = "A"; break; // LATIN CAPITAL LETTER A ACUTE
256	case '\u00c2': replace = "A"; break; // LATIN CAPITAL LETTER A CIRCUMFLEX
257	case '\u00c4': replace = "A"; break; // LATIN CAPITAL LETTER A DIAERESIS
258	case '\u00c6': replace = "Ae"; break; // LATIN CAPITAL LETTER A E
259	case '\u00c7': replace = "C"; break; // LATIN CAPITAL LETTER C CEDILLA
260	case '\u00c8': replace = "E"; break; // LATIN CAPITAL LETTER E GRAVE
261	case '\u00c9': replace = "E"; break; // LATIN CAPITAL LETTER E ACUTE
262	case '\u00ca': replace = "E"; break; // LATIN CAPITAL LETTER E CIRCUMFLEX
263	case '\u00cb': replace = "E"; break; // LATIN CAPITAL LETTER E DIAERESIS
264	case '\u00cc': replace = "I"; break; // LATIN CAPITAL LETTER I GRAVE;
265	case '\u00cd': replace = "I"; break; // LATIN CAPITAL LETTER I ACUTE
266	case '\u00ce': replace = "I"; break; // LATIN CAPITAL LETTER I CIRCUMFLEX
267	case '\u00cf': replace = "I"; break; // LATIN CAPITAL LETTER I DIAERESIS
268	case '\u00d2': replace = "O"; break; // LATIN CAPITAL LETTER O GRAVE
269	case '\u00d3': replace = "O"; break; // LATIN CAPITAL LETTER O ACUTE
270	case '\u00d4': replace = "O"; break; // LATIN CAPITAL LETTER O CIRCUMFLEX
271	case '\u00d6': replace = "O"; break; // LATIN CAPITAL LETTER O DIAERESIS
272	case '\u00d9': replace = "U"; break; // LATIN CAPITAL LETTER U GRAVE
273	case '\u00da': replace = "U"; break; // LATIN CAPITAL LETTER U ACUTE
274	case '\u00db': replace = "U"; break; // LATIN CAPITAL LETTER U CIRCUMFLEX
275	case '\u00dc': replace = "U"; break; // LATIN CAPITAL LETTER U DIAERESIS
276	case '\u00e0': replace = "a"; break; // LATIN SMALL LETTER A GRAVE
277	case '\u00e1': replace = "a"; break; // LATIN SMALL LETTER A ACUTE
278	case '\u00e2': replace = "a"; break; // LATIN SMALL LETTER A CIRCUMFLEX
279	case '\u00e4': replace = "a"; break; // LATIN SMALL LETTER A DIAERESIS
280	case '\u00e6': replace = "ae"; break; // LATIN SMALL LETTER A E
281	case '\u00e7': replace = "c"; break; // LATIN SMALL LETTER C CEDILLA
282	case '\u00e8': replace = "e"; break; // LATIN SMALL LETTER E GRAVE
283	case '\u00e9': replace = "e"; break; // LATIN SMALL LETTER E ACUTE
284	case '\u00ea': replace = "e"; break; // LATIN SMALL LETTER E CIRCUMFLEX
285	case '\u00eb': replace = "e"; break; // LATIN SMALL LETTER E DIAERESIS
286	case '\u00ec': replace = "i"; break; // LATIN SMALL LETTER I GRAVE
287	case '\u00ed': replace = "i"; break; // LATIN SMALL LETTER I ACUTE
288	case '\u00ee': replace = "i"; break; // LATIN SMALL LETTER I CIRCUMFLEX
289	case '\u00ef': replace = "i"; break; // LATIN SMALL LETTER I DIAERESIS
290	case '\u00f2': replace = "o"; break; // LATIN SMALL LETTER O GRAVE
291	case '\u00f3': replace = "o"; break; // LATIN SMALL LETTER O ACUTE
292	case '\u00f4': replace = "o"; break; // LATIN SMALL LETTER O CIRCUMFLEX
293	case '\u00f6': replace = "o"; break; // LATIN SMALL LETTER O DIAERESIS
294	case '\u00f9': replace = "u"; break; // LATIN SMALL LETTER U GRAVE
295	case '\u00fa': replace = "u"; break; // LATIN SMALL LETTER U ACUTE
296	case '\u00fb': replace = "u"; break; // LATIN SMALL LETTER U CIRCUMFLEX
297	case '\u00fc': replace = "u"; break; // LATIN SMALL LETTER U DIAERESIS
298	case '\u0100': replace = "A"; break; // LATIN CAPITAL LETTER A MACRON
299	case '\u0101': replace = "a"; break; // LATIN SMALL LETTER A MACRON
300	case '\u0102': replace = "A"; break; // LATIN CAPITAL LETTER A BREVE
301	case '\u0103': replace = "a"; break; // LATIN SMALL LETTER A BREVE
302	case '\u0112': replace = "E"; break; // LATIN CAPITAL LETTER E MACRON
303	case '\u0113': replace = "e"; break; // LATIN SMALL LETTER E MACRON
304	case '\u0114': replace = "E"; break; // LATIN CAPITAL LETTER E BREVE
305	case '\u0115': replace = "e"; break; // LATIN SMALL LETTER E BREVE
306	case '\u0118': replace = "Ae"; break; // LATIN CAPITAL LETTER E OGONEK
307	case '\u0119': replace = "ae"; break; // LATIN SMALL LETTER E OGONEK
308	case '\u012a': replace = "I"; break; // LATIN CAPITAL LETTER I MACRON
309	case '\u012b': replace = "i"; break; // LATIN SMALL LETTER I MACRON
310	case '\u012c': replace = "I"; break; // LATIN CAPITAL LETTER I BREVE
311	case '\u012d': replace = "i"; break; // LATIN SMALL LETTER I BREVE
312	case '\u014c': replace = "O"; break; // LATIN CAPITAL LETTER O MACRON
313	case '\u014d': replace = "o"; break; // LATIN SMALL LETTER O MACRON
314	case '\u014e': replace = "O"; break; // LATIN CAPITAL LETTER O BREVE
315	case '\u014f': replace = "o"; break; // LATIN SMALL LETTER O BREVE
316	case '\u0152': replace = "Oe"; break; // LATIN CAPITAL LETTER O E
317	case '\u0153': replace = "oe"; break; // LATIN SMALL LETTER O E
318	case '\u016a': replace = "U"; break; // LATIN CAPITAL LETTER U MACRON
319	case '\u016b': replace = "u"; break; // LATIN SMALL LETTER U MACRON
320	case '\u016c': replace = "U"; break; // LATIN CAPITAL LETTER U BREVE
321	case '\u016d': replace = "u"; break; // LATIN SMALL LETTER U BREVE
322	case '\u017f': replace = "s"; break; // LATIN SMALL LETTER LONG S
323	case '\u00df': replace = "ss"; break; // LATIN SMALL LETTER SHARP S
324	// new in MPDL project by J. Willenborg
325	case '\u1e8d': replace = "e"; break; // LATIN SMALL LETTER E WITH TILDE
326	default: replace += c; break;
327	}
328	buf.append(replace);
329	// update offsets if replacement is a different length
330	if (offsets != null) {
331	int r = replace.length();
332	if (r == 0) this.offsets = arrayKill(this.offsets, i - n);
333	else if (r == 2)
334	this.offsets = arrayInsert(this.offsets, i - n + 1, this.offsets[i - n], r - 1);
335	n += 1 - r;
336	}
337	}
338	return buf.toString();
339	// new Mpdl code: added by J. Willenborg: most of the latin replacements also in english
340	} else if (language.equals("en")) {
341	StringBuffer buf = new StringBuffer();
342	int n = 0;
343	for (int i = 0; i < s.length(); i++) {
344	char c = s.charAt(i);
345	String replace = new String();
346	switch (c) {
347	case '\u0300': replace = ""; break; // COMBINING GRAVE ACCENT
348	case '\u0301': replace = ""; break; // COMBINING ACCUTE ACCENT
349	case '\u0302': replace = ""; break; // COMBINING CIRCUMFLEX ACCENT
350
351	case '\u00c0': replace = "A"; break; // LATIN CAPITAL LETTER A GRAVE
352	case '\u00c1': replace = "A"; break; // LATIN CAPITAL LETTER A ACUTE
353	case '\u00c2': replace = "A"; break; // LATIN CAPITAL LETTER A CIRCUMFLEX
354	case '\u00c4': replace = "A"; break; // LATIN CAPITAL LETTER A DIAERESIS
355	case '\u00c6': replace = "Ae"; break; // LATIN CAPITAL LETTER A E
356	case '\u00c7': replace = "C"; break; // LATIN CAPITAL LETTER C CEDILLA
357	case '\u00c8': replace = "E"; break; // LATIN CAPITAL LETTER E GRAVE
358	case '\u00c9': replace = "E"; break; // LATIN CAPITAL LETTER E ACUTE
359	case '\u00ca': replace = "E"; break; // LATIN CAPITAL LETTER E CIRCUMFLEX
360	case '\u00cb': replace = "E"; break; // LATIN CAPITAL LETTER E DIAERESIS
361	case '\u00cc': replace = "I"; break; // LATIN CAPITAL LETTER I GRAVE;
362	case '\u00cd': replace = "I"; break; // LATIN CAPITAL LETTER I ACUTE
363	case '\u00ce': replace = "I"; break; // LATIN CAPITAL LETTER I CIRCUMFLEX
364	case '\u00cf': replace = "I"; break; // LATIN CAPITAL LETTER I DIAERESIS
365	case '\u00d2': replace = "O"; break; // LATIN CAPITAL LETTER O GRAVE
366	case '\u00d3': replace = "O"; break; // LATIN CAPITAL LETTER O ACUTE
367	case '\u00d4': replace = "O"; break; // LATIN CAPITAL LETTER O CIRCUMFLEX
368	case '\u00d6': replace = "O"; break; // LATIN CAPITAL LETTER O DIAERESIS
369	case '\u00d9': replace = "U"; break; // LATIN CAPITAL LETTER U GRAVE
370	case '\u00da': replace = "U"; break; // LATIN CAPITAL LETTER U ACUTE
371	case '\u00db': replace = "U"; break; // LATIN CAPITAL LETTER U CIRCUMFLEX
372	case '\u00dc': replace = "U"; break; // LATIN CAPITAL LETTER U DIAERESIS
373	case '\u00e0': replace = "a"; break; // LATIN SMALL LETTER A GRAVE
374	case '\u00e1': replace = "a"; break; // LATIN SMALL LETTER A ACUTE
375	case '\u00e2': replace = "a"; break; // LATIN SMALL LETTER A CIRCUMFLEX
376	case '\u00e4': replace = "a"; break; // LATIN SMALL LETTER A DIAERESIS
377	case '\u00e6': replace = "ae"; break; // LATIN SMALL LETTER A E
378	case '\u00e7': replace = "c"; break; // LATIN SMALL LETTER C CEDILLA
379	case '\u00e8': replace = "e"; break; // LATIN SMALL LETTER E GRAVE
380	case '\u00e9': replace = "e"; break; // LATIN SMALL LETTER E ACUTE
381	case '\u00ea': replace = "e"; break; // LATIN SMALL LETTER E CIRCUMFLEX
382	case '\u00eb': replace = "e"; break; // LATIN SMALL LETTER E DIAERESIS
383	case '\u00ec': replace = "i"; break; // LATIN SMALL LETTER I GRAVE
384	case '\u00ed': replace = "i"; break; // LATIN SMALL LETTER I ACUTE
385	case '\u00ee': replace = "i"; break; // LATIN SMALL LETTER I CIRCUMFLEX
386	case '\u00ef': replace = "i"; break; // LATIN SMALL LETTER I DIAERESIS
387	case '\u00f2': replace = "o"; break; // LATIN SMALL LETTER O GRAVE
388	case '\u00f3': replace = "o"; break; // LATIN SMALL LETTER O ACUTE
389	case '\u00f4': replace = "o"; break; // LATIN SMALL LETTER O CIRCUMFLEX
390	case '\u00f6': replace = "o"; break; // LATIN SMALL LETTER O DIAERESIS
391	case '\u00f9': replace = "u"; break; // LATIN SMALL LETTER U GRAVE
392	case '\u00fa': replace = "u"; break; // LATIN SMALL LETTER U ACUTE
393	case '\u00fb': replace = "u"; break; // LATIN SMALL LETTER U CIRCUMFLEX
394	case '\u00fc': replace = "u"; break; // LATIN SMALL LETTER U DIAERESIS
395	case '\u0100': replace = "A"; break; // LATIN CAPITAL LETTER A MACRON
396	case '\u0101': replace = "a"; break; // LATIN SMALL LETTER A MACRON
397	case '\u0102': replace = "A"; break; // LATIN CAPITAL LETTER A BREVE
398	case '\u0103': replace = "a"; break; // LATIN SMALL LETTER A BREVE
399	case '\u0112': replace = "E"; break; // LATIN CAPITAL LETTER E MACRON
400	case '\u0113': replace = "e"; break; // LATIN SMALL LETTER E MACRON
401	case '\u0114': replace = "E"; break; // LATIN CAPITAL LETTER E BREVE
402	case '\u0115': replace = "e"; break; // LATIN SMALL LETTER E BREVE
403	case '\u0118': replace = "Ae"; break; // LATIN CAPITAL LETTER E OGONEK
404	case '\u0119': replace = "ae"; break; // LATIN SMALL LETTER E OGONEK
405	case '\u012a': replace = "I"; break; // LATIN CAPITAL LETTER I MACRON
406	case '\u012b': replace = "i"; break; // LATIN SMALL LETTER I MACRON
407	case '\u012c': replace = "I"; break; // LATIN CAPITAL LETTER I BREVE
408	case '\u012d': replace = "i"; break; // LATIN SMALL LETTER I BREVE
409	case '\u014c': replace = "O"; break; // LATIN CAPITAL LETTER O MACRON
410	case '\u014d': replace = "o"; break; // LATIN SMALL LETTER O MACRON
411	case '\u014e': replace = "O"; break; // LATIN CAPITAL LETTER O BREVE
412	case '\u014f': replace = "o"; break; // LATIN SMALL LETTER O BREVE
413	case '\u0152': replace = "Oe"; break; // LATIN CAPITAL LETTER O E
414	case '\u0153': replace = "oe"; break; // LATIN SMALL LETTER O E
415	case '\u016a': replace = "U"; break; // LATIN CAPITAL LETTER U MACRON
416	case '\u016b': replace = "u"; break; // LATIN SMALL LETTER U MACRON
417	case '\u016c': replace = "U"; break; // LATIN CAPITAL LETTER U BREVE
418	case '\u016d': replace = "u"; break; // LATIN SMALL LETTER U BREVE
419	case '\u017f': replace = "s"; break; // LATIN SMALL LETTER LONG S
420	case '\u00df': replace = "ss"; break; // LATIN SMALL LETTER SHARP S
421	// new in MPDL project by J. Willenborg
422	case '\u1e8d': replace = "e"; break; // LATIN SMALL LETTER E WITH TILDE
423	// by Malcolm
424	case '\u00ad': break; // soft hyphen
425	case '\u2329': break; // BRA
426	case '\u232a': break; // KET
427	default: replace += c; break;
428	}
429	buf.append(replace);
430	// update offsets if replacement is a different length
431	if (offsets != null) {
432	int r = replace.length();
433	if (r == 0)
434	this.offsets = arrayKill(this.offsets, i - n);
435	else if (r == 2)
436	this.offsets = arrayInsert(this.offsets, i - n + 1, this.offsets[i - n], r - 1);
437	n += 1 - r;
438	}
439	}
440	return buf.toString();
441	} else if (language.equals("fr")) {
442	// new Mpdl code: added by J. Willenborg: some of Malcolms code did not work without errors so it has to be taken away
443	StringBuffer buf = new StringBuffer();
444	int n = 0;
445	for (int i = 0; i < s.length(); i++) {
446	char c = s.charAt(i);
447	String replace = new String();
448	switch (c) {
449	case '\u00e6': replace = "ae"; break; // LATIN SMALL LETTER A E
450	case '\u017f': replace = "s"; break; // LATIN SMALL LETTER LONG S
451	case '\u00df': replace = "ss"; break; // LATIN SMALL LETTER SHARP S
452	case '\u00ad': break; // soft hyphen
453	case '-': break;
454	default: replace += c; break;
455	}
456	buf.append(replace);
457	// update offsets if replacement is a different length
458	if (offsets != null) {
459	int r = replace.length();
460	if (r == 0)
461	this.offsets = arrayKill(this.offsets, i - n);
462	else if (r == 2)
463	this.offsets = arrayInsert(this.offsets, i - n + 1, this.offsets[i - n], r - 1);
464	n += 1 - r;
465	}
466	}
467	return buf.toString();
468	} else if (language.equals("de")) {
469	StringBuffer buf = new StringBuffer();
470	int n = 0;
471	for (int i = 0; i < s.length(); i++) {
472	char c = s.charAt(i);
473	String replace = new String();
474	switch (c) {
475	case '\u00c4': replace = "Ae"; break;
476	case '\u00d6': replace = "Oe"; break;
477	case '\u00dc': replace = "Ue"; break;
478	case '\u00df': replace = "ss"; break;
479	case '\u00e4': replace = "ae"; break;
480	case '\u00f6': replace = "oe"; break;
481	case '\u00fc': replace = "ue"; break;
482	case '\u00e9': replace = "e"; break;
483	case '\u00ad': break; // soft hyphen
484	case '-': break;
485	default: replace += c; break;
486	}
487	buf.append(replace);
488	// update offsets if replacement is a different length
489	if (offsets != null) {
490	int r = replace.length();
491	if (r == 0)
492	this.offsets = arrayKill(this.offsets, i - n);
493	else if (r == 2)
494	this.offsets = arrayInsert(this.offsets, i - n + 1, this.offsets[i - n], r - 1);
495	n += 1 - r;
496	}
497	}
498	return buf.toString();
499	} else if (language.equals("zh")) {
500	StringBuffer buf = new StringBuffer();
501	int n = 0;
502	for (int i = 0; i < s.length(); i++) {
503	char c = s.charAt(i);
504	String replace = new String();
505	switch (c) {
506	case '\u00b9': replace = "1"; break;
507	case '\u00b2': replace = "2"; break;
508	case '\u00b3': replace = "3"; break;
509	case '\u2074': replace = "4"; break;
510	case '\u2075': replace = "5"; break;
511	// original by Malcolm Hyman: with the following replacements // TODO uncomment these 3 lines
512	// case '\u3000': replace = " "; break;
513	// case '\u3001': replace = ","; break;
514	// case '\u3002': replace = "."; break;
515	// case '\u200b': break; // BREAKS EVERYTHING!
516	default: replace += c; break;
517	}
518	buf.append(replace);
519	// update offsets if replacement is a different length
520	if (offsets != null) {
521	int r = replace.length();
522	if (r == 0)
523	this.offsets = arrayKill(this.offsets, i - n);
524	else if (r == 2)
525	this.offsets = arrayInsert(this.offsets, i - n + 1, this.offsets[i - n], r - 1);
526	n += 1 - r;
527	}
528	}
529	return buf.toString();
530	} else if (language.equals("akk") \|\|
531	language.equals("qam") \|\|
532	language.equals("qpc") \|\|
533	language.equals("elx") \|\|
534	language.equals("sux") \|\|
535	language.equals("hit") \|\|
536	language.equals("qhu") \|\|
537	language.equals("peo") \|\|
538	language.equals("uga") \|\|
539	language.equals("ura") \|\|
540	language.equals("qcu")) {
541	StringBuffer buf = new StringBuffer();
542	int n = 0;
543	char last = '\u0000';
544	for (int i = 0; i < s.length(); i++) {
545	char c = s.charAt(i);
546	c = Character.toLowerCase(c);
547	String replace = new String();
548	switch (c) {
549	case '{': replace += "-"; break;
550	case '}': replace += "-"; break;
551	// These are from PSD::ATF::Unicode by Steve Tinney
552	case '\u0161': replace += "sz"; break;
553	case '\u1e63': replace += "s,"; break;
554	case '\u1e6d': replace += "t,"; break;
555	case '\u014b': replace += "j"; break;
556	case '\u015b': replace += "s'"; break;
557	case '\u2080': replace += "0"; break;
558	case '\u2081': replace += "1"; break;
559	case '\u2082': replace += "2"; break;
560	case '\u2083': replace += "3"; break;
561	case '\u2084': replace += "4"; break;
562	case '\u2085': replace += "5"; break;
563	case '\u2086': replace += "6"; break;
564	case '\u2087': replace += "7"; break;
565	case '\u2088': replace += "8"; break;
566	case '\u2089': replace += "9"; break;
567
568	case 'c': // shin (except where used as modifier)
569	if ((i > 0) && ((last == '~') \|\| (last == '@')))
570	replace += "c";
571	else replace += "sz";
572	break;
573	default: replace += c; break;
574	}
575	// suppress grapheme boundary before or after word boundary
576	if (replace.equals("-")) {
577	if ((i + 1 == s.length()) \|\| (s.charAt(i + 1) == ' ') \|\| (i == 0) \|\| (s.charAt(i - 1) == ' '))
578	replace = "";
579	}
580	last = c;
581	buf.append(replace);
582	// update offsets if replacement is a different length
583	if (offsets != null) {
584	int r = replace.length();
585	if (r == 0)
586	this.offsets = arrayKill(this.offsets, i - n);
587	else if (r == 2)
588	this.offsets = arrayInsert(this.offsets, i - n + 1, this.offsets[i - n], r - 1);
589	n += 1 - r;
590	}
591	}
592	return buf.toString();
593	} else if (language.equals("el") \|\| language.equals("grc")) {
594	StringBuffer buf = new StringBuffer();
595	int n = 0;
596	for (int i = 0; i < s.length(); i++) {
597	char c = s.charAt(i);
598	String replace = new String();
599	switch (c) {
600	case '\u03c2': replace = "\u03c3"; break; // GREEK SMALL LETTER FINAL SIGMA
601	case '<': break;
602	case '>': break;
603	case '[': break;
604	case ']': break;
605	case '1': break;
606	case '2': break;
607	case '\u03ac': replace = "\u1f71"; break;
608	case '\u03ad': replace = "\u1f73"; break;
609	case '\u03ae': replace = "\u1f75"; break;
610	case '\u03af': replace = "\u1f77"; break;
611	case '\u03cc': replace = "\u1f79"; break;
612	case '\u03cd': replace = "\u1f7b"; break;
613	case '\u03ce': replace = "\u1f7d"; break;
614	case '-': break; // same treatment as soft hyphen
615	case '\u00ad': break; // soft hyphen
616	default: replace += c; break;
617	}
618	buf.append(replace);
619	// update offsets if replacement is a different length
620	if (offsets != null) {
621	int r = replace.length();
622	if (r == 0)
623	this.offsets = arrayKill(this.offsets, i - n);
624	else if (r == 2)
625	this.offsets = arrayInsert(this.offsets, i - n + 1, this.offsets[i - n], r - 1);
626	n += 1 - r;
627	}
628	}
629	return buf.toString();
630	} else if (language.equals("el_atonic")) {
631	StringBuffer buf = new StringBuffer();
632	int n = 0;
633	for (int i = 0; i < s.length(); i++) {
634	char c = s.charAt(i);
635	String replace = new String();
636	switch (c) {
637	case '\u03c2': replace = "\u03c3"; break; // GREEK SMALL LETTER FINAL SIGMA
638	// map characters with diacritics to their plain equivalent
639	// cf. <code>BetaCode.java</code>
640	case '\u03aa': replace = "\u0399"; break;
641	case '\u03ab': replace = "\u03a5"; break;
642	case '\u03ac': replace = "\u0381"; break;
643	case '\u03ad': replace = "\u0385"; break;
644	case '\u03ae': replace = "\u0387"; break;
645	case '\u03af': replace = "\u0389"; break;
646	case '\u03ca': replace = "\u03b9"; break;
647	case '\u03cb': replace = "\u03c5"; break;
648	case '\u03cc': replace = "\u03bf"; break;
649	case '\u03cd': replace = "\u03c5"; break;
650	case '\u03ce': replace = "\u03c9"; break;
651	case '\u1f00': replace = "\u03b1"; break;
652	case '\u1f01': replace = "\u03b1"; break;
653	case '\u1f02': replace = "\u03b1"; break;
654	case '\u1f03': replace = "\u03b1"; break;
655	case '\u1f04': replace = "\u03b1"; break;
656	case '\u1f05': replace = "\u03b1"; break;
657	case '\u1f06': replace = "\u03b1"; break;
658	case '\u1f07': replace = "\u03b1"; break;
659	case '\u1f08': replace = "\u0391"; break;
660	case '\u1f09': replace = "\u0391"; break;
661	case '\u1f0a': replace = "\u0391"; break;
662	case '\u1f0b': replace = "\u0391"; break;
663	case '\u1f0c': replace = "\u0391"; break;
664	case '\u1f0d': replace = "\u0391"; break;
665	case '\u1f0e': replace = "\u0391"; break;
666	case '\u1f0f': replace = "\u0391"; break;
667	case '\u1f10': replace = "\u03b5"; break;
668	case '\u1f11': replace = "\u03b5"; break;
669	case '\u1f12': replace = "\u03b5"; break;
670	case '\u1f13': replace = "\u03b5"; break;
671	case '\u1f14': replace = "\u03b5"; break;
672	case '\u1f15': replace = "\u03b5"; break;
673	case '\u1f18': replace = "\u0395"; break;
674	case '\u1f19': replace = "\u0395"; break;
675	case '\u1f1a': replace = "\u0395"; break;
676	case '\u1f1b': replace = "\u0395"; break;
677	case '\u1f1c': replace = "\u0395"; break;
678	case '\u1f1d': replace = "\u0395"; break;
679	case '\u1f20': replace = "\u03b7"; break;
680	case '\u1f21': replace = "\u03b7"; break;
681	case '\u1f22': replace = "\u03b7"; break;
682	case '\u1f23': replace = "\u03b7"; break;
683	case '\u1f24': replace = "\u03b7"; break;
684	case '\u1f25': replace = "\u03b7"; break;
685	case '\u1f26': replace = "\u03b7"; break;
686	case '\u1f27': replace = "\u03b7"; break;
687	case '\u1f28': replace = "\u0397"; break;
688	case '\u1f29': replace = "\u0397"; break;
689	case '\u1f2a': replace = "\u0397"; break;
690	case '\u1f2b': replace = "\u0397"; break;
691	case '\u1f2c': replace = "\u0397"; break;
692	case '\u1f2d': replace = "\u0397"; break;
693	case '\u1f2e': replace = "\u0397"; break;
694	case '\u1f2f': replace = "\u0397"; break;
695	case '\u1f30': replace = "\u03b9"; break;
696	case '\u1f31': replace = "\u03b9"; break;
697	case '\u1f32': replace = "\u03b9"; break;
698	case '\u1f33': replace = "\u03b9"; break;
699	case '\u1f34': replace = "\u03b9"; break;
700	case '\u1f35': replace = "\u03b9"; break;
701	case '\u1f36': replace = "\u03b9"; break;
702	case '\u1f37': replace = "\u03b9"; break;
703	case '\u1f38': replace = "\u0399"; break;
704	case '\u1f39': replace = "\u0399"; break;
705	case '\u1f3a': replace = "\u0399"; break;
706	case '\u1f3b': replace = "\u0399"; break;
707	case '\u1f3c': replace = "\u0399"; break;
708	case '\u1f3d': replace = "\u0399"; break;
709	case '\u1f3e': replace = "\u0399"; break;
710	case '\u1f3f': replace = "\u0399"; break;
711	case '\u1f40': replace = "\u03bf"; break;
712	case '\u1f41': replace = "\u03bf"; break;
713	case '\u1f42': replace = "\u03bf"; break;
714	case '\u1f43': replace = "\u03bf"; break;
715	case '\u1f44': replace = "\u03bf"; break;
716	case '\u1f45': replace = "\u03bf"; break;
717	case '\u1f48': replace = "\u039f"; break;
718	case '\u1f49': replace = "\u039f"; break;
719	case '\u1f4a': replace = "\u039f"; break;
720	case '\u1f4b': replace = "\u039f"; break;
721	case '\u1f4c': replace = "\u039f"; break;
722	case '\u1f4d': replace = "\u039f"; break;
723	case '\u1f50': replace = "\u03c5"; break;
724	case '\u1f51': replace = "\u03c5"; break;
725	case '\u1f52': replace = "\u03c5"; break;
726	case '\u1f53': replace = "\u03c5"; break;
727	case '\u1f54': replace = "\u03c5"; break;
728	case '\u1f55': replace = "\u03c5"; break;
729	case '\u1f56': replace = "\u03c5"; break;
730	case '\u1f57': replace = "\u03c5"; break;
731	case '\u1f58': replace = "\u03a5"; break;
732	case '\u1f59': replace = "\u03a5"; break;
733	case '\u1f5a': replace = "\u03a5"; break;
734	case '\u1f5b': replace = "\u03a5"; break;
735	case '\u1f5c': replace = "\u03a5"; break;
736	case '\u1f5d': replace = "\u03a5"; break;
737	case '\u1f5e': replace = "\u03a5"; break;
738	case '\u1f5f': replace = "\u03a5"; break;
739	case '\u1f60': replace = "\u03c9"; break;
740	case '\u1f61': replace = "\u03c9"; break;
741	case '\u1f62': replace = "\u03c9"; break;
742	case '\u1f63': replace = "\u03c9"; break;
743	case '\u1f64': replace = "\u03c9"; break;
744	case '\u1f65': replace = "\u03c9"; break;
745	case '\u1f66': replace = "\u03c9"; break;
746	case '\u1f67': replace = "\u03c9"; break;
747	case '\u1f68': replace = "\u03a9"; break;
748	case '\u1f69': replace = "\u03a9"; break;
749	case '\u1f6a': replace = "\u03a9"; break;
750	case '\u1f6b': replace = "\u03a9"; break;
751	case '\u1f6c': replace = "\u03a9"; break;
752	case '\u1f6d': replace = "\u03a9"; break;
753	case '\u1f6e': replace = "\u03a9"; break;
754	case '\u1f6f': replace = "\u03a9"; break;
755	case '\u1f70': replace = "\u03b1"; break;
756	case '\u1f71': replace = "\u03b1"; break;
757	case '\u1f72': replace = "\u03b5"; break;
758	case '\u1f73': replace = "\u03b5"; break;
759	case '\u1f74': replace = "\u03b7"; break;
760	case '\u1f75': replace = "\u03b7"; break;
761	case '\u1f76': replace = "\u03b9"; break;
762	case '\u1f77': replace = "\u03b9"; break;
763	case '\u1f78': replace = "\u03bf"; break;
764	case '\u1f79': replace = "\u03bf"; break;
765	case '\u1f7a': replace = "\u03c5"; break;
766	case '\u1f7b': replace = "\u03c5"; break;
767	case '\u1f7c': replace = "\u03c9"; break;
768	case '\u1f7d': replace = "\u03c9"; break;
769	case '\u1f80': replace = "\u03b1"; break;
770	case '\u1f81': replace = "\u03b1"; break;
771	case '\u1f82': replace = "\u03b1"; break;
772	case '\u1f83': replace = "\u03b1"; break;
773	case '\u1f84': replace = "\u03b1"; break;
774	case '\u1f85': replace = "\u03b1"; break;
775	case '\u1f86': replace = "\u03b1"; break;
776	case '\u1f87': replace = "\u03b1"; break;
777	case '\u1f88': replace = "\u0391"; break;
778	case '\u1f89': replace = "\u0391"; break;
779	case '\u1f8a': replace = "\u0391"; break;
780	case '\u1f8b': replace = "\u0391"; break;
781	case '\u1f8c': replace = "\u0391"; break;
782	case '\u1f8d': replace = "\u0391"; break;
783	case '\u1f8e': replace = "\u0391"; break;
784	case '\u1f8f': replace = "\u0391"; break;
785	case '\u1f90': replace = "\u03b7"; break;
786	case '\u1f91': replace = "\u03b7"; break;
787	case '\u1f92': replace = "\u03b7"; break;
788	case '\u1f93': replace = "\u03b7"; break;
789	case '\u1f94': replace = "\u03b7"; break;
790	case '\u1f95': replace = "\u03b7"; break;
791	case '\u1f96': replace = "\u03b7"; break;
792	case '\u1f97': replace = "\u03b7"; break;
793	case '\u1f98': replace = "\u0397"; break;
794	case '\u1f99': replace = "\u0397"; break;
795	case '\u1f9a': replace = "\u0397"; break;
796	case '\u1f9b': replace = "\u0397"; break;
797	case '\u1f9c': replace = "\u0397"; break;
798	case '\u1f9d': replace = "\u0397"; break;
799	case '\u1f9e': replace = "\u0397"; break;
800	case '\u1f9f': replace = "\u0397"; break;
801	case '\u1fa0': replace = "\u03c9"; break;
802	case '\u1fa1': replace = "\u03c9"; break;
803	case '\u1fa2': replace = "\u03c9"; break;
804	case '\u1fa3': replace = "\u03c9"; break;
805	case '\u1fa4': replace = "\u03c9"; break;
806	case '\u1fa5': replace = "\u03c9"; break;
807	case '\u1fa6': replace = "\u03c9"; break;
808	case '\u1fa7': replace = "\u03c9"; break;
809	case '\u1fa8': replace = "\u03a9"; break;
810	case '\u1fa9': replace = "\u03a9"; break;
811	case '\u1faa': replace = "\u03a9"; break;
812	case '\u1fab': replace = "\u03a9"; break;
813	case '\u1fac': replace = "\u03a9"; break;
814	case '\u1fad': replace = "\u03a9"; break;
815	case '\u1fae': replace = "\u03a9"; break;
816	case '\u1faf': replace = "\u03a9"; break;
817	case '\u1fb2': replace = "\u03b1"; break;
818	case '\u1fb3': replace = "\u03b1"; break;
819	case '\u1fb4': replace = "\u03b1"; break;
820	case '\u1fb6': replace = "\u03b1"; break;
821	case '\u1fb7': replace = "\u03b1"; break;
822	case '\u1fba': replace = "\u0391"; break;
823	case '\u1fbb': replace = "\u0391"; break;
824	case '\u1fbc': replace = "\u0391"; break;
825	case '\u1fc2': replace = "\u03b7"; break;
826	case '\u1fc3': replace = "\u03b7"; break;
827	case '\u1fc4': replace = "\u03b7"; break;
828	case '\u1fc6': replace = "\u03b7"; break;
829	case '\u1fc7': replace = "\u03b7"; break;
830	case '\u1fca': replace = "\u0397"; break;
831	case '\u1fcb': replace = "\u0397"; break;
832	case '\u1fcc': replace = "\u0397"; break;
833	case '\u1fd2': replace = "\u03b9"; break;
834	case '\u1fd3': replace = "\u03b9"; break;
835	case '\u1fd6': replace = "\u03b9"; break;
836	case '\u1fd7': replace = "\u03b9"; break;
837	case '\u1fda': replace = "\u0399"; break;
838	case '\u1fdb': replace = "\u039f"; break;
839	case '\u1fe2': replace = "\u03c5"; break;
840	case '\u1fe3': replace = "\u03c5"; break;
841	case '\u1fe4': replace = "\u03c1"; break;
842	case '\u1fe5': replace = "\u03c1"; break;
843	case '\u1fe6': replace = "\u03c5"; break;
844	case '\u1fe7': replace = "\u03c5"; break;
845	case '\u1fea': replace = "\u03a5"; break;
846	case '\u1feb': replace = "\u03a5"; break;
847	case '\u1fec': replace = "\u03a1"; break;
848	case '\u1ff2': replace = "\u03c9"; break;
849	case '\u1ff3': replace = "\u03c9"; break;
850	case '\u1ff4': replace = "\u03c9"; break;
851	case '\u1ff6': replace = "\u03c9"; break;
852	case '\u1ff7': replace = "\u03c9"; break;
853	case '\u1ff8': replace = "\u039f"; break;
854	case '\u1ff9': replace = "\u039f"; break;
855	case '\u1ffa': replace = "\u03a9"; break;
856	case '\u1ffb': replace = "\u03a9"; break;
857	case '\u1ffc': replace = "\u03a9"; break;
858
859	case '<': break;
860	case '>': break;
861	case '-': break; // same treatment as soft hyphen
862	case '\u00ad': break; // soft hyphen
863	default: replace += c; break;
864	}
865	buf.append(replace);
866	// update offsets if replacement is a different length
867	if (offsets != null) {
868	int r = replace.length();
869	if (r == 0)
870	this.offsets = arrayKill(this.offsets, i - n);
871	else if (r == 2)
872	this.offsets = arrayInsert(this.offsets, i - n + 1, this.offsets[i - n], r - 1);
873	n += 1 - r;
874	}
875	}
876	return buf.toString();
877	} else { // unknown or no language
878	return s;
879	}
880	}
881
882	public String deNormalizeToRegExpr(String s) {
883	// TODO all characters in all languages
884	if (language.equals("la") \|\| language.equals("lat")) {
885	StringBuffer buf = new StringBuffer();
886	if (s.indexOf("ae") != -1) {
887	String str1 = s;
888	str1 = str1.replaceAll("ae", "\u0119");
889	String str2 = s;
890	str2 = str2.replaceAll("ae", "\u00c6");
891	String str3 = s;
892	str3 = str3.replaceAll("ae", "\u00e6");
893	buf.append(str1 + "\|" + str2 + "\|" + str3 + "\|");
894	}
895	if (s.indexOf("oe") != -1) {
896	String str1 = s;
897	str1 = str1.replaceAll("oe", "\u0152");
898	String str2 = s;
899	str2 = str2.replaceAll("oe", "\u0153");
900	buf.append(str1 + "\|" + str2 + "\|");
901	}
902	if (s.indexOf("ss") != -1) {
903	String str1 = s;
904	str1 = str1.replaceAll("ss", "\u00df");
905	buf.append(str1 + "\|");
906	}
907	boolean beginWord = true;
908	for (int i = 0; i < s.length(); i++) {
909	char c = s.charAt(i);
910	if (! beginWord)
911	c = Character.toLowerCase(c);
912	beginWord = Character.isWhitespace(c);
913	String replace = new String();
914	switch (c) {
915	case 'a': replace = "[a\u00c0\u00c1\u00c2\u00c4\u00e0\u00e1\u00e2\u00e4]"; break;
916	case 'c': replace = "[c\u00c7\u00e7]"; break;
917	case 'e': replace = "[e\u00c8\u00c9\u00ca\u00cb\u00e8\u00e9\u00ea\u00eb\u0113\u0115\u1ebd]"; break;
918	case 'i': replace = "[ij\u00cc\u00cd\u00ce\u00cf\u00ec\u00ed\u00ee\u00ef\u012a\u012b\u012c\u012d]"; break;
919	case 'o': replace = "[o\u00d2\u00d3\u00d4\u00d6\u00f2\u00f3\u00f4\u00f6\u014c\u014d\u014e\u014f]"; break;
920	case 'u': replace = "[uv\u00d9\u00da\u00db\u00dc\u00f9\u00fa\u00fb\u00fc\u016a\u016b\u016c\u016d]"; break;
921	case 's': replace = "[s\u017f]"; break;
922	default: replace += c; break;
923	}
924	buf.append(replace);
925	}
926	return buf.toString();
927	} else if (language.equals("en")) {
928	StringBuffer buf = new StringBuffer();
929	if (s.indexOf("ae") != -1) {
930	String str1 = s;
931	str1 = str1.replaceAll("ae", "\u0119");
932	String str2 = s;
933	str2 = str2.replaceAll("ae", "\u00c6");
934	String str3 = s;
935	str3 = str3.replaceAll("ae", "\u00e6");
936	buf.append(str1 + "\|" + str2 + "\|" + str3 + "\|");
937	}
938	if (s.indexOf("oe") != -1) {
939	String str1 = s;
940	str1 = str1.replaceAll("oe", "\u0152");
941	String str2 = s;
942	str2 = str2.replaceAll("oe", "\u0153");
943	buf.append(str1 + "\|" + str2 + "\|");
944	}
945	if (s.indexOf("ss") != -1) {
946	String str1 = s;
947	str1 = str1.replaceAll("ss", "\u00df");
948	buf.append(str1 + "\|");
949	}
950	boolean beginWord = true;
951	for (int i = 0; i < s.length(); i++) {
952	char c = s.charAt(i);
953	if (! beginWord)
954	c = Character.toLowerCase(c);
955	beginWord = Character.isWhitespace(c);
956	String replace = new String();
957	switch (c) {
958	case 'a': replace = "[a\u00c0\u00c1\u00c2\u00c4\u00e0\u00e1\u00e2\u00e4]"; break;
959	case 'c': replace = "[c\u00c7\u00e7]"; break;
960	case 'e': replace = "[e\u00c8\u00c9\u00ca\u00cb\u00e8\u00e9\u00ea\u00eb\u0113\u0115\u1e8d]"; break;
961	case 'i': replace = "[i\u00cc\u00cd\u00ce\u00cf\u00ec\u00ed\u00ee\u00ef\u012a\u012b\u012c\u012d]"; break;
962	case 'o': replace = "[o\u00d2\u00d3\u00d4\u00d6\u00f2\u00f3\u00f4\u00f6\u014c\u014d\u014e\u014f]‚"; break;
963	case 'u': replace = "[u\u00d9\u00da\u00db\u00dc\u00f9\u00fa\u00fb\u00fc\u016a\u016b\u016c\u016d]"; break;
964	case 's': replace = "[s\u017f]"; break;
965	default: replace += c; break;
966	}
967	buf.append(replace);
968	}
969	return buf.toString();
970	} else if (language.equals("de")) {
971	StringBuffer buf = new StringBuffer();
972	if (s.indexOf("ss") != -1) {
973	String str1 = s;
974	str1 = str1.replaceAll("ss", "\u00df");
975	buf.append(str1 + "\|");
976	}
977	if (s.indexOf("ae") != -1) {
978	String str1 = s;
979	str1 = str1.replaceAll("ae", "\u00e4");
980	buf.append(str1 + "\|");
981	}
982	if (s.indexOf("oe") != -1) {
983	String str1 = s;
984	str1 = str1.replaceAll("oe", "\u00f6");
985	buf.append(str1 + "\|");
986	}
987	if (s.indexOf("ue") != -1) {
988	String str1 = s;
989	str1 = str1.replaceAll("ue", "\u00fc");
990	buf.append(str1 + "\|");
991	}
992	boolean beginWord = true;
993	for (int i = 0; i < s.length(); i++) {
994	char c = s.charAt(i);
995	if (! beginWord)
996	c = Character.toLowerCase(c);
997	beginWord = Character.isWhitespace(c);
998	String replace = new String();
999	switch (c) {
1000	case 'e': replace = "[e\u00e9]"; break;
1001	default: replace += c; break;
1002	}
1003	buf.append(replace);
1004	}
1005	return buf.toString();
1006	} else { // unknown or no language
1007	return s;
1008	}
1009	}
1010
1011	/**
1012	* Returns the offset table.
1013	*
1014	* @return offset table
1015	*/
1016	public int[] getOffsetTable() {
1017	return offsets;
1018	}
1019
1020	/**
1021	* Returns a copy of an integer array with the element at
1022	* <code>index</code> removed ("killed").
1023	*
1024	* @param array integer array
1025	* @param index index of element to remove
1026	*/
1027	static private int[] arrayKill(int[] array, int index) {
1028	int[] newArray = new int[array.length - 1];
1029	System.arraycopy(array, 0, newArray, 0, index);
1030	System.arraycopy(array, index + 1, newArray, index, array.length - index - 1);
1031	return newArray;
1032	}
1033
1034	/**
1035	* Returns a copy of an integer array with <code>count</code> elements
1036	* inserted at <code>index</code>.
1037	*
1038	* @param array integer array
1039	* @param index index to insert new elements
1040	* @param value value to insert into new slots
1041	* @param count number of new slots to insert
1042	*/
1043	static private int[] arrayInsert(int[] array, int index, int value, int count) {
1044	int[] newArray = new int[array.length + count];
1045	System.arraycopy(array, 0, newArray, 0, index);
1046	for (int i = 0; i < count; i++) newArray[index + i] = value;
1047	System.arraycopy(array, index, newArray, index + count, array.length - index);
1048	return newArray;
1049	}
1050
1051	/**
1052	* We provide <code>main()</code> so that our services will be available
1053	* outside Java (i.e., so we can run as a Un*x-style filter).
1054	*/
1055	static public void main(String[] argv) throws ApplicationException {
1056	if (argv.length != 1) {
1057	System.err.println("You must specify a language.");
1058	System.exit(1);
1059	}
1060	String rec;
1061	StringBuffer buf = new StringBuffer();
1062	BufferedReader bin = null;
1063	try {
1064	bin = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
1065	while ((rec = bin.readLine()) != null)
1066	buf.append(rec + "\n");
1067	}
1068	catch (UnsupportedEncodingException e) {
1069	System.err.println(e);
1070	System.exit(1);
1071	} catch (IOException e) {
1072	System.err.println(e);
1073	System.exit(1);
1074	}
1075	MpdlNormalizer orth = new MpdlNormalizer(argv[0]);
1076	System.out.print(orth.normalize(buf.toString()));
1077	}
1078	}

Download in other formats:

Original Format