Context Navigation

Back to normalization/1

normalization/1: OrthographicRules.java

File OrthographicRules.java, 34.6 KB (added by Wolfgang Schmidle, 13 years ago)

Line
1	import java.io.*;
2
3	/**
4	* This class provides pluggable rules for orthographic normalization, with
5	* offset tracking.
6	*
7	* @author Malcolm D. Hyman
8	* @version $2007-06-12 18:58:58 mdh$
9	*/
10	public class OrthographicRules {
11	private String ruleset;
12	public int[] offsets;
13
14	static final public String IT_VOWELS = "AEIOUaeiou" +
15	"\u00c6\u00e6" + // AE ligatures
16	"\u0152\u0153"; // OE ligatures
17	static final public String IT_CONS = "BCDFGHKLMNPQRSTVWXZ" +
18	"bcdfghklmnpqrstvwxz" +
19	"\u017f\u00df"; // long/sharp S
20
21
22	/**
23	* Constructor.
24	*
25	* @param ruleset name of rule set to apply
26	*/
27	public OrthographicRules(String ruleset) {
28	this.ruleset = ruleset;
29	}
30
31	/**
32	* Applies the normalization rules in <code>ruleset</code> to
33	* <code>s</code>, with offset tracking.<p>
34	*
35	* <strong>WARNING:</strong>
36	* Arboreal will not work properly if a normalization substitution
37	* replaces a source character with more than two target characters!
38	* This is simply a BUG, and should be fixed. Fortunately, however,
39	* one does not often need such a replacement.<p>
40	*
41	* FIXME: If the orthographic rules eliminate all characters in a
42	* word, word counting in ContentRenderPane will not work correctly!
43	*
44	* @param s source string
45	* @param offsets character offset table
46	* @return normalized string
47	*/
48	public String normalize(String s, int[] offsets) {
49	this.offsets = offsets;
50	if (ruleset.equals("la") \|\| ruleset.equals("lat")) {
51	StringBuffer buf = new StringBuffer();
52	boolean beginWord = true;
53	int n = 0;
54	for (int i = 0; i < s.length(); i++) {
55	char c = s.charAt(i);
56	if (! beginWord) c = Character.toLowerCase(c);
57	beginWord = Character.isWhitespace(c);
58	String replace = new String();
59
60	// names in comments are from Unicode Standard
61	switch (c) {
62	case 'j': replace = "i"; break;
63	case 'v': replace = "u"; break;
64
65	/*
66	* Linguistic note: /u/ and /v/ are rarely phonemic
67	* in Latin, as in alui 's/he nourished' vs.
68	* alvi 'of a belly', volui 's/he wished' or 'it rolled'
69	* vs. volvi 'to be rolled', (in)seruit 's/he joined
70	* together' vs. (in)servit 's/he serves'.
71	*/
72
73	case 'q':
74	if ((i < s.length() - 1) && (s.charAt(i + 1) == ';'))
75	replace = "qu";
76	else
77	replace = "q";
78	break;
79	case ';':
80	if ((i > 0) && (s.charAt(i - 1) == 'q'))
81	replace = "e";
82	else if ((i == 0) \|\| ! Character.isLetter(s.charAt(i - 1)))
83	replace = ";";
84	else
85	replace = "";
86	break;
87
88	case '\u0300': replace = ""; break; // COMBINING GRAVE ACCENT
89	case '\u0301': replace = ""; break; // COMBINING ACCUTE ACCENT
90	case '\u0302': replace = ""; break; // COMBINING CIRCUMFLEX ACCENT
91
92	case '\u00c0': replace = "A"; break; // LATIN CAPITAL LETTER A GRAVE
93	case '\u00c1': replace = "A"; break; // LATIN CAPITAL LETTER A ACUTE
94	case '\u00c2': replace = "A"; break; // LATIN CAPITAL LETTER A CIRCUMFLEX
95	case '\u00c4': replace = "A"; break; // LATIN CAPITAL LETTER A DIAERESIS
96	case '\u00c6': replace = "Ae"; break; // LATIN CAPITAL LETTER A E
97	case '\u00c7': replace = "C"; break; // LATIN CAPITAL LETTER C CEDILLA
98	case '\u00c8': replace = "E"; break; // LATIN CAPITAL LETTER E GRAVE
99	case '\u00c9': replace = "E"; break; // LATIN CAPITAL LETTER E ACUTE
100	case '\u00ca': replace = "E"; break; // LATIN CAPITAL LETTER E CIRCUMFLEX
101	case '\u00cb': replace = "E"; break; // LATIN CAPITAL LETTER E DIAERESIS
102	case '\u00cc': replace = "I"; break; // LATIN CAPITAL LETTER I GRAVE;
103	case '\u00cd': replace = "I"; break; // LATIN CAPITAL LETTER I ACUTE
104	case '\u00ce': replace = "I"; break; // LATIN CAPITAL LETTER I CIRCUMFLEX
105	case '\u00cf': replace = "I"; break; // LATIN CAPITAL LETTER I DIAERESIS
106	case '\u00d2': replace = "O"; break; // LATIN CAPITAL LETTER O GRAVE
107	case '\u00d3': replace = "O"; break; // LATIN CAPITAL LETTER O ACUTE
108	case '\u00d4': replace = "O"; break; // LATIN CAPITAL LETTER O CIRCUMFLEX
109	case '\u00d6': replace = "O"; break; // LATIN CAPITAL LETTER O DIAERESIS
110	case '\u00d9': replace = "U"; break; // LATIN CAPITAL LETTER U GRAVE
111	case '\u00da': replace = "U"; break; // LATIN CAPITAL LETTER U ACUTE
112	case '\u00db': replace = "U"; break; // LATIN CAPITAL LETTER U CIRCUMFLEX
113	case '\u00dc': replace = "U"; break; // LATIN CAPITAL LETTER U DIAERESIS
114	case '\u00e0': replace = "a"; break; // LATIN SMALL LETTER A GRAVE
115	case '\u00e1': replace = "a"; break; // LATIN SMALL LETTER A ACUTE
116	case '\u00e2': replace = "a"; break; // LATIN SMALL LETTER A CIRCUMFLEX
117	case '\u00e4': replace = "a"; break; // LATIN SMALL LETTER A DIAERESIS
118	case '\u00e6': replace = "ae"; break; // LATIN SMALL LETTER A E
119	case '\u00e7': replace = "c"; break; // LATIN SMALL LETTER C CEDILLA
120	case '\u00e8': replace = "e"; break; // LATIN SMALL LETTER E GRAVE
121	case '\u00e9': replace = "e"; break; // LATIN SMALL LETTER E ACUTE
122	case '\u00ea': replace = "e"; break; // LATIN SMALL LETTER E CIRCUMFLEX
123	case '\u00eb': replace = "e"; break; // LATIN SMALL LETTER E DIAERESIS
124	case '\u00ec': replace = "i"; break; // LATIN SMALL LETTER I GRAVE
125	case '\u00ed': replace = "i"; break; // LATIN SMALL LETTER I ACUTE
126	case '\u00ee': replace = "i"; break; // LATIN SMALL LETTER I CIRCUMFLEX
127	case '\u00ef': replace = "i"; break; // LATIN SMALL LETTER I DIAERESIS
128	case '\u00f2': replace = "o"; break; // LATIN SMALL LETTER O GRAVE
129	case '\u00f3': replace = "o"; break; // LATIN SMALL LETTER O ACUTE
130	case '\u00f4': replace = "o"; break; // LATIN SMALL LETTER O CIRCUMFLEX
131	case '\u00f6': replace = "o"; break; // LATIN SMALL LETTER O DIAERESIS
132	case '\u00f9': replace = "u"; break; // LATIN SMALL LETTER U GRAVE
133	case '\u00fa': replace = "u"; break; // LATIN SMALL LETTER U ACUTE
134	case '\u00fb': replace = "u"; break; // LATIN SMALL LETTER U CIRCUMFLEX
135	case '\u00fc': replace = "u"; break; // LATIN SMALL LETTER U DIAERESIS
136	case '\u0100': replace = "A"; break; // LATIN CAPITAL LETTER A MACRON
137	case '\u0101': replace = "a"; break; // LATIN SMALL LETTER A MACRON
138	case '\u0102': replace = "A"; break; // LATIN CAPITAL LETTER A BREVE
139	case '\u0103': replace = "a"; break; // LATIN SMALL LETTER A BREVE
140	case '\u0112': replace = "E"; break; // LATIN CAPITAL LETTER E MACRON
141	case '\u0113': replace = "e"; break; // LATIN SMALL LETTER E MACRON
142	case '\u0114': replace = "E"; break; // LATIN CAPITAL LETTER E BREVE
143	case '\u0115': replace = "e"; break; // LATIN SMALL LETTER E BREVE
144	case '\u0118': replace = "Ae"; break; // LATIN CAPITAL LETTER E OGONEK
145	case '\u0119': replace = "ae"; break; // LATIN SMALL LETTER E OGONEK
146	case '\u012a': replace = "I"; break; // LATIN CAPITAL LETTER I MACRON
147	case '\u012b': replace = "i"; break; // LATIN SMALL LETTER I MACRON
148	case '\u012c': replace = "I"; break; // LATIN CAPITAL LETTER I BREVE
149	case '\u012d': replace = "i"; break; // LATIN SMALL LETTER I BREVE
150	case '\u014c': replace = "O"; break; // LATIN CAPITAL LETTER O MACRON
151	case '\u014d': replace = "o"; break; // LATIN SMALL LETTER O MACRON
152	case '\u014e': replace = "O"; break; // LATIN CAPITAL LETTER O BREVE
153	case '\u014f': replace = "o"; break; // LATIN SMALL LETTER O BREVE
154	case '\u0152': replace = "Oe"; break; // LATIN CAPITAL LETTER O E
155	case '\u0153': replace = "oe"; break; // LATIN SMALL LETTER O E
156	case '\u016a': replace = "U"; break; // LATIN CAPITAL LETTER U MACRON
157	case '\u016b': replace = "u"; break; // LATIN SMALL LETTER U MACRON
158	case '\u016c': replace = "U"; break; // LATIN CAPITAL LETTER U BREVE
159	case '\u016d': replace = "u"; break; // LATIN SMALL LETTER U BREVE
160	case '\u017f': replace = "s"; break; // LATIN SMALL LETTER LONG S
161	case '\u00df': replace = "ss"; break; // LATIN SMALL LETTER SHARP S
162	case '\u00ad': break; // soft hyphen
163	case '\u2329': break; // BRA
164	case '\u232a': break; // KET
165	default: replace += c; break;
166	}
167
168	buf.append(replace);
169
170	// update offsets if replacement is a different length
171	if (offsets != null) {
172	int r = replace.length();
173	if (r == 0) this.offsets = arrayKill(this.offsets, i - n);
174	else if (r == 2)
175	this.offsets =
176	arrayInsert(this.offsets, i - n + 1,
177	this.offsets[i - n], r - 1);
178	n += 1 - r;
179	}
180	}
181	return buf.toString();
182	}
183
184	else if (ruleset.equals("it")) {
185	StringBuffer buf = new StringBuffer();
186	boolean beginWord = true;
187	int n = 0;
188	for (int i = 0; i < s.length(); i++) {
189	char c = s.charAt(i);
190	if (! beginWord) c = Character.toLowerCase(c);
191	String replace = new String();
192
193	if ((c == 'u') && (i > 0) && (i < s.length() - 1)) {
194	char c_n = s.charAt(i + 1);
195	char c_p;
196	int y = i - 1;
197	while ((((c_p = s.charAt(y)) == '-') \|\|
198	(c_p == '\u00ad')) && (y-- > 0));
199	if ((IT_VOWELS.indexOf(c_p) > -1) &&
200	(IT_VOWELS.indexOf(c_n) > -1)) replace = "v";
201	else replace = "u";
202	}
203	else if ((c == 'j') && (i > 0)) {
204	char c_p = Character.toLowerCase(s.charAt(i - 1));
205	if (c_p == 'i') replace = "i";
206	else replace = "j";
207	}
208	else if ((c == 'v') && (i < s.length() - 1)) {
209	char c_p;
210	char c_n;
211	int y = i + 1;
212	while ((((c_n = s.charAt(y)) == '-') \|\|
213	(c_n == '\u00ad')) && (y++ > 0));
214	if ((beginWord \|\|
215	(! Character.isLetter(c_p = s.charAt(i - 1))) \|\|
216	IT_CONS.indexOf(c_p) > -1) &&
217	IT_CONS.indexOf(c_n) > -1)
218	replace = "u"; else replace = "v";
219	}
220	else if ((c == 't') && (i < s.length() - 2)) {
221	char c_n = Character.toLowerCase(s.charAt(i + 1));
222	char c_n2 = Character.toLowerCase(s.charAt(i + 2));
223	if ((c_n == 'i') && (c_n2 == 'o'))
224	replace = "z";
225	else
226	replace = "t";
227	}
228	/*
229	* 'h' only appears at the beginning of some pres. indic.
230	* forms of 'avere' -- with the exception of some unimportant
231	* loanwords in modern Italian (which I choose to ignore)
232	*/
233	else if (c == 'h') {
234	int l = s.length();
235	if ((i < l - 1) &&
236	(beginWord \|\|
237	((i > 0) &&
238	(! Character.isLetter(s.charAt(i - 1)))))) {
239	char c_n = Character.toLowerCase(s.charAt(i + 1));
240	if (c_n == 'a') {
241	if (i < l - 2) {
242	char c_n2 =
243	Character.toLowerCase(s.charAt(i + 2));
244	if (((! Character.isLetter(c_n2)) && // 'ha'
245	(c_n2 != '-') && (c_n2 != '\u00ad')) \|\|
246	(c_n2 == 'i') \|\| // 'hai'
247	(c_n2 == 'n')) { // 'hanno'
248	replace = "h";
249	}
250	else {
251	replace = "";
252	}
253	}
254	else {
255	replace = "h"; // 'ha'
256	}
257	}
258	else if (c_n == 'o') {
259	if (i < l - 2) {
260	char c_n2 =
261	Character.toLowerCase(s.charAt(i + 2));
262	if (! Character.isLetter(c_n2)) {
263	replace = "h";
264	}
265	else {
266	replace = "";
267	}
268	}
269	else {
270	replace = "h"; // 'ho'
271	}
272	}
273	else {
274	replace = "";
275	}
276	}
277	else {
278	replace = "h"; // not word initial (or else isolated)
279	}
280	}
281	else if ((c == '\u00ad') \|\| (c == '-')) { // (soft) hyphen
282	if ((i < s.length() - 1) &&
283	(! Character.isWhitespace(s.charAt(i + 1)))) {
284	replace = "";
285	}
286	else {
287	replace = "-";
288	}
289	}
290	else {
291	switch (c) {
292	case '\u00c1': replace = "\u00c0"; break; // LATIN CAPITAL LETTER A ACUTE
293	case '\u00c6': replace = "Ae"; break; // LATIN CAPITAL LETTER A E
294	case '\u00c9': replace = "\u00c8"; break; // LATIN CAPITAL LETTER E ACUTE
295	case '\u00cd': replace = "\u00cc"; break; // LATIN CAPITAL LETTER I ACUTE
296	case '\u00d3': replace = "\u00d2"; break; // LATIN CAPITAL LETTER O ACUTE
297	case '\u00da': replace = "\u00d9"; break; // LATIN CAPITAL LETTER U ACUTE
298	case '\u00e1': replace = "\u00e0"; break; // LATIN SMALL LETTER A ACUTE
299	case '\u00e6': replace = "ae"; break; // LATIN SMALL LETTER A E
300	case '\u00e9': replace = "\u00e8"; break; // LATIN SMALL LETTER E ACUTE
301	case '\u00ed': replace = "\u00ec"; break; // LATIN SMALL LETTER I ACUTE
302	case '\u00f3': replace = "\u00f2"; break; // LATIN SMALL LETTER O ACUTE
303	case '\u00fa': replace = "\u00f9"; break; // LATIN SMALL LETTER U ACUTE
304	case '\u0152': replace = "Oe"; break; // LATIN CAPITAL LETTER O E
305	case '\u0153': replace = "oe"; break; // LATIN SMALL LETTER O E
306	case '\u017f': replace = "s"; break; // LATIN SMALL LETTER LONG S
307	case '\u00df': replace = "ss"; break; // LATIN SMALL LETTER SHARP S
308	default: replace += c; break;
309	}
310	}
311	beginWord = Character.isWhitespace(c) \|\|
312	(c == '\u2019') \|\| (c == '\'');
313	buf.append(replace);
314
315	// update offsets if replacement is a different length
316	if (offsets != null) {
317	int r = replace.length();
318	if (r == 0) this.offsets = arrayKill(this.offsets, i - n);
319	else if (r == 2)
320	this.offsets =
321	arrayInsert(this.offsets, i - n + 1,
322	this.offsets[i - n], r - 1);
323	n += 1 - r;
324	}
325	}
326	return buf.toString();
327	}
328
329	else if (ruleset.equals("fr")) {
330	StringBuffer buf = new StringBuffer();
331	boolean beginWord = true;
332	int n = 0;
333	for (int i = 0; i < s.length(); i++) {
334	char c = s.charAt(i);
335	if (! beginWord) c = Character.toLowerCase(c);
336	String replace = new String();
337
338	if ((c == 'u') && (i > 0) && (i < s.length() - 1)) {
339	char c_n = s.charAt(i + 1);
340	char c_p;
341	int y = i - 1;
342	while ((((c_p = s.charAt(y)) == '-') \|\|
343	(c_p == '\u00ad')) && (y-- > 0));
344	// sic: OK, I think, to use IT_VOWELS
345	if ((IT_VOWELS.indexOf(c_p) > -1) &&
346	(IT_VOWELS.indexOf(c_n) > -1)) replace = "v";
347	else replace = "u";
348	}
349	else if ((c == 'v') && (i < s.length() - 1)) {
350	char c_p;
351	char c_n;
352	int y = i + 1;
353	while ((((c_n = s.charAt(y)) == '-') \|\|
354	(c_n == '\u00ad')) && (y++ > 0));
355	if ((beginWord \|\|
356	(! Character.isLetter(c_p = s.charAt(i - 1))) \|\|
357	IT_CONS.indexOf(c_p) > -1) &&
358	IT_CONS.indexOf(c_n) > -1)
359	replace = "u"; else replace = "v";
360	}
361	else {
362	switch (c) {
363	case '\u00e6': replace = "ae"; break; // LATIN SMALL LETTER A E
364	case '\u017f': replace = "s"; break; // LATIN SMALL LETTER LONG S
365	case '\u00df': replace = "ss"; break; // LATIN SMALL LETTER SHARP S
366	case '\u00ad': break; // soft hyphen
367	case '-': break;
368	default: replace += c; break;
369	}
370	}
371	beginWord = Character.isWhitespace(c);
372	buf.append(replace);
373
374	// update offsets if replacement is a different length
375	if (offsets != null) {
376	int r = replace.length();
377	if (r == 0) this.offsets = arrayKill(this.offsets, i - n);
378	else if (r == 2)
379	this.offsets =
380	arrayInsert(this.offsets, i - n + 1,
381	this.offsets[i - n], r - 1);
382	n += 1 - r;
383	}
384	}
385	return buf.toString();
386	}
387
388	else if (ruleset.equals("de")) {
389	StringBuffer buf = new StringBuffer();
390	boolean beginWord = true;
391	int n = 0;
392	for (int i = 0; i < s.length(); i++) {
393	char c = s.charAt(i);
394	if (! beginWord) c = Character.toLowerCase(c);
395	beginWord = Character.isWhitespace(c) \|\|
396	(" .,:;?!(<['\"\u2018\u201c".indexOf(c) > -1);
397	String replace = new String();
398
399	switch (c) {
400	case '\u00c4': replace = "Ae"; break;
401	case '\u00d6': replace = "Oe"; break;
402	case '\u00dc': replace = "Ue"; break;
403	case '\u00df': replace = "ss"; break;
404	case '\u00e4': replace = "ae"; break;
405	case '\u00f6': replace = "oe"; break;
406	case '\u00fc': replace = "ue"; break;
407	case '\u00e9': replace = "e"; break;
408	case '\u00ad': break; // soft hyphen
409	case '-': break;
410	default: replace += c; break;
411	}
412
413	buf.append(replace);
414
415	// update offsets if replacement is a different length
416	if (offsets != null) {
417	int r = replace.length();
418	if (r == 0) this.offsets = arrayKill(this.offsets, i - n);
419	else if (r == 2)
420	this.offsets =
421	arrayInsert(this.offsets, i - n + 1,
422	this.offsets[i - n], r - 1);
423	n += 1 - r;
424	}
425	}
426	return buf.toString();
427	}
428
429	else if (ruleset.equals("zh")) {
430	StringBuffer buf = new StringBuffer();
431	int n = 0;
432	for (int i = 0; i < s.length(); i++) {
433	char c = s.charAt(i);
434	String replace = new String();
435
436	switch (c) {
437	case '\u00b9': replace = "1"; break;
438	case '\u00b2': replace = "2"; break;
439	case '\u00b3': replace = "3"; break;
440	case '\u2074': replace = "4"; break;
441	case '\u2075': replace = "5"; break;
442	case '\u3000': replace = " "; break;
443	case '\u3001': replace = ","; break;
444	case '\u3002': replace = "."; break;
445	// case '\u200b': break; // BREAKS EVERYTHING!
446	default: replace += c; break;
447	}
448
449	buf.append(replace);
450
451	// update offsets if replacement is a different length
452	if (offsets != null) {
453	int r = replace.length();
454	if (r == 0) this.offsets = arrayKill(this.offsets, i - n);
455	else if (r == 2)
456	this.offsets =
457	arrayInsert(this.offsets, i - n + 1,
458	this.offsets[i - n], r - 1);
459	n += 1 - r;
460	}
461	}
462	return buf.toString();
463	}
464
465	else if (ruleset.equals("ar")) {
466	StringBuffer buf = new StringBuffer();
467	boolean beginWord = true;
468	int n = 0;
469	for (int i = 0; i < s.length(); i++) {
470	char c = s.charAt(i);
471	beginWord = Character.isWhitespace(c);
472	String replace = new String();
473
474	switch (c) {
475	case '_': break; // TATWEEL in Buckwalter
476	case 'o': break; // SUKUN in Buckwalter
477	case '\u0640': break; // TATWEEL
478	case '\u0652': break; // SUKUN
479	case '^': break; // Buckwalter extension for capitalization
480	case '\u200f': break; // R-L MARK
481	default: replace += c; break;
482	}
483
484	buf.append(replace);
485
486	// update offsets if replacement is a different length
487	if (offsets != null) {
488	int r = replace.length();
489	if (r == 0) this.offsets = arrayKill(this.offsets, i - n);
490	else if (r == 2)
491	this.offsets =
492	arrayInsert(this.offsets, i - n + 1,
493	this.offsets[i - n], r - 1);
494	n += 1 - r;
495	}
496	}
497	return buf.toString();
498	}
499
500	else if (ruleset.equals("akk") \|\|
501	ruleset.equals("qam") \|\|
502	ruleset.equals("qpc") \|\|
503	ruleset.equals("elx") \|\|
504	ruleset.equals("sux") \|\|
505	ruleset.equals("hit") \|\|
506	ruleset.equals("qhu") \|\|
507	ruleset.equals("peo") \|\|
508	ruleset.equals("uga") \|\|
509	ruleset.equals("ura") \|\|
510	ruleset.equals("qcu")) {
511
512
513	StringBuffer buf = new StringBuffer();
514	int n = 0;
515	char last = '\u0000';
516	for (int i = 0; i < s.length(); i++) {
517	char c = s.charAt(i);
518	c = Character.toLowerCase(c);
519	String replace = new String();
520
521	switch (c) {
522	case '{': replace += "-"; break;
523	case '}': replace += "-"; break;
524
525	// These are from PSD::ATF::Unicode by Steve Tinney
526	case '\u0161': replace += "sz"; break;
527	case '\u1e63': replace += "s,"; break;
528	case '\u1e6d': replace += "t,"; break;
529	case '\u014b': replace += "j"; break;
530	case '\u015b': replace += "s'"; break;
531	case '\u2080': replace += "0"; break;
532	case '\u2081': replace += "1"; break;
533	case '\u2082': replace += "2"; break;
534	case '\u2083': replace += "3"; break;
535	case '\u2084': replace += "4"; break;
536	case '\u2085': replace += "5"; break;
537	case '\u2086': replace += "6"; break;
538	case '\u2087': replace += "7"; break;
539	case '\u2088': replace += "8"; break;
540	case '\u2089': replace += "9"; break;
541
542	case 'c': // shin (except where used as modifier)
543	if ((i > 0) && ((last == '~') \|\| (last == '@')))
544	replace += "c";
545	else replace += "sz";
546	break;
547
548	default: replace += c; break;
549	}
550
551	// suppress grapheme boundary before or after word boundary
552	if (replace.equals("-")) {
553	if ((i + 1 == s.length()) \|\| (s.charAt(i + 1) == ' ') \|\|
554	(i == 0) \|\| (s.charAt(i - 1) == ' ')) replace = "";
555	}
556
557	last = c;
558
559	buf.append(replace);
560
561	// update offsets if replacement is a different length
562	if (offsets != null) {
563	int r = replace.length();
564	if (r == 0) this.offsets = arrayKill(this.offsets, i - n);
565	else if (r == 2)
566	this.offsets =
567	arrayInsert(this.offsets, i - n + 1,
568	this.offsets[i - n], r - 1);
569	n += 1 - r;
570	}
571	}
572	return buf.toString();
573	}
574
575	if (ruleset.equals("el")) {
576	StringBuffer buf = new StringBuffer();
577	boolean beginWord = true;
578	int n = 0;
579	for (int i = 0; i < s.length(); i++) {
580	char c = s.charAt(i);
581	if (! beginWord) c = Character.toLowerCase(c);
582	beginWord = Character.isWhitespace(c);
583	String replace = new String();
584
585	switch (c) {
586	case '\u03c2': replace = "\u03c3"; break; // GREEK SMALL LETTER FINAL SIGMA
587	case '<': break;
588	case '>': break;
589	case '[': break;
590	case ']': break;
591	case '1': break;
592	case '2': break;
593	case '\u03ac': replace = "\u1f71"; break;
594	case '\u03ad': replace = "\u1f73"; break;
595	case '\u03ae': replace = "\u1f75"; break;
596	case '\u03af': replace = "\u1f77"; break;
597	case '\u03cc': replace = "\u1f79"; break;
598	case '\u03cd': replace = "\u1f7b"; break;
599	case '\u03ce': replace = "\uu1f7d"; break;
600	case '-': break; // same treatment as soft hyphen
601	case '\u00ad': break; // soft hyphen
602	default: replace += c; break;
603	}
604
605	buf.append(replace);
606
607	// update offsets if replacement is a different length
608	if (offsets != null) {
609	int r = replace.length();
610	if (r == 0) this.offsets = arrayKill(this.offsets, i - n);
611	else if (r == 2)
612	this.offsets =
613	arrayInsert(this.offsets, i - n + 1,
614	this.offsets[i - n], r - 1);
615	n += 1 - r;
616	}
617	}
618	return buf.toString();
619	}
620
621	if (ruleset.equals("el_atonic")) {
622	StringBuffer buf = new StringBuffer();
623	boolean beginWord = true;
624	int n = 0;
625	for (int i = 0; i < s.length(); i++) {
626	char c = s.charAt(i);
627	if (! beginWord) c = Character.toLowerCase(c);
628	beginWord = Character.isWhitespace(c);
629	String replace = new String();
630
631	switch (c) {
632	case '\u03c2': replace = "\u03c3"; break; // GREEK SMALL LETTER FINAL SIGMA
633	// map characters with diacritics to their plain equivalent
634	// cf. <code>BetaCode.java</code>
635	case '\u03aa': replace = "\u0399"; break;
636	case '\u03ab': replace = "\u03a5"; break;
637	case '\u03ac': replace = "\u0381"; break;
638	case '\u03ad': replace = "\u0385"; break;
639	case '\u03ae': replace = "\u0387"; break;
640	case '\u03af': replace = "\u0389"; break;
641	case '\u03ca': replace = "\u03b9"; break;
642	case '\u03cb': replace = "\u03c5"; break;
643	case '\u03cc': replace = "\u03bf"; break;
644	case '\u03cd': replace = "\u03c5"; break;
645	case '\u03ce': replace = "\u03c9"; break;
646	case '\u1f00': replace = "\u03b1"; break;
647	case '\u1f01': replace = "\u03b1"; break;
648	case '\u1f02': replace = "\u03b1"; break;
649	case '\u1f03': replace = "\u03b1"; break;
650	case '\u1f04': replace = "\u03b1"; break;
651	case '\u1f05': replace = "\u03b1"; break;
652	case '\u1f06': replace = "\u03b1"; break;
653	case '\u1f07': replace = "\u03b1"; break;
654	case '\u1f08': replace = "\u0391"; break;
655	case '\u1f09': replace = "\u0391"; break;
656	case '\u1f0a': replace = "\u0391"; break;
657	case '\u1f0b': replace = "\u0391"; break;
658	case '\u1f0c': replace = "\u0391"; break;
659	case '\u1f0d': replace = "\u0391"; break;
660	case '\u1f0e': replace = "\u0391"; break;
661	case '\u1f0f': replace = "\u0391"; break;
662	case '\u1f10': replace = "\u03b5"; break;
663	case '\u1f11': replace = "\u03b5"; break;
664	case '\u1f12': replace = "\u03b5"; break;
665	case '\u1f13': replace = "\u03b5"; break;
666	case '\u1f14': replace = "\u03b5"; break;
667	case '\u1f15': replace = "\u03b5"; break;
668	case '\u1f18': replace = "\u0395"; break;
669	case '\u1f19': replace = "\u0395"; break;
670	case '\u1f1a': replace = "\u0395"; break;
671	case '\u1f1b': replace = "\u0395"; break;
672	case '\u1f1c': replace = "\u0395"; break;
673	case '\u1f1d': replace = "\u0395"; break;
674	case '\u1f20': replace = "\u03b7"; break;
675	case '\u1f21': replace = "\u03b7"; break;
676	case '\u1f22': replace = "\u03b7"; break;
677	case '\u1f23': replace = "\u03b7"; break;
678	case '\u1f24': replace = "\u03b7"; break;
679	case '\u1f25': replace = "\u03b7"; break;
680	case '\u1f26': replace = "\u03b7"; break;
681	case '\u1f27': replace = "\u03b7"; break;
682	case '\u1f28': replace = "\u0397"; break;
683	case '\u1f29': replace = "\u0397"; break;
684	case '\u1f2a': replace = "\u0397"; break;
685	case '\u1f2b': replace = "\u0397"; break;
686	case '\u1f2c': replace = "\u0397"; break;
687	case '\u1f2d': replace = "\u0397"; break;
688	case '\u1f2e': replace = "\u0397"; break;
689	case '\u1f2f': replace = "\u0397"; break;
690	case '\u1f30': replace = "\u03b9"; break;
691	case '\u1f31': replace = "\u03b9"; break;
692	case '\u1f32': replace = "\u03b9"; break;
693	case '\u1f33': replace = "\u03b9"; break;
694	case '\u1f34': replace = "\u03b9"; break;
695	case '\u1f35': replace = "\u03b9"; break;
696	case '\u1f36': replace = "\u03b9"; break;
697	case '\u1f37': replace = "\u03b9"; break;
698	case '\u1f38': replace = "\u0399"; break;
699	case '\u1f39': replace = "\u0399"; break;
700	case '\u1f3a': replace = "\u0399"; break;
701	case '\u1f3b': replace = "\u0399"; break;
702	case '\u1f3c': replace = "\u0399"; break;
703	case '\u1f3d': replace = "\u0399"; break;
704	case '\u1f3e': replace = "\u0399"; break;
705	case '\u1f3f': replace = "\u0399"; break;
706	case '\u1f40': replace = "\u03bf"; break;
707	case '\u1f41': replace = "\u03bf"; break;
708	case '\u1f42': replace = "\u03bf"; break;
709	case '\u1f43': replace = "\u03bf"; break;
710	case '\u1f44': replace = "\u03bf"; break;
711	case '\u1f45': replace = "\u03bf"; break;
712	case '\u1f48': replace = "\u039f"; break;
713	case '\u1f49': replace = "\u039f"; break;
714	case '\u1f4a': replace = "\u039f"; break;
715	case '\u1f4b': replace = "\u039f"; break;
716	case '\u1f4c': replace = "\u039f"; break;
717	case '\u1f4d': replace = "\u039f"; break;
718	case '\u1f50': replace = "\u03c5"; break;
719	case '\u1f51': replace = "\u03c5"; break;
720	case '\u1f52': replace = "\u03c5"; break;
721	case '\u1f53': replace = "\u03c5"; break;
722	case '\u1f54': replace = "\u03c5"; break;
723	case '\u1f55': replace = "\u03c5"; break;
724	case '\u1f56': replace = "\u03c5"; break;
725	case '\u1f57': replace = "\u03c5"; break;
726	case '\u1f58': replace = "\u03a5"; break;
727	case '\u1f59': replace = "\u03a5"; break;
728	case '\u1f5a': replace = "\u03a5"; break;
729	case '\u1f5b': replace = "\u03a5"; break;
730	case '\u1f5c': replace = "\u03a5"; break;
731	case '\u1f5d': replace = "\u03a5"; break;
732	case '\u1f5e': replace = "\u03a5"; break;
733	case '\u1f5f': replace = "\u03a5"; break;
734	case '\u1f60': replace = "\u03c9"; break;
735	case '\u1f61': replace = "\u03c9"; break;
736	case '\u1f62': replace = "\u03c9"; break;
737	case '\u1f63': replace = "\u03c9"; break;
738	case '\u1f64': replace = "\u03c9"; break;
739	case '\u1f65': replace = "\u03c9"; break;
740	case '\u1f66': replace = "\u03c9"; break;
741	case '\u1f67': replace = "\u03c9"; break;
742	case '\u1f68': replace = "\u03a9"; break;
743	case '\u1f69': replace = "\u03a9"; break;
744	case '\u1f6a': replace = "\u03a9"; break;
745	case '\u1f6b': replace = "\u03a9"; break;
746	case '\u1f6c': replace = "\u03a9"; break;
747	case '\u1f6d': replace = "\u03a9"; break;
748	case '\u1f6e': replace = "\u03a9"; break;
749	case '\u1f6f': replace = "\u03a9"; break;
750	case '\u1f70': replace = "\u03b1"; break;
751	case '\u1f71': replace = "\u03b1"; break;
752	case '\u1f72': replace = "\u03b5"; break;
753	case '\u1f73': replace = "\u03b5"; break;
754	case '\u1f74': replace = "\u03b7"; break;
755	case '\u1f75': replace = "\u03b7"; break;
756	case '\u1f76': replace = "\u03b9"; break;
757	case '\u1f77': replace = "\u03b9"; break;
758	case '\u1f78': replace = "\u03bf"; break;
759	case '\u1f79': replace = "\u03bf"; break;
760	case '\u1f7a': replace = "\u03c5"; break;
761	case '\u1f7b': replace = "\u03c5"; break;
762	case '\u1f7c': replace = "\u03c9"; break;
763	case '\u1f7d': replace = "\u03c9"; break;
764	case '\u1f80': replace = "\u03b1"; break;
765	case '\u1f81': replace = "\u03b1"; break;
766	case '\u1f82': replace = "\u03b1"; break;
767	case '\u1f83': replace = "\u03b1"; break;
768	case '\u1f84': replace = "\u03b1"; break;
769	case '\u1f85': replace = "\u03b1"; break;
770	case '\u1f86': replace = "\u03b1"; break;
771	case '\u1f87': replace = "\u03b1"; break;
772	case '\u1f88': replace = "\u0391"; break;
773	case '\u1f89': replace = "\u0391"; break;
774	case '\u1f8a': replace = "\u0391"; break;
775	case '\u1f8b': replace = "\u0391"; break;
776	case '\u1f8c': replace = "\u0391"; break;
777	case '\u1f8d': replace = "\u0391"; break;
778	case '\u1f8e': replace = "\u0391"; break;
779	case '\u1f8f': replace = "\u0391"; break;
780	case '\u1f90': replace = "\u03b7"; break;
781	case '\u1f91': replace = "\u03b7"; break;
782	case '\u1f92': replace = "\u03b7"; break;
783	case '\u1f93': replace = "\u03b7"; break;
784	case '\u1f94': replace = "\u03b7"; break;
785	case '\u1f95': replace = "\u03b7"; break;
786	case '\u1f96': replace = "\u03b7"; break;
787	case '\u1f97': replace = "\u03b7"; break;
788	case '\u1f98': replace = "\u0397"; break;
789	case '\u1f99': replace = "\u0397"; break;
790	case '\u1f9a': replace = "\u0397"; break;
791	case '\u1f9b': replace = "\u0397"; break;
792	case '\u1f9c': replace = "\u0397"; break;
793	case '\u1f9d': replace = "\u0397"; break;
794	case '\u1f9e': replace = "\u0397"; break;
795	case '\u1f9f': replace = "\u0397"; break;
796	case '\u1fa0': replace = "\u03c9"; break;
797	case '\u1fa1': replace = "\u03c9"; break;
798	case '\u1fa2': replace = "\u03c9"; break;
799	case '\u1fa3': replace = "\u03c9"; break;
800	case '\u1fa4': replace = "\u03c9"; break;
801	case '\u1fa5': replace = "\u03c9"; break;
802	case '\u1fa6': replace = "\u03c9"; break;
803	case '\u1fa7': replace = "\u03c9"; break;
804	case '\u1fa8': replace = "\u03a9"; break;
805	case '\u1fa9': replace = "\u03a9"; break;
806	case '\u1faa': replace = "\u03a9"; break;
807	case '\u1fab': replace = "\u03a9"; break;
808	case '\u1fac': replace = "\u03a9"; break;
809	case '\u1fad': replace = "\u03a9"; break;
810	case '\u1fae': replace = "\u03a9"; break;
811	case '\u1faf': replace = "\u03a9"; break;
812	case '\u1fb2': replace = "\u03b1"; break;
813	case '\u1fb3': replace = "\u03b1"; break;
814	case '\u1fb4': replace = "\u03b1"; break;
815	case '\u1fb6': replace = "\u03b1"; break;
816	case '\u1fb7': replace = "\u03b1"; break;
817	case '\u1fba': replace = "\u0391"; break;
818	case '\u1fbb': replace = "\u0391"; break;
819	case '\u1fbc': replace = "\u0391"; break;
820	case '\u1fc2': replace = "\u03b7"; break;
821	case '\u1fc3': replace = "\u03b7"; break;
822	case '\u1fc4': replace = "\u03b7"; break;
823	case '\u1fc6': replace = "\u03b7"; break;
824	case '\u1fc7': replace = "\u03b7"; break;
825	case '\u1fca': replace = "\u0397"; break;
826	case '\u1fcb': replace = "\u0397"; break;
827	case '\u1fcc': replace = "\u0397"; break;
828	case '\u1fd2': replace = "\u03b9"; break;
829	case '\u1fd3': replace = "\u03b9"; break;
830	case '\u1fd6': replace = "\u03b9"; break;
831	case '\u1fd7': replace = "\u03b9"; break;
832	case '\u1fda': replace = "\u0399"; break;
833	case '\u1fdb': replace = "\u039f"; break;
834	case '\u1fe2': replace = "\u03c5"; break;
835	case '\u1fe3': replace = "\u03c5"; break;
836	case '\u1fe4': replace = "\u03c1"; break;
837	case '\u1fe5': replace = "\u03c1"; break;
838	case '\u1fe6': replace = "\u03c5"; break;
839	case '\u1fe7': replace = "\u03c5"; break;
840	case '\u1fea': replace = "\u03a5"; break;
841	case '\u1feb': replace = "\u03a5"; break;
842	case '\u1fec': replace = "\u03a1"; break;
843	case '\u1ff2': replace = "\u03c9"; break;
844	case '\u1ff3': replace = "\u03c9"; break;
845	case '\u1ff4': replace = "\u03c9"; break;
846	case '\u1ff6': replace = "\u03c9"; break;
847	case '\u1ff7': replace = "\u03c9"; break;
848	case '\u1ff8': replace = "\u039f"; break;
849	case '\u1ff9': replace = "\u039f"; break;
850	case '\u1ffa': replace = "\u03a9"; break;
851	case '\u1ffb': replace = "\u03a9"; break;
852	case '\u1ffc': replace = "\u03a9"; break;
853
854	case '<': break;
855	case '>': break;
856	case '-': break; // same treatment as soft hyphen
857	case '\u00ad': break; // soft hyphen
858	default: replace += c; break;
859	}
860
861	buf.append(replace);
862
863	// update offsets if replacement is a different length
864	if (offsets != null) {
865	int r = replace.length();
866	if (r == 0) this.offsets = arrayKill(this.offsets, i - n);
867	else if (r == 2)
868	this.offsets =
869	arrayInsert(this.offsets, i - n + 1,
870	this.offsets[i - n], r - 1);
871	n += 1 - r;
872	}
873	}
874	return buf.toString();
875	}
876
877	else { // unknown or no ruleset
878	return s;
879	}
880	}
881
882	/**
883	* Applies the normalization rules in <code>ruleset</code> to
884	* <code>s</code>, without offset tracking.
885	*
886	* @param s source string
887	* @return normalized string
888	*/
889	public String normalize(String s) {
890	return normalize(s, null);
891	}
892
893	/**
894	* Returns the offset table.
895	*
896	* @return offset table
897	*/
898	public int[] getOffsetTable() {
899	return offsets;
900	}
901
902	/**
903	* Returns a copy of an integer array with the element at
904	* <code>index</code> removed ("killed").
905	*
906	* @param array integer array
907	* @param index index of element to remove
908	*/
909	static private int[] arrayKill(int[] array, int index) {
910	int[] newArray = new int[array.length - 1];
911	System.arraycopy(array, 0, newArray, 0, index);
912	System.arraycopy(array, index + 1, newArray, index,
913	array.length - index - 1);
914	return newArray;
915	}
916
917	/**
918	* Returns a copy of an integer array with <code>count</code> elements
919	* inserted at <code>index</code>.
920	*
921	* @param array integer array
922	* @param index index to insert new elements
923	* @param value value to insert into new slots
924	* @param count number of new slots to insert
925	*/
926	static private int[] arrayInsert(int[] array, int index, int value,
927	int count) {
928	int[] newArray = new int[array.length + count];
929	System.arraycopy(array, 0, newArray, 0, index);
930	for (int i = 0; i < count; i++) newArray[index + i] = value;
931	System.arraycopy(array, index, newArray, index + count,
932	array.length - index);
933	return newArray;
934	}
935
936	/**
937	* We provide <code>main()</code> so that our services will be available
938	* outside Java (i.e., so we can run as a Un*x-style filter).
939	*/
940	static public void main(String[] argv) {
941	if (argv.length != 1) {
942	System.err.println("You must specify a ruleset.");
943	System.exit(1);
944	}
945
946	String rec;
947	StringBuffer buf = new StringBuffer();
948	BufferedReader bin = null;
949	try {
950	bin = new BufferedReader(new InputStreamReader(System.in,
951	"UTF-8"));
952	while ((rec = bin.readLine()) != null) buf.append(rec + "\n");
953	}
954	catch (UnsupportedEncodingException e) {
955	System.err.println(e);
956	System.exit(1);
957	}
958	catch (IOException e) {
959	System.err.println(e);
960	System.exit(1);
961	}
962
963	OrthographicRules orth = new OrthographicRules(argv[0]);
964	System.out.print(orth.normalize(buf.toString()));
965	}
966	}

Download in other formats:

Original Format