--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/software/eXist/mpdl-modules/src/de/mpg/mpiwg/berlin/mpdl/lt/analyzer/lang/MpdlNormalizerLexZH.lex	Tue Feb 22 16:03:45 2011 +0100
@@ -0,0 +1,119 @@
+/*
+ * Normalization rules for Chinese text
+ * [this is a JFlex specification]
+ *
+ * Wolfgang Schmidle 
+ * version 0.96
+ * 2011-02-21
+ *
+ */
+
+package de.mpg.mpiwg.berlin.mpdl.lt.analyzer.lang;
+
+%%
+
+%public
+%class MpdlNormalizerLexZH
+%type java.lang.String
+%unicode
+
+// classical Chinese: zh, zho, zho-Hant
+
+%states DISP, DICT, SEARCH
+
+%{
+	private String original = "";
+	private String normalized = "";
+	private int problem = 0;
+	
+	private void add (String norm) {
+		original += yytext(); 
+		normalized += norm;
+	}
+%}
+
+ZWS = [\u{200b}]
+
+END = \n
+
+%%
+
+// Normalization in Chinese means that character variants will be replaced by their standard characters
+// if there is no doubt about what the standard character is.
+
+// The input is supposed to be a single Chinese character, but strings of characters are also handled correctly.
+
+<DISP, DICT, SEARCH> {
+
+// Codepoint < FFFF
+
+倂 { add("併"); }  // 5002 --> 4F75
+傁 | 叜 { add("叟"); }  // 5081, 53DC --> 53DF
+竒 { add("奇"); }  // 7AD2 --> 5947
+幷 { add("并"); }  // 5E77 --> 5E76
+牀 { add("床"); }  // 7240 --> 5E8A
+旹 { add("時"); }  // 65F9 --> 6642
+歴 { add("歷"); }  // 6B74 --> 6B77
+爲 { add("為"); }  // 7232 --> 70BA
+隂 { add("陰"); }  // 9682 --> 9670
+靣 { add("面"); }  // 9763 --> 9762
+精 { add("精"); }  // FA1D --> 7CBE (FA1D is a compatibility ideograph)
+
+// Codepoint > FFFF
+
+// note that [ABC] is not equivalent to A | B | C  for codepoints above FFFF due to their internal encoding:
+// for example, 庶 (U+2F88D) is represented as a sequence of two codepoints: D87E DC8D
+// i.e. never use [ABC] but A | B | C
+
+庶 { add("庶"); }  // 2F88D --> 5EB6  (2F88D is a compatibility ideograph) 
+
+}
+
+<DICT, SEARCH> {
+
+// remove Zero Width Space (if there is any in the the input string)
+
+{ZWS} { add(""); }
+
+}
+
+// default
+
+@ { problem = 1; add(yytext()); }
+. { add(yytext()); }
+
+
+<DISP, SEARCH> {
+
+{END} {
+		switch (problem) {
+			case 1: return original;
+			default: return normalized;
+		}
+	}
+}
+
+<DICT> {
+
+{END} {
+		switch (problem) {
+			case 1: return "";
+			default: return normalized;
+		}
+	}
+}
+
+
+/*
+
+Annahmen:
+- die Routine wird wortweise aufgerufen, mit einem \n am Ende des Strings
+- Wörter mit Zeilenumbrüchen wurden bereits wieder zusammengesetzt
+
+TO DO:
+
+ZH: Liste ergänzen
+ZH: was ist, wenn man wirklich die Variante, die im Text steht, nachschlagen will? Dann muss man das Zeichen wohl selbst rauskopieren.
+ZH: sollen lateinische Buchstaben bewirken, dass problem = 1 ist?
+
+*/