--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/software/mpdl-services/mpiwg-mpdl-lt/bin/de/mpg/mpiwg/berlin/mpdl/lt/text/norm/lang/MpdlNormalizerLexDE.lex	Tue Nov 27 12:35:19 2012 +0100
@@ -0,0 +1,140 @@
+/*
+ * Normalization rules for German text
+ * [this is a JFlex specification]
+ *
+ * Wolfgang Schmidle 
+ * version 2011-07-12
+ *
+ */
+
+package de.mpg.mpiwg.berlin.mpdl.lt.text.norm.lang;
+
+%%
+
+%public
+%class MpdlNormalizerLexDE
+%type java.lang.String
+%unicode
+
+// German: de, deu, ger
+
+%states DISP, DICT, SEARCH
+%state CELEX, GRIMM
+
+%{
+	private String original = "";
+	private String normalized = "";
+	private int problem = 0;
+	
+	private void add (String norm) {
+		original += yytext(); 
+		normalized += norm;
+	}
+
+	private static final String LB = "[\u002d\u00ad] ";
+%}
+
+hyphen = [-\u{00ad}]  // hyphen and soft hyphen
+LB = {hyphen} \u0020
+// lb = ({hyphen} \u0020)?
+
+END = \n
+
+Alphabet = [abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ]
+
+%%
+
+// jump over empty xml elements
+"<"[^><]+"/>" { add(yytext()); }
+"-<"[^><]+"/>" { add(yytext()); }
+"<"[^><]+"></"[^><]+">" { add(yytext()); }
+"-<"[^><]+"></"[^><]+">" { add(yytext()); }
+
+ſ { add("s"); }
+
+// Fraktur
+
+<DISP, DICT, SEARCH,
+GRIMM> {
+
+uͦ {add("u"); }
+aͤ {add("ä"); }
+oͤ {add("ö"); }
+uͤ {add("ü"); }
+
+}
+
+<CELEX> {
+
+// normalize ä ö ü ß only for Celex!
+
+ä | Ä | aͤ { add("ae"); }
+ö | Ö | oͤ { add("oe"); }
+ü | Ü | uͤ { add("ue"); }
+uͦ {add("u"); }
+ß { add("ss"); }
+
+{Alphabet} { add(yytext()); }
+
+. { problem = 1; add(yytext()); }
+
+}
+
+<GRIMM> {
+
+ß { add("sz"); }
+
+}
+
+
+// default
+
+@ { problem = 1; add(yytext()); }
+{LB} { add(yytext()); }
+. { add(yytext()); }
+
+
+<DISP> {
+
+{END} {
+		switch (problem) {
+			case 1: return original;
+			default: return normalized;
+		}
+	}
+}
+
+<DICT, CELEX, GRIMM> {
+
+{END} {
+		switch (problem) {
+			case 1: return "";
+			default: return normalized.replaceAll(LB, "");
+		}
+	}
+}
+
+<SEARCH> {
+
+{END} {
+		switch (problem) {
+			case 1: return original;
+			default: return normalized.replaceAll(LB, "").toLowerCase();
+		}
+	}
+}
+
+
+
+/*
+
+Annahmen:
+- die Routine wird wortweise aufgerufen, mit einem \n am Ende des Strings
+- Zeilenumbrüche innerhalb des Wortes sind als hyphen/soft hyphen plus space markiert
+
+TO DO:
+
+DE: Trennung von Deutsch und Fraktur?
+DE: Celex: hyphens weg?
+
+*/