/*
 * Normalization rules for German text
 * [this is a JFlex specification]
 *
 * Wolfgang Schmidle 
 * version 2011-08-10
 *
 */

package de.mpg.mpiwg.berlin.mpdl.lt.analyzer.lang;

%%

%public
%class MpdlNormalizerLexDE
%type java.lang.String
%unicode

// German: de, deu, ger

%state DISP
%states DICT, DICT_ASCII, GRIMM
%states SEARCH, SEARCH_ASCII

%{
	public static final int CELEX = DICT_ASCII;

	private String original = "";
	private String normalized = "";
	private int problem = 0;
	
	private void add (String norm) {
		original += yytext(); 
		normalized += norm;
	}

	private static final String LB = "[\u002d\u00ad] ";
%}

hyphen = [-\u{00ad}]  // hyphen and soft hyphen
LB = {hyphen} \u0020
// lb = ({hyphen} \u0020)?

END = \n

Alphabet = [abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ]

%%

ſ { add("s"); }

// Fraktur

<DISP, DICT, GRIMM, SEARCH> {

uͦ {add("u"); }
aͤ {add("ä"); }
oͤ {add("ö"); }
uͤ {add("ü"); }

}

<DICT_ASCII, SEARCH_ASCII> {

// normalize ä ö ü ß only for Celex!

ä | Ä | aͤ { add("ae"); }
ö | Ö | oͤ { add("oe"); }
ü | Ü | uͤ { add("ue"); }
uͦ {add("u"); }
ß { add("ss"); }

{Alphabet} { add(yytext()); }

. { problem = 1; add(yytext()); }

}

<GRIMM> {

ß { add("sz"); }

}


// default

@ { problem = 1; add(yytext()); }
{LB} { add(yytext()); }
. { add(yytext()); }


<DISP> {

{END} {
		switch (problem) {
			case 1: return original;
			default: return normalized;
		}
	}
}

<DICT, DICT_ASCII, GRIMM> {

{END} {
		switch (problem) {
			case 1: return "";
			default: return normalized.replaceAll(LB, "");
		}
	}
}

<SEARCH, SEARCH_ASCII> {

{END} {
		switch (problem) {
			case 1: return original;
			default: return normalized.replaceAll(LB, "").toLowerCase();
		}
	}
}



/*

Annahmen:
- die Routine wird wortweise aufgerufen, mit einem \n am Ende des Strings
- Zeilenumbrüche innerhalb des Wortes sind als hyphen/soft hyphen plus space markiert

TO DO:

DE: Trennung von Deutsch und Fraktur?
DE: Celex: hyphens weg?

*/