Mercurial > hg > mpdl-group
diff software/eXist/mpdl-modules/src/de/mpg/mpiwg/berlin/mpdl/lt/analyzer/lang/MpdlNormalizerLexFR.lex @ 9:1ec29fdd0db8
neue .lex Dateien f?r Normalisierung / externe Objekte update
author | Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> |
---|---|
date | Tue, 22 Feb 2011 16:03:45 +0100 |
parents | |
children | 5df60f24e997 |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/software/eXist/mpdl-modules/src/de/mpg/mpiwg/berlin/mpdl/lt/analyzer/lang/MpdlNormalizerLexFR.lex Tue Feb 22 16:03:45 2011 +0100 @@ -0,0 +1,104 @@ +/* + * Normalization rules for French text + * [this is a JFlex specification] + * + * Wolfgang Schmidle + * version 0.96 + * 2011-02-21 + * + */ + +package de.mpg.mpiwg.berlin.mpdl.lt.analyzer.lang; + +%% + +%public +%class MpdlNormalizerLexFR +%type java.lang.String +%unicode + +// French: fr + +%states DISP, DICT, SEARCH +%state CELEX + +%{ + private String original = ""; + private String normalized = ""; + private int problem = 0; + + private void add (String norm) { + original += yytext(); + normalized += norm; + } +%} + +END = \n + +Alphabet = [abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ] + +%% + +<DISP, DICT, SEARCH, CELEX> { + +ſ { add("s"); } +ß { add("ss"); } +æ { add("ae"); } + +} + +<CELEX> { + +[œŒ] { add("oe"); } +[áàâ] { add("a"); } +[éèê] { add("e"); } +[íìî] { add("i"); } +[óòô] { add("o"); } +[úùû] { add("u"); } +’ { add(""); } + +{Alphabet} { add(yytext()); } + +. { problem = 1; add(yytext()); } // in particular "@" + +} + +// default + +@ { problem = 1; add(yytext()); } +. { add(yytext()); } + + +<DISP, SEARCH> { + +{END} { + switch (problem) { + case 1: return original; + default: return normalized; + } + } +} + +<DICT, CELEX> { + +{END} { + switch (problem) { + case 1: return ""; + default: return normalized; + } + } +} + + + +/* + +Annahmen: +- die Routine wird wortweise aufgerufen, mit einem \n am Ende des Strings +- Wörter mit Zeilenumbrüchen wurden bereits wieder zusammengesetzt + +TO DO: + +FR: richtig? vollständig? + +*/