Mercurial > hg > mpdl-group
diff software/mpdl-services/mpiwg-mpdl-lt/src/de/mpg/mpiwg/berlin/mpdl/lt/text/norm/lang/MpdlNormalizerLexEL.lex @ 19:4a3641ae14d2
Erstellung
author | Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> |
---|---|
date | Wed, 09 Nov 2011 15:32:05 +0100 |
parents | |
children | e845310098ba |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/software/mpdl-services/mpiwg-mpdl-lt/src/de/mpg/mpiwg/berlin/mpdl/lt/text/norm/lang/MpdlNormalizerLexEL.lex Wed Nov 09 15:32:05 2011 +0100 @@ -0,0 +1,139 @@ +/* + * Normalization rules for Greek text + * [this is a JFlex specification] + * + * Wolfgang Schmidle + * version 2011-08-03 + * + */ + +package de.mpg.mpiwg.berlin.mpdl.lt.analyzer.lang; + +%% + +%public +%class MpdlNormalizerLexEL +%type java.lang.String +%unicode + +// Greek: el, grc + +%states DISP, DICT, SEARCH +%state SIGMA + +%{ + private String original = ""; + private String normalized = ""; + private int problem = 0; + + private void add (String norm) { + original += yytext(); + normalized += norm; + } + + private static final String LB = "[\u002d\u00ad] "; +%} + +hyphen = [-\u{00ad}] // hyphen and soft hyphen +LB = {hyphen} \u0020 +// lb = ({hyphen} \u0020)? + +END = \n + +wordend = [νρς]? {END} + +Latin = [abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ] + + +%% + + +// always replace tonos by oxia +// (although this should really be corrected in the text rather than normalized) +ά { add("ά"); } +έ { add("έ"); } +ή { add("ή"); } +ί { add("ί"); } +ό { add("ό"); } +ύ { add("ύ"); } +ώ { add("ώ"); } + + +<DICT, SEARCH, SIGMA> { + +ὰ / {wordend} { add("ά"); } +ᾲ / {wordend} { add("ᾴ"); } +ὲ / {wordend} { add("έ"); } +ὴ / {wordend} { add("ή"); } +ῂ / {wordend} { add("ῄ"); } +ὶ / {wordend} { add("ί"); } +ὸ / {wordend} { add("ό"); } +ὺ / {wordend} { add("ύ"); } +ὼ / {wordend} { add("ώ"); } +ῲ / {wordend} { add("ῴ"); } + +// other candidates: Ὰ Ὲ Ὴ Ὶ Ὺ Ὸ Ὼ + +} + +<SIGMA> { + +ς { add("σ"); } + +} + +// default + +@ { problem = 1; add(yytext()); } +{Latin} { problem = 1; add(yytext()); } + +{LB} { add(yytext()); } +. { add(yytext()); } + + +<DISP> { + +{END} { + switch (problem) { + case 1: return original; + default: return normalized; + } + } +} + +<DICT, SIGMA> { + +{END} { + switch (problem) { + case 1: return ""; + default: return normalized.replaceAll(LB, ""); + } + } +} + +<SEARCH> { + +{END} { + switch (problem) { + case 1: return original; + default: return normalized.replaceAll(LB, "").toLowerCase(); + } + } +} + + +/* + +Annahmen: +- die Routine wird wortweise aufgerufen, mit einem \n am Ende des Strings +- Zeilenumbrüche innerhalb des Wortes sind als hyphen/soft hyphen plus space markiert + +TO DO: + +EL: tonos --> oxia wieder rausnehmen, weil es im Text geändert werden muss? +EL: gibt es noch weitere Fälle, wo legitimerweise ein Gravis vorkommen kann? +EL: kommen Großbuchstaben mit Gravis bei uns jemals vor, und sollen sie normalisiert werden? +EL: neuer State BETACODE ? +EL: nicht falsche Zeichen definieren, sondern erlaubte Zeichen + +*/