Mercurial > hg > mpdl-group
diff software/mpdl-services/mpiwg-mpdl-lt/src/de/mpg/mpiwg/berlin/mpdl/lt/text/norm/lang/MpdlNormalizerLexIT.lex @ 19:4a3641ae14d2
Erstellung
author | Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> |
---|---|
date | Wed, 09 Nov 2011 15:32:05 +0100 |
parents | |
children | e845310098ba |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/software/mpdl-services/mpiwg-mpdl-lt/src/de/mpg/mpiwg/berlin/mpdl/lt/text/norm/lang/MpdlNormalizerLexIT.lex Wed Nov 09 15:32:05 2011 +0100 @@ -0,0 +1,183 @@ +/* + * Normalization rules for Italian text + * [this is a JFlex specification] + * + * Wolfgang Schmidle + * version 2011-07-12 + * + */ + +package de.mpg.mpiwg.berlin.mpdl.lt.analyzer.lang; + +%% + +%public +%class MpdlNormalizerLexIT +%type java.lang.String +%unicode + +// Italian: it, ita + +%states DISP, DICT, SEARCH + +%{ + private static final int CONS = 1; + private static final int VOWEL = 2; + private int cv = 0; // consonant = 1, vowel = 2, everything else = 0 + + private String original = ""; + private String normalized = ""; + private int problem = 0; + + private void add (String norm) { + original += yytext(); + normalized += norm; + } + + private static final String LB = "[\u002d\u00ad] "; +%} + +Vowel = [AEIOUaeiouÆæęàèòùœ] +Cons = [BCDFGHKLMNPQRSTVWXZbcdfghklmnpqrstvwxzſß] +LR = [lLrR] + + +hyphen = [\u002d\u00ad] // hyphen and soft hyphen +LB = {hyphen} \u0020 +lb = ({hyphen} \u0020)? + +END = \n + +prefixCons = (in{lb}ter | per | ſu{lb}per | ſer) + +%% + +<DICT, SEARCH> { + +À { add("Á"); } +È { add("É"); } +Ì { add("Í"); } +Ò { add("Ó"); } +Ù { add("Ú"); } +à { add("á"); } +è { add("é"); } +ì { add("í"); } +ò { add("ó"); } +ù { add("ú"); } + +} + +<DISP, DICT, SEARCH> { + +ſ { cv = CONS; add("s"); } +ß { cv = CONS; add("ss"); } +æ { cv = VOWEL; add("ae"); } +Æ { cv = VOWEL; add("AE"); } +œ { cv = VOWEL; add("oe"); } +Œ { cv = VOWEL; add("OE"); } + +ij { cv = VOWEL; add("ii"); } + +tio { cv = VOWEL; add("zio"); } +TIO { cv = VOWEL; add("ZIO"); } + +// h-Regeln aus Arboreal: +^ ha / {END} { add(yytext()); } +^ hai / {END} { add(yytext()); } +^ han{lb}no / {END} { add(yytext()); } +^ ho / {END} { add(yytext()); } +^ h { add(""); } + + +// u/v rules are taken from MpdlNormalizerLexLA.lex + +// 1. rules for u --> v + +^ {prefixCons} / {lb} { cv = VOWEL; add(yytext().replace("ſ", "s")); } + +^ [uU] / {Vowel} { cv = VOWEL; add(yytext().replaceAll("u", "v").replaceAll("U", "V")); } + + +[uU] / {Vowel} { + switch(cv) { + case VOWEL: add(yytext().replace("u", "v").replace("U", "V")); break; + default: cv = VOWEL; add(yytext()); break; + } + } + +// 2. rules for v --> u + +qv { cv = CONS; add("qu"); } // the replaced v still counts as consonant +Qv { cv = CONS; add("Qu"); } +QV { cv = CONS; add("QU"); } + +{LR} [vV] { + switch(cv) { + case CONS: add(yytext().replace("v", "u").replace("V", "U")); break; + default: cv = CONS; add(yytext()); break; + } + } + +v / {lb} {Cons} { cv = CONS; add("u"); } +V / {lb} {Cons} { cv = CONS; add("U"); } + +// 3. override default rule for . + +{Vowel} { cv = VOWEL; add(yytext()); } +{Cons} { cv = CONS; add(yytext()); } +@ { problem = 1; cv = 0; add(yytext()); } +{LB} { add(yytext()); } +. { cv = 0; add(yytext()); } + +} + + +<DISP> { + +{END} { + switch (problem) { + case 1: return original; + default: return normalized; + } + } +} + +<DICT> { + +{END} { + switch (problem) { + case 1: return ""; + default: return normalized.replaceAll(LB, ""); + } + } +} + +<SEARCH> { + +{END} { + switch (problem) { + case 1: return original; + default: return normalized.replaceAll(LB, "").toLowerCase(); + } + } +} + + +/* + +Annahmen: +- die Routine wird wortweise aufgerufen, mit einem \n am Ende des Strings +- Zeilenumbrüche innerhalb des Wortes sind als hyphen/soft hyphen plus space markiert + +TO DO: + +IT: all these rules are taken from Arboreal; do we need them all? +IT: richtig? vollständig? +IT: Sind die u/v-Regeln wirklich genau wie in LA ? insbesondere: gleiche Vokal-Klasse? +IT: Änderungen in den lateinischen u/v-Regeln übernehmen? +IT: italienische Beispielwörter für die u/v-Regeln angeben +IT: Brauchen wir die Gravis-Regeln aus Arboreal in DICT wirklich? +IT: wenn ja: gehört À --> Á etc. in die Wörterbuch-Schicht? Und einschränken auf letzte Silbe? +IT: ist prefixCons = (inter | per | ſuper | ſer) auch für Italienisch gültig? + +*/