Mercurial > hg > mpdl-group
view software/mpdl-services/mpiwg-mpdl-lt/src/de/mpg/mpiwg/berlin/mpdl/lt/text/norm/lang/MpdlNormalizerLexIT.lex @ 19:4a3641ae14d2
Erstellung
author | Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> |
---|---|
date | Wed, 09 Nov 2011 15:32:05 +0100 |
parents | |
children | e845310098ba |
line wrap: on
line source
/* * Normalization rules for Italian text * [this is a JFlex specification] * * Wolfgang Schmidle * version 2011-07-12 * */ package de.mpg.mpiwg.berlin.mpdl.lt.analyzer.lang; %% %public %class MpdlNormalizerLexIT %type java.lang.String %unicode // Italian: it, ita %states DISP, DICT, SEARCH %{ private static final int CONS = 1; private static final int VOWEL = 2; private int cv = 0; // consonant = 1, vowel = 2, everything else = 0 private String original = ""; private String normalized = ""; private int problem = 0; private void add (String norm) { original += yytext(); normalized += norm; } private static final String LB = "[\u002d\u00ad] "; %} Vowel = [AEIOUaeiouÆæęàèòùœ] Cons = [BCDFGHKLMNPQRSTVWXZbcdfghklmnpqrstvwxzſß] LR = [lLrR] hyphen = [\u002d\u00ad] // hyphen and soft hyphen LB = {hyphen} \u0020 lb = ({hyphen} \u0020)? END = \n prefixCons = (in{lb}ter | per | ſu{lb}per | ſer) %% <DICT, SEARCH> { À { add("Á"); } È { add("É"); } Ì { add("Í"); } Ò { add("Ó"); } Ù { add("Ú"); } à { add("á"); } è { add("é"); } ì { add("í"); } ò { add("ó"); } ù { add("ú"); } } <DISP, DICT, SEARCH> { ſ { cv = CONS; add("s"); } ß { cv = CONS; add("ss"); } æ { cv = VOWEL; add("ae"); } Æ { cv = VOWEL; add("AE"); } œ { cv = VOWEL; add("oe"); } Œ { cv = VOWEL; add("OE"); } ij { cv = VOWEL; add("ii"); } tio { cv = VOWEL; add("zio"); } TIO { cv = VOWEL; add("ZIO"); } // h-Regeln aus Arboreal: ^ ha / {END} { add(yytext()); } ^ hai / {END} { add(yytext()); } ^ han{lb}no / {END} { add(yytext()); } ^ ho / {END} { add(yytext()); } ^ h { add(""); } // u/v rules are taken from MpdlNormalizerLexLA.lex // 1. rules for u --> v ^ {prefixCons} / {lb} { cv = VOWEL; add(yytext().replace("ſ", "s")); } ^ [uU] / {Vowel} { cv = VOWEL; add(yytext().replaceAll("u", "v").replaceAll("U", "V")); } [uU] / {Vowel} { switch(cv) { case VOWEL: add(yytext().replace("u", "v").replace("U", "V")); break; default: cv = VOWEL; add(yytext()); break; } } // 2. rules for v --> u qv { cv = CONS; add("qu"); } // the replaced v still counts as consonant Qv { cv = CONS; add("Qu"); } QV { cv = CONS; add("QU"); } {LR} [vV] { switch(cv) { case CONS: add(yytext().replace("v", "u").replace("V", "U")); break; default: cv = CONS; add(yytext()); break; } } v / {lb} {Cons} { cv = CONS; add("u"); } V / {lb} {Cons} { cv = CONS; add("U"); } // 3. override default rule for . {Vowel} { cv = VOWEL; add(yytext()); } {Cons} { cv = CONS; add(yytext()); } @ { problem = 1; cv = 0; add(yytext()); } {LB} { add(yytext()); } . { cv = 0; add(yytext()); } } <DISP> { {END} { switch (problem) { case 1: return original; default: return normalized; } } } <DICT> { {END} { switch (problem) { case 1: return ""; default: return normalized.replaceAll(LB, ""); } } } <SEARCH> { {END} { switch (problem) { case 1: return original; default: return normalized.replaceAll(LB, "").toLowerCase(); } } } /* Annahmen: - die Routine wird wortweise aufgerufen, mit einem \n am Ende des Strings - Zeilenumbrüche innerhalb des Wortes sind als hyphen/soft hyphen plus space markiert TO DO: IT: all these rules are taken from Arboreal; do we need them all? IT: richtig? vollständig? IT: Sind die u/v-Regeln wirklich genau wie in LA ? insbesondere: gleiche Vokal-Klasse? IT: Änderungen in den lateinischen u/v-Regeln übernehmen? IT: italienische Beispielwörter für die u/v-Regeln angeben IT: Brauchen wir die Gravis-Regeln aus Arboreal in DICT wirklich? IT: wenn ja: gehört À --> Á etc. in die Wörterbuch-Schicht? Und einschränken auf letzte Silbe? IT: ist prefixCons = (inter | per | ſuper | ſer) auch für Italienisch gültig? */