mpdl-group: software/eXist/mpdl-modules/src/de/mpg/mpiwg/berlin/mpdl/donatus/analysis/lang/GermanStemmer.java annotate

author	Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de>
date	Wed, 24 Nov 2010 17:24:23 +0100
parents
children

rev	line source
0 408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	1 package de.mpg.mpiwg.berlin.mpdl.donatus.analysis.lang;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	2 // This file is encoded in UTF-8
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	3
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	4 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	5 * Licensed to the Apache Software Foundation (ASF) under one or more
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	6 * contributor license agreements. See the NOTICE file distributed with
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	7 * this work for additional information regarding copyright ownership.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	8 * The ASF licenses this file to You under the Apache License, Version 2.0
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	9 * (the "License"); you may not use this file except in compliance with
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	10 * the License. You may obtain a copy of the License at
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	11 *
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	12 * http://www.apache.org/licenses/LICENSE-2.0
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	13 *
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	14 * Unless required by applicable law or agreed to in writing, software
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	15 * distributed under the License is distributed on an "AS IS" BASIS,
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	17 * See the License for the specific language governing permissions and
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	18 * limitations under the License.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	19 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	20
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	21 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	22 * A stemmer for German words. The algorithm is based on the report
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	23 * "A Fast and Simple Stemming Algorithm for German Words" by Jörg
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	24 * Caumanns (joerg.caumanns at isst.fhg.de).
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	25 *
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	26 *
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	27 * @version $Id: GermanStemmer.java 564236 2007-08-09 15:21:19Z gsingers $
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	28 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	29 public class GermanStemmer
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	30 {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	31 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	32 * Buffer for the terms while stemming them.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	33 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	34 private StringBuffer sb = new StringBuffer();
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	35
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	36 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	37 * Amount of characters that are removed with <tt>substitute()</tt> while stemming.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	38 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	39 private int substCount = 0;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	40
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	41 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	42 * Stemms the given term to an unique <tt>discriminator</tt>.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	43 *
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	44 * @param term The term that should be stemmed.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	45 * @return Discriminator for <tt>term</tt>
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	46 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	47 public String stem( String term )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	48 {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	49 // Use lowercase for medium stemming.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	50 term = term.toLowerCase();
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	51 if ( !isStemmable( term ) )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	52 return term;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	53 // Reset the StringBuffer.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	54 sb.delete( 0, sb.length() );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	55 sb.insert( 0, term );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	56 // Stemming starts here...
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	57 substitute( sb );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	58 strip( sb );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	59 optimize( sb );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	60 resubstitute( sb );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	61 removeParticleDenotion( sb );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	62 return sb.toString();
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	63 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	64
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	65 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	66 * Checks if a term could be stemmed.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	67 *
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	68 * @return true if, and only if, the given term consists in letters.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	69 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	70 private boolean isStemmable( String term )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	71 {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	72 for ( int c = 0; c < term.length(); c++ ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	73 if ( !Character.isLetter( term.charAt( c ) ) )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	74 return false;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	75 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	76 return true;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	77 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	78
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	79 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	80 * suffix stripping (stemming) on the current term. The stripping is reduced
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	81 * to the seven "base" suffixes "e", "s", "n", "t", "em", "er" and * "nd",
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	82 * from which all regular suffixes are build of. The simplification causes
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	83 * some overstemming, and way more irregular stems, but still provides unique.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	84 * discriminators in the most of those cases.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	85 * The algorithm is context free, except of the length restrictions.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	86 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	87 private void strip( StringBuffer buffer )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	88 {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	89 boolean doMore = true;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	90 while ( doMore && buffer.length() > 3 ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	91 if ( ( buffer.length() + substCount > 5 ) &&
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	92 buffer.substring( buffer.length() - 2, buffer.length() ).equals( "nd" ) )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	93 {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	94 buffer.delete( buffer.length() - 2, buffer.length() );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	95 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	96 else if ( ( buffer.length() + substCount > 4 ) &&
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	97 buffer.substring( buffer.length() - 2, buffer.length() ).equals( "em" ) ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	98 buffer.delete( buffer.length() - 2, buffer.length() );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	99 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	100 else if ( ( buffer.length() + substCount > 4 ) &&
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	101 buffer.substring( buffer.length() - 2, buffer.length() ).equals( "er" ) ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	102 buffer.delete( buffer.length() - 2, buffer.length() );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	103 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	104 else if ( buffer.charAt( buffer.length() - 1 ) == 'e' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	105 buffer.deleteCharAt( buffer.length() - 1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	106 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	107 else if ( buffer.charAt( buffer.length() - 1 ) == 's' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	108 buffer.deleteCharAt( buffer.length() - 1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	109 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	110 else if ( buffer.charAt( buffer.length() - 1 ) == 'n' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	111 buffer.deleteCharAt( buffer.length() - 1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	112 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	113 // "t" occurs only as suffix of verbs.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	114 else if ( buffer.charAt( buffer.length() - 1 ) == 't' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	115 buffer.deleteCharAt( buffer.length() - 1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	116 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	117 else {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	118 doMore = false;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	119 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	120 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	121 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	122
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	123 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	124 * Does some optimizations on the term. This optimisations are
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	125 * contextual.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	126 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	127 private void optimize( StringBuffer buffer )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	128 {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	129 // Additional step for female plurals of professions and inhabitants.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	130 if ( buffer.length() > 5 && buffer.substring( buffer.length() - 5, buffer.length() ).equals( "erin*" ) ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	131 buffer.deleteCharAt( buffer.length() -1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	132 strip( buffer );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	133 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	134 // Additional step for irregular plural nouns like "Matrizen -> Matrix".
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	135 if ( buffer.charAt( buffer.length() - 1 ) == ( 'z' ) ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	136 buffer.setCharAt( buffer.length() - 1, 'x' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	137 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	138 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	139
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	140 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	141 * Removes a particle denotion ("ge") from a term.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	142 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	143 private void removeParticleDenotion( StringBuffer buffer )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	144 {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	145 if ( buffer.length() > 4 ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	146 for ( int c = 0; c < buffer.length() - 3; c++ ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	147 if ( buffer.substring( c, c + 4 ).equals( "gege" ) ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	148 buffer.delete( c, c + 2 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	149 return;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	150 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	151 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	152 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	153 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	154
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	155 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	156 * Do some substitutions for the term to reduce overstemming:
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	157 *
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	158 * - Substitute Umlauts with their corresponding vowel: äöü -> aou,
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	159 * "ß" is substituted by "ss"
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	160 * - Substitute a second char of a pair of equal characters with
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	161 * an asterisk: ?? -> ?*
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	162 * - Substitute some common character combinations with a token:
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	163 * sch/ch/ei/ie/ig/st -> $/§/%/&/#/!
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	164 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	165 private void substitute( StringBuffer buffer )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	166 {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	167 substCount = 0;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	168 for ( int c = 0; c < buffer.length(); c++ ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	169 // Replace the second char of a pair of the equal characters with an asterisk
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	170 if ( c > 0 && buffer.charAt( c ) == buffer.charAt ( c - 1 ) ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	171 buffer.setCharAt( c, '*' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	172 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	173 // Substitute Umlauts.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	174 else if ( buffer.charAt( c ) == 'ä' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	175 buffer.setCharAt( c, 'a' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	176 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	177 else if ( buffer.charAt( c ) == 'ö' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	178 buffer.setCharAt( c, 'o' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	179 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	180 else if ( buffer.charAt( c ) == 'ü' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	181 buffer.setCharAt( c, 'u' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	182 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	183 // Fix bug so that 'ß' at the end of a word is replaced.
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	184 else if ( buffer.charAt( c ) == 'ß' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	185 buffer.setCharAt( c, 's' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	186 buffer.insert( c + 1, 's' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	187 substCount++;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	188 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	189 // Take care that at least one character is left left side from the current one
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	190 if ( c < buffer.length() - 1 ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	191 // Masking several common character combinations with an token
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	192 if ( ( c < buffer.length() - 2 ) && buffer.charAt( c ) == 's' &&
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	193 buffer.charAt( c + 1 ) == 'c' && buffer.charAt( c + 2 ) == 'h' )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	194 {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	195 buffer.setCharAt( c, '$' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	196 buffer.delete( c + 1, c + 3 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	197 substCount =+ 2;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	198 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	199 else if ( buffer.charAt( c ) == 'c' && buffer.charAt( c + 1 ) == 'h' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	200 buffer.setCharAt( c, '§' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	201 buffer.deleteCharAt( c + 1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	202 substCount++;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	203 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	204 else if ( buffer.charAt( c ) == 'e' && buffer.charAt( c + 1 ) == 'i' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	205 buffer.setCharAt( c, '%' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	206 buffer.deleteCharAt( c + 1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	207 substCount++;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	208 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	209 else if ( buffer.charAt( c ) == 'i' && buffer.charAt( c + 1 ) == 'e' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	210 buffer.setCharAt( c, '&' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	211 buffer.deleteCharAt( c + 1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	212 substCount++;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	213 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	214 else if ( buffer.charAt( c ) == 'i' && buffer.charAt( c + 1 ) == 'g' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	215 buffer.setCharAt( c, '#' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	216 buffer.deleteCharAt( c + 1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	217 substCount++;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	218 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	219 else if ( buffer.charAt( c ) == 's' && buffer.charAt( c + 1 ) == 't' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	220 buffer.setCharAt( c, '!' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	221 buffer.deleteCharAt( c + 1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	222 substCount++;
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	223 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	224 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	225 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	226 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	227
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	228 /**
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	229 * Undoes the changes made by substitute(). That are character pairs and
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	230 * character combinations. Umlauts will remain as their corresponding vowel,
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	231 * as "ß" remains as "ss".
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	232 */
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	233 private void resubstitute( StringBuffer buffer )
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	234 {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	235 for ( int c = 0; c < buffer.length(); c++ ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	236 if ( buffer.charAt( c ) == '*' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	237 char x = buffer.charAt( c - 1 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	238 buffer.setCharAt( c, x );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	239 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	240 else if ( buffer.charAt( c ) == '$' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	241 buffer.setCharAt( c, 's' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	242 buffer.insert( c + 1, new char[]{'c', 'h'}, 0, 2 );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	243 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	244 else if ( buffer.charAt( c ) == '§' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	245 buffer.setCharAt( c, 'c' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	246 buffer.insert( c + 1, 'h' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	247 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	248 else if ( buffer.charAt( c ) == '%' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	249 buffer.setCharAt( c, 'e' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	250 buffer.insert( c + 1, 'i' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	251 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	252 else if ( buffer.charAt( c ) == '&' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	253 buffer.setCharAt( c, 'i' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	254 buffer.insert( c + 1, 'e' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	255 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	256 else if ( buffer.charAt( c ) == '#' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	257 buffer.setCharAt( c, 'i' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	258 buffer.insert( c + 1, 'g' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	259 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	260 else if ( buffer.charAt( c ) == '!' ) {
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	261 buffer.setCharAt( c, 's' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	262 buffer.insert( c + 1, 't' );
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	263 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	264 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	265 }
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	266
408254cf2f1d Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	267 }

0

408254cf2f1d Erstellung