mpdl-group: software/mpdl-services/mpiwg-mpdl-lt/src/de/mpg/mpiwg/berlin/mpdl/lt/text/tokenize/Tokenizer.java annotate

author	Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de>
date	Wed, 09 Nov 2011 15:32:05 +0100
parents
children	e845310098ba

rev	line source
19 4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	1 package de.mpg.mpiwg.berlin.mpdl.lt.text.tokenize;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	2
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	3 import java.io.IOException;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	4 import java.io.Reader;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	5 import java.util.ArrayList;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	6
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	7 import org.apache.lucene.analysis.standard.StandardTokenizer;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	8 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	9 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	10 import org.apache.lucene.util.AttributeSource;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	11 import org.apache.lucene.util.CharacterUtils;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	12 import org.apache.lucene.util.CharacterUtils.CharacterBuffer;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	13 import org.apache.lucene.util.Version;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	14
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	15 import de.mpg.mpiwg.berlin.mpdl.exception.ApplicationException;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	16 import de.mpg.mpiwg.berlin.mpdl.lt.general.Language;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	17 import de.mpg.mpiwg.berlin.mpdl.lt.text.norm.Normalizer;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	18
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	19 public class Tokenizer extends org.apache.lucene.analysis.Tokenizer {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	20 // variables are copied from Lucene 3.4. CharTokenizer
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	21 private int offset = 0, bufferIndex = 0, dataLen = 0, finalOffset = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	22 private static int MAX_WORD_LEN = 4096; // old value was 255
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	23 private static int IO_BUFFER_SIZE = 4096;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	24 private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	25 private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	26 private CharacterUtils charUtils = CharacterUtils.getInstance(Version.LUCENE_34);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	27 private CharacterBuffer ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	28 // application variables
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	29 private String language = "eng"; // default: english
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	30 private String[] normFunctions = {"norm"}; // default: use norm function
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	31 private Normalizer normalizer;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	32
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	33 public Tokenizer(Reader input) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	34 super(input);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	35 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	36
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	37 public Tokenizer(AttributeSource source, Reader input) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	38 super(source, input);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	39 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	40
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	41 public void setLanguage(String lang) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	42 String language = Language.getInstance().getLanguageId(lang);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	43 this.language = language;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	44 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	45
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	46 public void setNormFunctions(String[] normFunctions) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	47 this.normFunctions = normFunctions;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	48 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	49
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	50 public ArrayList<Token> getTokens() throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	51 if (Language.getInstance().isChinese(language)) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	52 return getTokensByChineseTokenizer(input, normFunctions);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	53 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	54 ArrayList<Token> tokens = new ArrayList<Token>();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	55 try {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	56 reset(input);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	57 CharTermAttribute charTermAttribute = getAttribute(CharTermAttribute.class);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	58 OffsetAttribute offsetAttribute = getAttribute(OffsetAttribute.class);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	59 while (incrementToken()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	60 String term = charTermAttribute.toString();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	61 int start = offsetAttribute.startOffset();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	62 int end = offsetAttribute.endOffset();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	63 String normedTerm = normalizer.normalize(term);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	64 Token token = new Token(start, end, normedTerm);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	65 tokens.add(token);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	66 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	67 end(); // TODO needed ?
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	68 close(); // TODO needed ?
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	69 } catch (IOException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	70 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	71 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	72 return tokens;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	73 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	74
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	75 /** Returns true iff a character should be included in a token. */
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	76 protected boolean isTokenChar(int codepoint) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	77 boolean isTokenChar = true;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	78 char c = (char) codepoint;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	79 switch (c) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	80 case ' ': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	81 case '.': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	82 case ',': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	83 case '!': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	84 case '?': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	85 case ';': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	86 case ':': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	87 case '(': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	88 case ')': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	89 case '[': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	90 case ']': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	91 case '{': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	92 case '}': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	93 case '<': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	94 case '>': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	95 case '/': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	96 case '=': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	97 case '&': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	98 case '+': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	99 case '#': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	100 case '"': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	101 case '�': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	102 case '�': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	103 case '�': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	104 case '�': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	105 case '\'': isTokenChar = false; break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	106 case '\t': isTokenChar = false; break; // do not break words which have tabs in it
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	107 case '\n': isTokenChar = false; break; // do not break words which are on another line
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	108 case '\u2425': isTokenChar = false; break; // special char for marking xml elements
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	109 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	110 return isTokenChar;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	111 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	112
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	113 /** Called on each token character to normalize it before it is added to the
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	114 * token. The default implementation does nothing. Subclasses may use this
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	115 * to, e.g., lowercase tokens. */
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	116 protected char normalize(char c) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	117 return c;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	118 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	119 protected int normalize(int c) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	120 return c;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	121 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	122
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	123 /*
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	124 * Code is copied from Lucene 3.4. CharTokenizer.incrementToken()
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	125 * @see org.apache.lucene.analysis.TokenStream#incrementToken()
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	126 */
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	127 public boolean incrementToken() throws IOException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	128 clearAttributes();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	129 int length = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	130 int start = -1; // this variable is always initialized
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	131 char[] buffer = termAtt.buffer();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	132 while (true) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	133 if (bufferIndex >= dataLen) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	134 offset += dataLen;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	135 if(! charUtils.fill(ioBuffer, input)) { // read supplementary char aware with CharacterUtils
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	136 dataLen = 0; // so next offset += dataLen won't decrement offset
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	137 if (length > 0) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	138 break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	139 } else {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	140 finalOffset = correctOffset(offset);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	141 return false;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	142 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	143 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	144 dataLen = ioBuffer.getLength();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	145 bufferIndex = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	146 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	147 // use CharacterUtils here to support < 3.1 UTF-16 code unit behavior if the char based methods are gone
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	148 int c = charUtils.codePointAt(ioBuffer.getBuffer(), bufferIndex);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	149 bufferIndex += Character.charCount(c);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	150 if (isTokenChar(c)) { // if it's a token char
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	151 if (length == 0) { // start of token
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	152 start = offset + bufferIndex - 1;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	153 } else if (length >= buffer.length-1) { // check if a supplementary could run out of bounds
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	154 buffer = termAtt.resizeBuffer(2 + length); // make sure a supplementary fits in the buffer
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	155 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	156 length += Character.toChars(normalize(c), buffer, length); // buffer it, normalized
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	157 if (length >= MAX_WORD_LEN) // buffer overflow! make sure to check for >= surrogate pair could break == test
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	158 break;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	159 } else if (length > 0) // at non-Letter w/ chars
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	160 break; // return 'em
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	161 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	162 termAtt.setLength(length);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	163 offsetAtt.setOffset(correctOffset(start), finalOffset = correctOffset(start + length));
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	164 return true;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	165 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	166
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	167 /*
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	168 * Code is copied from Lucene 3.4. CharTokenizer.end()
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	169 * @see org.apache.lucene.analysis.TokenStream#end()
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	170 */
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	171 @Override
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	172 public final void end() {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	173 // set final offset
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	174 offsetAtt.setOffset(finalOffset, finalOffset);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	175 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	176
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	177 /*
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	178 * Code is copied from Lucene 3.4. CharTokenizer.reset()
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	179 * @see org.apache.lucene.analysis.Tokenizer#reset(java.io.Reader)
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	180 */
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	181 @Override
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	182 public void reset(Reader input) throws IOException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	183 super.reset(input);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	184 bufferIndex = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	185 offset = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	186 dataLen = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	187 finalOffset = 0;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	188 ioBuffer.reset(); // make sure to reset the IO buffer!!
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	189 this.normalizer = new Normalizer(normFunctions, language);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	190 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	191
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	192 private ArrayList<Token> getTokensByChineseTokenizer(Reader input, String[] normFunctions) throws ApplicationException {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	193 StandardTokenizer chineseTokenizer = new StandardTokenizer(Version.LUCENE_34, input); // is recommended instead of ChineseTokenizer which is deprecated
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	194 ArrayList<Token> tokens = new ArrayList<Token>();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	195 try {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	196 reset(input);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	197 chineseTokenizer.reset(input);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	198 CharTermAttribute charTermAttribute = chineseTokenizer.getAttribute(CharTermAttribute.class);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	199 OffsetAttribute offsetAttribute = chineseTokenizer.getAttribute(OffsetAttribute.class);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	200 while (chineseTokenizer.incrementToken()) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	201 String term = charTermAttribute.toString();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	202 String normedTerm = normalizer.normalize(term);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	203 int start = offsetAttribute.startOffset();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	204 int end = offsetAttribute.endOffset();
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	205 Token token = new Token(start, end, normedTerm);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	206 tokens.add(token);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	207 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	208 chineseTokenizer.end(); // TODO needed ?
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	209 chineseTokenizer.close(); // TODO needed ?
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	210 end(); // TODO needed ?
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	211 close(); // TODO needed ?
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	212 } catch (IOException e) {
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	213 throw new ApplicationException(e);
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	214 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	215 return tokens;
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	216 }
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	217
4a3641ae14d2 Erstellung Josef Willenborg <jwillenborg@mpiwg-berlin.mpg.de> parents: diff changeset	218 }

19

4a3641ae14d2 Erstellung