it.unibz.instasearch.indexing.tokenizers.TermSplitTokenizer.java Source code

Introduction

Here is the source code for it.unibz.instasearch.indexing.tokenizers.TermSplitTokenizer.java

Source

/*
 * Copyright (c) 2009 Andrejs Jermakovics.
 * 
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     Andrejs Jermakovics - initial implementation
 */
package it.unibz.instasearch.indexing.tokenizers;

import java.io.IOException;
import java.util.LinkedList;

import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;

/**
 * Splits terms. Returns the original term and its split parts 
 */
public abstract class TermSplitTokenizer extends TokenFilter {

    private LinkedList<Token> tokens = new LinkedList<Token>();

    private TermAttribute termAtt;
    private OffsetAttribute offsetAtt;
    private PositionIncrementAttribute posAtt;

    public TermSplitTokenizer(TokenStream in) {
        super(in);

        assert (in.hasAttribute(TermAttribute.class));
        assert (in.hasAttribute(OffsetAttribute.class));
        assert (in.hasAttribute(PositionIncrementAttribute.class));

        termAtt = (TermAttribute) addAttribute(TermAttribute.class);
        offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
        posAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
    }

    @Override
    public boolean incrementToken() throws IOException {
        if (!tokens.isEmpty()) {
            applyToken(tokens.removeFirst());
        } else if (input.incrementToken()) {
            splitIntoTokens();

            if (!tokens.isEmpty()) {
                if (!returnOriginalTerm())
                    applyToken(tokens.removeFirst());
            }
        } else {
            return false; // does not have any more tokens
        }

        return true;
    }

    private void splitIntoTokens() {
        String term = termAtt.term();
        String[] termParts = splitTerm(term);

        if (termParts.length > 1) {
            int termPos = offsetAtt.startOffset();

            for (int i = 0; i < termParts.length; i++) {
                String termPart = termParts[i];
                int termPartPos = termPos + term.indexOf(termPart);
                int termPartEndPos = termPartPos + termPart.length();

                Token newToken = new Token(termPart, termPartPos, termPartEndPos);
                newToken.setPositionIncrement(0); // in the same position

                tokens.add(newToken);
            }
        }
    }

    private void applyToken(Token token) {
        termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
        posAtt.setPositionIncrement(token.getPositionIncrement());
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
    }

    /**
     * Return original term together with the parts
     * @return returnOriginalTerm
     */
    protected boolean returnOriginalTerm() {
        return false;
    }

    /**
     * Split term into an array of terms
     * 
     * @param term
     * @return split term
     */
    public abstract String[] splitTerm(String term);
}