Example usage for org.apache.lucene.analysis Token getPositionLength

Introduction

In this page you can find the example usage for org.apache.lucene.analysis Token getPositionLength.

Prototype

@Override
public int getPositionLength() 

Source Link

Usage

From source file:com.github.cstoku.neologd.unidic.lucene.analysis.ja.JapaneseTokenizer.java

License:Apache License

@Override
public boolean incrementToken() throws IOException {

    // parse() is able to return w/o producing any new
    // tokens, when the tokens it had produced were entirely
    // punctuation.  So we loop here until we get a real
    // token or we end:
    while (pending.size() == 0) {
        if (end) {
            return false;
        }//from   w  ww .  j  a va  2  s  . c o  m

        // Push Viterbi forward some more:
        parse();
    }

    final Token token = pending.remove(pending.size() - 1);

    int position = token.getPosition();
    int length = token.getLength();
    clearAttributes();
    assert length > 0;
    //System.out.println("off=" + token.getOffset() + " len=" + length + " vs " + token.getSurfaceForm().length);
    termAtt.copyBuffer(token.getSurfaceForm(), token.getOffset(), length);
    offsetAtt.setOffset(correctOffset(position), correctOffset(position + length));
    basicFormAtt.setToken(token);
    posAtt.setToken(token);
    readingAtt.setToken(token);
    inflectionAtt.setToken(token);
    if (token.getPosition() == lastTokenPos) {
        posIncAtt.setPositionIncrement(0);
        posLengthAtt.setPositionLength(token.getPositionLength());
    } else {
        assert token.getPosition() > lastTokenPos;
        posIncAtt.setPositionIncrement(1);
        posLengthAtt.setPositionLength(1);
    }
    if (VERBOSE) {
        System.out.println(Thread.currentThread().getName() + ":    incToken: return token=" + token);
    }
    lastTokenPos = token.getPosition();
    return true;
}

---