Example usage for org.apache.lucene.analysis.cjk CJKBigramFilter DOUBLE_TYPE

Introduction

In this page you can find the example usage for org.apache.lucene.analysis.cjk CJKBigramFilter DOUBLE_TYPE.

Prototype

String DOUBLE_TYPE

To view the source code for org.apache.lucene.analysis.cjk CJKBigramFilter DOUBLE_TYPE.

Click Source Link

Document

when we emit a bigram, it's then marked as this type

Usage

From source file:com.github.buzztaiki.lucene.lastuni.CJKLastUniGramFilter.java

License:Apache License

@Override
public final boolean incrementToken() throws IOException {
    if (state == State.LAST) {
        return false;
    }//from   w w w  .  j a  v  a2s  . co m
    if (state == State.UNIGRAM) {
        state = State.INHERIT;
        clearAttributes();
        termAtt.copyBuffer(lastBuffer, 0, lastEnd - lastStart);
        offsetAtt.setOffset(lastStart, lastEnd);
        typeAtt.setType(lastType);
    } else {
        boolean cont = input.incrementToken();
        char[] buffer = termAtt.buffer().clone();
        int start = offsetAtt.startOffset();
        int end = offsetAtt.endOffset();
        String type = typeAtt.type();

        if ((lastType.equals(CJKBigramFilter.DOUBLE_TYPE) || lastType.equals(OLD_DOUBLE_TYPE))
                && lastEnd - lastStart >= 2 && (!cont && tokenizeLastUni || start >= lastEnd)) {
            state = (cont ? State.UNIGRAM : State.LAST);
            clearAttributes();
            termAtt.copyBuffer(lastBuffer, lastEnd - lastStart - 1, 1);
            offsetAtt.setOffset(lastEnd - 1, lastEnd);
            typeAtt.setType(lastType);
            setLastValues(buffer, start, end, type);
        } else {
            if (!cont) {
                return false;
            }
            state = State.INHERIT;
            setLastValues(buffer, start, end, type);
        }
    }
    return true;
}

---