Example usage for org.apache.lucene.analysis Token setEmpty

Introduction

In this page you can find the example usage for org.apache.lucene.analysis Token setEmpty.

Prototype

@Override
    public final CharTermAttribute setEmpty() 

Source Link

Usage

From source file:com._4dconcept.lucene.highlighter.TokenGroup.java

License:Apache License

public void addToken(float score) {
    if (numTokens < MAX_NUM_TOKENS_PER_GROUP) {
        int termStartOffset = offsetAtt.startOffset();
        int termEndOffset = offsetAtt.endOffset();
        if (numTokens == 0) {
            startOffset = matchStartOffset = termStartOffset;
            endOffset = matchEndOffset = termEndOffset;
            tot += score;/*from w w w .j  a va  2 s  .c o  m*/
        } else {
            startOffset = Math.min(startOffset, termStartOffset);
            endOffset = Math.max(endOffset, termEndOffset);
            if (score > 0) {
                if (tot == 0) {
                    matchStartOffset = offsetAtt.startOffset();
                    matchEndOffset = offsetAtt.endOffset();
                } else {
                    matchStartOffset = Math.min(matchStartOffset, termStartOffset);
                    matchEndOffset = Math.max(matchEndOffset, termEndOffset);
                }
                tot += score;
            }
        }
        Token token = new Token(termStartOffset, termEndOffset);
        token.setEmpty().append(termAtt);
        tokens[numTokens] = token;
        scores[numTokens] = score;
        numTokens++;
    }
}

---

From source file:edu.mit.ll.vizlinc.highlight.TokenGroup.java

License:Apache License

void addToken(float score) {
    if (numTokens < MAX_NUM_TOKENS_PER_GROUP) {
        int termStartOffset = offsetAtt.startOffset();
        int termEndOffset = offsetAtt.endOffset();
        if (numTokens == 0) {
            startOffset = matchStartOffset = termStartOffset;
            endOffset = matchEndOffset = termEndOffset;
            tot += score;//from w ww.  j av  a  2s . c  om
        } else {
            startOffset = Math.min(startOffset, termStartOffset);
            endOffset = Math.max(endOffset, termEndOffset);
            if (score > 0) {
                if (tot == 0) {
                    matchStartOffset = offsetAtt.startOffset();
                    matchEndOffset = offsetAtt.endOffset();
                } else {
                    matchStartOffset = Math.min(matchStartOffset, termStartOffset);
                    matchEndOffset = Math.max(matchEndOffset, termEndOffset);
                }
                tot += score;
            }
        }
        Token token = new Token(termStartOffset, termEndOffset);
        token.setEmpty().append(termAtt);
        tokens[numTokens] = token;
        scores[numTokens] = score;
        numTokens++;
    }
}

---

From source file:edu.mit.ll.vizlinc.highlight.TokenStreamFromTermPositionVector.java

License:Apache License

/**
 * Constructor./*from w  ww. j a v a  2 s.  c om*/
 * 
 * @param termPositionVector TermPositionVector that contains the data for
 *        creating the TokenStream. Must have positions and offsets.
 */
public TokenStreamFromTermPositionVector(final TermPositionVector termPositionVector) {
    termAttribute = addAttribute(CharTermAttribute.class);
    positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
    offsetAttribute = addAttribute(OffsetAttribute.class);
    final String[] terms = termPositionVector.getTerms();
    for (int i = 0; i < terms.length; i++) {
        final TermVectorOffsetInfo[] offsets = termPositionVector.getOffsets(i);
        final int[] termPositions = termPositionVector.getTermPositions(i);
        for (int j = 0; j < termPositions.length; j++) {
            Token token;
            if (offsets != null) {
                token = new Token(terms[i].toCharArray(), 0, terms[i].length(), offsets[j].getStartOffset(),
                        offsets[j].getEndOffset());
            } else {
                token = new Token();
                token.setEmpty().append(terms[i]);
            }
            // Yes - this is the position, not the increment! This is for
            // sorting. This value
            // will be corrected before use.
            token.setPositionIncrement(termPositions[j]);
            this.positionedTokens.add(token);
        }
    }
    CollectionUtil.mergeSort(this.positionedTokens, tokenComparator);
    int lastPosition = -1;
    for (final Token token : this.positionedTokens) {
        int thisPosition = token.getPositionIncrement();
        token.setPositionIncrement(thisPosition - lastPosition);
        lastPosition = thisPosition;
    }
    this.tokensAtCurrentPosition = this.positionedTokens.iterator();
}

---

From source file:it.cnr.ilc.lc.clavius.search.ClaviusTokenGroup.java

void addToken(float score) {
    if (numTokens < MAX_NUM_TOKENS_PER_GROUP) {
        final int termStartOffset = offsetAtt.startOffset();
        final int termEndOffset = offsetAtt.endOffset();
        if (numTokens == 0) {
            startOffset = matchStartOffset = termStartOffset;
            endOffset = matchEndOffset = termEndOffset;
            tot += score;/*from   w  ww. j a va 2 s  .  c o  m*/
        } else {
            startOffset = Math.min(startOffset, termStartOffset);
            endOffset = Math.max(endOffset, termEndOffset);
            if (score > 0) {
                if (tot == 0) {
                    matchStartOffset = termStartOffset;
                    matchEndOffset = termEndOffset;
                } else {
                    matchStartOffset = Math.min(matchStartOffset, termStartOffset);
                    matchEndOffset = Math.max(matchEndOffset, termEndOffset);
                }
                tot += score;
            }
        }
        Token token = new Token();
        token.setOffset(termStartOffset, termEndOffset);
        token.setEmpty().append(termAtt);
        tokens[numTokens] = token;
        scores[numTokens] = score;
        numTokens++;
    }
}

---