Example usage for org.apache.lucene.search.highlight TokenGroup isDistinct

Introduction

In this page you can find the example usage for org.apache.lucene.search.highlight TokenGroup isDistinct.

Prototype

boolean isDistinct() 

Source Link

Usage

From source file:com._4dconcept.lucene.highlighter.GenericHighlighter.java

License:Apache License

public void highlight(String toHighlight, String field) throws IOException, ParseException {

    TokenStream tokenStream = analyzer.reusableTokenStream(field, new StringReader(toHighlight));
    QueryTermScorer queryTermScorer = new QueryTermScorer(query);

    TokenStream newStream = queryTermScorer.init(tokenStream);
    if (newStream != null) {
        tokenStream = newStream;/*from www .j av a  2 s.  com*/
    }

    //tokenStream.addAttribute(PositionIncrementAttribute.class);
    tokenStream.reset();

    queryTermScorer.startFragment(null);

    int lastEndOffset = 0;

    TokenGroup tokenGroup = new TokenGroup(tokenStream);

    for (boolean next = tokenStream.incrementToken(); next; next = tokenStream.incrementToken()) {

        if ((tokenGroup.numTokens > 0) && tokenGroup.isDistinct()) {
            lastEndOffset = extractText(tokenGroup, toHighlight, lastEndOffset);
        }
        tokenGroup.addToken(queryTermScorer.getTokenScore());
    }

    if (tokenGroup.numTokens > 0) {
        lastEndOffset = extractText(tokenGroup, toHighlight, lastEndOffset);
    }

    //Test what remains of the original text beyond the point where we stopped analyzing
    if ((lastEndOffset < toHighlight.length())) {
        //append it to the last fragment
        callback.terms(toHighlight.substring(lastEndOffset), lastEndOffset, tokenGroup.getTotalScore());
    }
}

---