Example usage for org.apache.lucene.analysis.standard StandardTokenizerImpl YYEOF

Introduction

In this page you can find the example usage for org.apache.lucene.analysis.standard StandardTokenizerImpl YYEOF.

Prototype

int YYEOF

To view the source code for org.apache.lucene.analysis.standard StandardTokenizerImpl YYEOF.

Click Source Link

Document

This character denotes the end of file

Usage

From source file:com.piza.search.AutoPhraseTokenizer.java

License:Apache License

@Override
public final boolean incrementToken() throws IOException {
    clearAttributes();/*from  w w  w  . j av  a2  s  .  c  o m*/
    skippedPositions = 0;

    while (true) {
        int tokenType = scanner.getNextToken();

        if (tokenType == StandardTokenizerImpl.YYEOF) {
            return false;
        }

        if (scanner.yylength() <= maxTokenLength) {
            posIncrAtt.setPositionIncrement(skippedPositions + 1);
            scanner.getText(termAtt);
            final int start = scanner.yychar();
            offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length()));
            typeAtt.setType(StandardTokenizer.TOKEN_TYPES[tokenType]);
            this.phraseProcess.addTerm(termAtt.toString(), offsetAtt.startOffset(), offsetAtt.endOffset());
            return true;
        } else
            // When we skip a too-long term, we still increment the
            // position increment
            skippedPositions++;
    }
}

---

From source file:eu.hlavki.lucene.analysis.identifier.IdentifierTokenizer.java

License:Apache License

@Override
public final boolean incrementToken() throws IOException {
    clearAttributes();//from   w  w  w.  j  ava 2s.c o  m
    skippedPositions = 0;

    while (true) {
        int tokenType = scanner.getNextToken();

        if (tokenType == StandardTokenizerImpl.YYEOF) {
            return false;
        }

        if (scanner.yylength() <= maxTokenLength) {
            posIncrAtt.setPositionIncrement(skippedPositions + 1);
            scanner.getText(termAtt);
            final int start = scanner.yychar();
            offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length()));
            typeAtt.setType(TOKEN_TYPES[tokenType]);
            return true;
        } else
            // When we skip a too-long term, we still increment the
            // position increment
            skippedPositions++;
    }
}

---