Example usage for org.antlr.v4.runtime TokenStream consume

Introduction

In this page you can find the example usage for org.antlr.v4.runtime TokenStream consume.

Prototype

void consume();

Source Link

Document

Consumes the current symbol in the stream.

Usage

From source file:ai.grakn.graql.internal.parser.QueryParser.java

License:Open Source License

/**
 * Consume a single query from the given token stream.
 *
 * @param tokenStream the {@link TokenStream} to consume
 * @return a new {@link TokenSource} containing the tokens comprising the query
 *//* w ww  .  ja v  a 2s . c o m*/
private TokenSource consumeOneQuery(TokenStream tokenStream) {
    List<Token> tokens = new ArrayList<>();

    boolean startedQuery = false;

    while (true) {
        Token token = tokenStream.LT(1);
        boolean isNewQuery = NEW_QUERY_TOKENS.contains(token.getType());
        boolean isEndOfTokenStream = token.getType() == IntStream.EOF;
        boolean isEndOfFirstQuery = startedQuery && isNewQuery;

        // Stop parsing tokens after reaching the end of the first query
        if (isEndOfTokenStream || isEndOfFirstQuery)
            break;

        if (isNewQuery)
            startedQuery = true;

        tokens.add(token);
        tokenStream.consume();
    }

    return new ListTokenSource(tokens);
}

---

From source file:illarion.easynpc.gui.syntax.AbstractAntlrTokenMaker.java

License:Open Source License

@Override
public Token getTokenList(@Nonnull Segment text, int initialTokenType, int startOffset) {
    try (Reader textReader = new CharArrayReader(text.array, text.offset, text.count)) {
        lexer.setInputStream(new ANTLRInputStream(textReader));
        TokenStream tokenStream = new UnbufferedTokenStream(lexer);

        resetTokenList();// w  w  w.  ja v a 2  s . c om

        while (true) {
            org.antlr.v4.runtime.Token currentToken = tokenStream.LT(1);

            if (currentToken.getType() == org.antlr.v4.runtime.Token.EOF) {
                break;
            }
            tokenStream.consume();

            // convert the ANTLR token to a RSyntaxTextArea token and add it to the linked list
            int tokenStart = currentToken.getCharPositionInLine() + text.offset;
            int tokenEnd = (tokenStart + currentToken.getText().length()) - 1;
            int tokenOffset = startOffset + currentToken.getCharPositionInLine();

            addToken(text.array, tokenStart, tokenEnd, convertTokenType(currentToken.getType()), tokenOffset);
        } // end while

        // add a null token to indicate end of line; note that the test grammar has no multiline token types
        addNullToken();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return firstToken;
}

---