Example usage for org.antlr.v4.runtime TokenStream LT

Introduction

In this page you can find the example usage for org.antlr.v4.runtime TokenStream LT.

Prototype

public Token LT(int k);

Source Link

Document

Get the Token instance associated with the value returned by #LA LA(k) .

Usage

From source file:ai.grakn.graql.internal.parser.QueryParser.java

License:Open Source License

/**
 * Consume a single query from the given token stream.
 *
 * @param tokenStream the {@link TokenStream} to consume
 * @return a new {@link TokenSource} containing the tokens comprising the query
 *///from   ww  w .j ava  2s.  co  m
private TokenSource consumeOneQuery(TokenStream tokenStream) {
    List<Token> tokens = new ArrayList<>();

    boolean startedQuery = false;

    while (true) {
        Token token = tokenStream.LT(1);
        boolean isNewQuery = NEW_QUERY_TOKENS.contains(token.getType());
        boolean isEndOfTokenStream = token.getType() == IntStream.EOF;
        boolean isEndOfFirstQuery = startedQuery && isNewQuery;

        // Stop parsing tokens after reaching the end of the first query
        if (isEndOfTokenStream || isEndOfFirstQuery)
            break;

        if (isNewQuery)
            startedQuery = true;

        tokens.add(token);
        tokenStream.consume();
    }

    return new ListTokenSource(tokens);
}

---

From source file:com.twosigma.beaker.groovy.autocomplete.GrammarPredicates.java

License:Apache License

public static boolean isClassName(TokenStream _input) {
    try {/*from  w  w  w.j  a v  a  2s  .  c  o m*/
        int i = 1;
        Token token = _input.LT(i);
        while (token != null && i < _input.size() && _input.LT(i + 1).getType() == GroovyParser.DOT) {
            i = i + 2;
            token = _input.LT(i);
        }
        if (token == null)
            return false;
        // TODO here
        return Character.isUpperCase(Character.codePointAt(token.getText(), 0));
    } catch (Exception e) {
        e.printStackTrace();
    }

    return false;
}

---

From source file:illarion.easynpc.gui.syntax.AbstractAntlrTokenMaker.java

License:Open Source License

@Override
public Token getTokenList(@Nonnull Segment text, int initialTokenType, int startOffset) {
    try (Reader textReader = new CharArrayReader(text.array, text.offset, text.count)) {
        lexer.setInputStream(new ANTLRInputStream(textReader));
        TokenStream tokenStream = new UnbufferedTokenStream(lexer);

        resetTokenList();/*w  w w. jav  a 2  s  . c om*/

        while (true) {
            org.antlr.v4.runtime.Token currentToken = tokenStream.LT(1);

            if (currentToken.getType() == org.antlr.v4.runtime.Token.EOF) {
                break;
            }
            tokenStream.consume();

            // convert the ANTLR token to a RSyntaxTextArea token and add it to the linked list
            int tokenStart = currentToken.getCharPositionInLine() + text.offset;
            int tokenEnd = (tokenStart + currentToken.getText().length()) - 1;
            int tokenOffset = startOffset + currentToken.getCharPositionInLine();

            addToken(text.array, tokenStart, tokenEnd, convertTokenType(currentToken.getType()), tokenOffset);
        } // end while

        // add a null token to indicate end of line; note that the test grammar has no multiline token types
        addNullToken();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return firstToken;
}

---

From source file:kr.ac.korea.dbserver.parser.SQLErrorStrategy.java

License:Apache License

protected void reportNoViableAltException(@NotNull Parser recognizer, @NotNull NoViableAltException e) {
    TokenStream tokens = recognizer.getInputStream();
    String msg;/*from  w  ww.j a v  a 2s  .co  m*/
    Token token = e.getStartToken();
    if (tokens != null) {
        if (tokens.LT(-1) != null && token.getType() == Token.EOF) {
            token = tokens.LT(-1);
        }
        msg = "syntax error at or near " + getTokenErrorDisplay(token);
    } else {
        msg = "no viable alternative at input " + escapeWSAndQuote("<unknown input>");
    }
    recognizer.notifyErrorListeners(token, msg, e);
}

---

From source file:org.ballerinalang.langserver.completions.CompletionCustomErrorStrategy.java

License:Open Source License

private Token getFirstTokenOfCursorLine(Parser recognizer) {
    TokenStream tokenStream = recognizer.getInputStream();
    Token firstCursorLineToken = null;/*  www .j  av  a  2  s. co m*/
    int cursorLine = this.context.get(DocumentServiceKeys.POSITION_KEY).getPosition().getLine() + 1;
    int cursorCol = this.context.get(DocumentServiceKeys.POSITION_KEY).getPosition().getCharacter() + 1;

    int index = 1;
    Token beforeCursorToken = tokenStream.LT(index);
    int type = beforeCursorToken.getType();
    int tLine = beforeCursorToken.getLine();
    int tCol = beforeCursorToken.getCharPositionInLine();

    if (cursorLine < tLine || (cursorLine == tLine && cursorCol <= tCol)) {
        return null;
    }

    firstCursorLineToken = (tLine == cursorLine) ? beforeCursorToken : null;

    while (type != BallerinaParser.EOF && (tLine <= cursorLine)) {
        int tokenIndex = beforeCursorToken.getTokenIndex();
        if (tLine == cursorLine
                && (firstCursorLineToken == null || firstCursorLineToken.getTokenIndex() > tokenIndex)) {
            firstCursorLineToken = beforeCursorToken;
        }
        index++;
        beforeCursorToken = tokenStream.LT(index);
        type = beforeCursorToken.getType();
        tLine = beforeCursorToken.getLine();
    }
    return firstCursorLineToken;
}

---

From source file:org.ballerinalang.langserver.completions.CompletionCustomErrorStrategy.java

License:Open Source License

private Token getLastTerminationToken(TokenStream tokenStream) {
    Token lastTerminationToken = null;/*  w w  w.j ava 2s.  com*/
    Position cursorPosition = this.context.get(DocumentServiceKeys.POSITION_KEY).getPosition();
    int cursorLine = cursorPosition.getLine() + 1;
    int cursorCol = cursorPosition.getCharacter() + 1;

    int index = 1;
    Token beforeCursorToken = tokenStream.LT(index);
    int type = beforeCursorToken.getType();
    int tLine = beforeCursorToken.getLine();
    int tCol = beforeCursorToken.getCharPositionInLine();

    while (type != BallerinaParser.EOF && ((tLine < cursorLine) || (tLine == cursorLine && tCol < cursorCol))) {
        // Checking for terminating tokens with the type number is inconsistent. Thus, uses string comparisons
        String tokenText = beforeCursorToken.getText();
        if (UtilSymbolKeys.SEMI_COLON_SYMBOL_KEY.equals(tokenText)
                || UtilSymbolKeys.OPEN_BRACE_KEY.equals(tokenText)
                || UtilSymbolKeys.CLOSE_BRACE_KEY.equals(tokenText)
                || UtilSymbolKeys.COMMA_SYMBOL_KEY.equals(tokenText)) {
            lastTerminationToken = beforeCursorToken;
        }
        index++;
        beforeCursorToken = tokenStream.LT(index);
        type = beforeCursorToken.getType();
        tLine = beforeCursorToken.getLine();
        tCol = beforeCursorToken.getCharPositionInLine();
    }
    return lastTerminationToken;
}

---