Example usage for org.antlr.v4.runtime Token EOF

Introduction

In this page you can find the example usage for org.antlr.v4.runtime Token EOF.

Prototype

int EOF

To view the source code for org.antlr.v4.runtime Token EOF.

Click Source Link

Usage

From source file:io.prestosql.sql.parser.StatementSplitter.java

License:Apache License

public static String squeezeStatement(String sql) {
    TokenSource tokens = getLexer(sql, ImmutableSet.of());
    StringBuilder sb = new StringBuilder();
    while (true) {
        Token token = tokens.nextToken();
        if (token.getType() == Token.EOF) {
            break;
        }//from w  ww . j a v  a  2s .  co m
        if (token.getType() == SqlBaseLexer.WS) {
            sb.append(' ');
        } else {
            sb.append(token.getText());
        }
    }
    return sb.toString().trim();
}

---

From source file:io.prestosql.sql.parser.StatementSplitter.java

License:Apache License

public static boolean isEmptyStatement(String sql) {
    TokenSource tokens = getLexer(sql, ImmutableSet.of());
    while (true) {
        Token token = tokens.nextToken();
        if (token.getType() == Token.EOF) {
            return true;
        }//from   w w  w. java  2  s  .  co m
        if (token.getChannel() != Token.HIDDEN_CHANNEL) {
            return false;
        }
    }
}

---

From source file:io.proleap.cobol.preprocessor.impl.CobolPreprocessorImpl.java

License:Open Source License

protected boolean isEOF(final TerminalNode node) {
    return Token.EOF == node.getSymbol().getType();
}

---

From source file:io.proleap.cobol.preprocessor.sub.util.TokenUtils.java

License:Open Source License

public static boolean isEOF(final TerminalNode node) {
    return Token.EOF == node.getSymbol().getType();
}

---

From source file:kr.ac.korea.dbserver.parser.SQLErrorStrategy.java

License:Apache License

protected void reportNoViableAltException(@NotNull Parser recognizer, @NotNull NoViableAltException e) {
    TokenStream tokens = recognizer.getInputStream();
    String msg;/*from ww w .j  a v a  2s.c o m*/
    Token token = e.getStartToken();
    if (tokens != null) {
        if (tokens.LT(-1) != null && token.getType() == Token.EOF) {
            token = tokens.LT(-1);
        }
        msg = "syntax error at or near " + getTokenErrorDisplay(token);
    } else {
        msg = "no viable alternative at input " + escapeWSAndQuote("<unknown input>");
    }
    recognizer.notifyErrorListeners(token, msg, e);
}

---

From source file:net.certiv.json.test.base.AbstractBase.java

License:Open Source License

public List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
    LexerATNSimulator interp = new LexerATNSimulator(atn,
            new DFA[] { new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null);
    List<String> tokenTypes = new ArrayList<String>();
    int ttype;/*from   w w w . j av  a 2  s .  co m*/
    boolean hitEOF = false;
    do {
        if (hitEOF) {
            tokenTypes.add("EOF");
            break;
        }
        int t = input.LA(1);
        ttype = interp.match(input, Lexer.DEFAULT_MODE);
        if (ttype == Token.EOF) {
            tokenTypes.add("EOF");
        } else {
            tokenTypes.add(lg.typeToTokenList.get(ttype));
        }

        if (t == IntStream.EOF) {
            hitEOF = true;
        }
    } while (ttype != Token.EOF);
    return tokenTypes;
}

---

From source file:net.openchrom.xxd.processor.supplier.rscripting.ui.editor.RErrorStrategy.java

License:Open Source License

/**
 * This is called by {@link #reportError} when the exception is a
 * {@link NoViableAltException}.// ww  w  . ja  v  a 2 s  . c  o  m
 *
 * @see #reportError
 *
 * @param recognizer
 *            the parser instance
 * @param e
 *            the recognition exception
 */
protected void reportNoViableAlternative(@NotNull Parser recognizer, @NotNull NoViableAltException e) {

    TokenStream tokens = recognizer.getInputStream();
    String input;
    if (tokens != null) {
        if (e.getStartToken().getType() == Token.EOF)
            input = "<EOF>";
        else
            input = tokens.getText(e.getStartToken(), e.getOffendingToken());
    } else {
        input = "<unknown input>";
    }
    String msg = " no alternative available at input " + escapeWSAndQuote(input);
    recognizer.notifyErrorListeners(e.getOffendingToken(), msg, e);
}

---

From source file:net.sourceforge.pmd.cpd.SwiftTokenizer.java

License:BSD License

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();

    try {/*from w w  w  .  j a  v a  2 s .c  om*/
        ANTLRInputStream ais = new ANTLRInputStream(buffer.toString());
        SwiftLexer lexer = new SwiftLexer(ais);

        lexer.removeErrorListeners();
        lexer.addErrorListener(new ErrorHandler());
        Token token = lexer.nextToken();

        while (token.getType() != Token.EOF) {
            if (token.getChannel() != Lexer.HIDDEN) {
                TokenEntry tokenEntry = new TokenEntry(token.getText(), sourceCode.getFileName(),
                        token.getLine());

                tokenEntries.add(tokenEntry);
            }
            token = lexer.nextToken();
        }
    } catch (ANTLRSyntaxError err) {
        // Wrap exceptions of the Swift tokenizer in a TokenMgrError, so
        // they are correctly handled
        // when CPD is executed with the '--skipLexicalErrors' command line
        // option
        throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line "
                + err.getLine() + ", column " + err.getColumn() + ".  Encountered: " + err.getMessage(),
                TokenMgrError.LEXICAL_ERROR);
    } finally {
        tokenEntries.add(TokenEntry.getEOF());
    }
}

---

From source file:net.team2xh.crt.gui.editor.EditorTextPane.java

License:Open Source License

private void highlightText() {

    getHighlighter().removeAllHighlights();
    colorize(COMMENT, 0, 0, getText().length());

    highlightCurrentLine();//from   w  w w . j ava 2  s.c  om

    CRTLexer lexer = new CRTLexer(new ANTLRInputStream(getText()));
    CRTParser parser = new CRTParser(new CommonTokenStream(lexer));
    ParserRuleContext tree = parser.script();
    ParseTreeWalker walker = new ParseTreeWalker();

    lexer = new CRTLexer(new ANTLRInputStream(getText()));

    for (Token token = lexer.nextToken(); token.getType() != Token.EOF; token = lexer.nextToken()) {
        int cursor = token.getCharPositionInLine();
        int line = token.getLine() - 1;
        int length = token.getText().length();

        switch (token.getType()) {
        case CRTLexer.ASSIGN:
        case CRTLexer.ATTRIBUTE:
        case CRTLexer.ADD:
        case CRTLexer.SUBTRACT:
        case CRTLexer.INTERSECTION:
        case CRTLexer.MULTIPLY:
        case CRTLexer.DIVIDE:
        case CRTLexer.MODULO:
        case CRTLexer.NOT:
        case CRTLexer.LESS:
        case CRTLexer.GREATER:
        case CRTLexer.LESS_EQUAL:
        case CRTLexer.GREATER_EQUAL:
        case CRTLexer.EQUAL:
        case CRTLexer.NOT_EQUAL:
        case CRTLexer.AND:
        case CRTLexer.OR:
        case CRTLexer.QUESTION:
        case CRTLexer.COLON:
            colorize(OPERATORS, line, cursor, length);
            break;

        case CRTLexer.TRANSLATE:
        case CRTLexer.SCALE:
        case CRTLexer.ROTATE:
            colorize(TRANSFORM, line, cursor, length);
            break;

        case CRTLexer.NAME:
            colorize(NAME, line, cursor, length);
            break;

        case CRTLexer.IDENTIFIER:
            colorize(IDENTIFIER, line, cursor, length);
            break;

        case CRTLexer.FLOAT:
        case CRTLexer.INTEGER:
        case CRTLexer.TRUE:
        case CRTLexer.FALSE:
            colorize(NUMBER, line, cursor, length);
            break;

        case CRTLexer.SCENE:
        case CRTLexer.SETTINGS:
        case CRTLexer.MACRO:
            colorize(BLOCK, line, cursor, length);
            break;

        case CRTLexer.STRING:
            colorize(STRING, line, cursor, length);
            break;

        // TODO: don't hide under line highlight
        case CRTLexer.INVALID:
            try {
                int index = getLineStartOffset(line) + cursor;
                int s = getLineStartOffset(index);
                int e = getLineEndOffset(index);
                highlight(cursor, cursor + length, eh);
                highlight(s, e, lh);
            } catch (BadLocationException e) {
            }
            break;

        default:
            colorize(NORMAL, line, cursor, length);
            break;
        }
    }

    // Reset lexer for errors because parser consumed it
    walker.walk(new SyntaxHighlightingListener(), tree);

}

---

From source file:nl.knaw.AntlrUtils.java

License:Apache License

public static String makeTokenTable(Lexer lexer) {
    AsciiTable table = new AsciiTable().setTextAlignment(TextAlignment.LEFT);
    CWC_LongestLine cwc = new CWC_LongestLine();
    table.getRenderer().setCWC(cwc);/*from w  w  w  . j  av a 2  s.c o m*/
    table.addRule();
    table.addRow("Pos", "Text", "Rule", "Next mode", "Token");
    table.addRule();

    Token token;
    do {
        token = lexer.nextToken();
        //      LOG.info(token.toString());
        if (token.getType() != Token.EOF) {
            String pos = token.getLine() + ":" + token.getCharPositionInLine();
            String text = "'" + token.getText() + "'";
            String rule = lexer.getRuleNames()[token.getType() - 1];
            String mode = lexer.getModeNames()[lexer._mode];
            table.addRow(pos, text, rule, mode, token);
        }
    } while (token.getType() != Token.EOF);
    table.addRule();
    table.getContext().setGrid(A7_Grids.minusBarPlusEquals());
    return table.render();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• »