Example usage for org.antlr.v4.runtime CommonTokenStream LT

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream LT.

Prototype

@Override
    public Token LT(int k) 

Source Link

Usage

From source file:com.sri.ai.praise.demo.HOGMPanel.java

License:Open Source License

/**
 * Checks to ensure the passed in string is not whitespace or comments only.
 * @param string/*w  w  w .j a  v a 2s .  co  m*/
 * @return
 */
private boolean containsRules(String string) {
    boolean result = true;

    ANTLRInputStream input = new ANTLRInputStream(string.trim());
    RuleLexer lexer = new RuleLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    try {
        Token token = tokens.LT(1);
        if (token.getType() == Recognizer.EOF) {
            result = false;
        }
    } catch (RuntimeException ex) {
        // This is another problem, i.e. invalid token, so will let follow
        // on logic handle this when it tries to parse.
    }

    return result;
}

---

From source file:de.fuerstenau.gradle.commentstripper.CommentStripper.java

License:Open Source License

private void doStrip(Path fin, Path fout, Charset cs, DocType docType) throws IOException {
    try (Reader in = Files.newBufferedReader(fin, cs);
            OutputStream out = new BufferedOutputStream(Files.newOutputStream(fout))) {
        final ANTLRInputStream stream = new ANTLRInputStream(in);

        final CommonTokenStream tokens;
        if (docType == DocType.JAVA) {
            tokens = new CommonTokenStream(new GroovyLexer(stream), GroovyLexer.COMMENT);
        } else {//  w w w .ja v  a2 s .  c  o m
            tokens = new CommonTokenStream(new Java8Lexer(stream), Java8Lexer.COMMENTS);
        }

        TokenStreamRewriter rew = new TokenStreamRewriter(tokens);

        while (tokens.LA(1) != EOF) {
            final Token t = tokens.LT(1);
            if (cond.shouldStrip(t.getText())) {
                rew.delete(t);
            }
            tokens.consume();
        }

        out.write(rew.getText().getBytes(cs));
        out.flush();
    }
}

---

From source file:sonar.AntlrMumpsTokenizer.java

License:Open Source License

@Override
public void tokenize(SourceCode source, Tokens pmdTokens) throws IOException {
    LOG.info("Tokenizing file: " + source.getFileName());
    String fileName = source.getFileName();
    ANTLRInputStream input = new ANTLRFileStream(fileName);
    MLexer lexer = new MLexer(input);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    final int eof = -1;
    Token antlrToken = tokenStream.LT(1);

    while (antlrToken.getType() != eof) {
        tokenStream.consume();// w w  w .  jav  a 2  s . c  o  m
        TokenEntry pmdToken = new TokenEntry(MLexer.tokenNames[antlrToken.getType()], fileName,
                antlrToken.getLine());
        pmdTokens.add(pmdToken);
        antlrToken = tokenStream.LT(1);
    }

    pmdTokens.add(TokenEntry.getEOF());
}

---