Example usage for org.antlr.v4.runtime CommonTokenStream consume

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream consume.

Prototype

@Override
    public void consume() 

Source Link

Usage

From source file:de.fuerstenau.gradle.commentstripper.CommentStripper.java

License:Open Source License

private void doStrip(Path fin, Path fout, Charset cs, DocType docType) throws IOException {
    try (Reader in = Files.newBufferedReader(fin, cs);
            OutputStream out = new BufferedOutputStream(Files.newOutputStream(fout))) {
        final ANTLRInputStream stream = new ANTLRInputStream(in);

        final CommonTokenStream tokens;
        if (docType == DocType.JAVA) {
            tokens = new CommonTokenStream(new GroovyLexer(stream), GroovyLexer.COMMENT);
        } else {/*from  www  .  ja  v  a  2s .c om*/
            tokens = new CommonTokenStream(new Java8Lexer(stream), Java8Lexer.COMMENTS);
        }

        TokenStreamRewriter rew = new TokenStreamRewriter(tokens);

        while (tokens.LA(1) != EOF) {
            final Token t = tokens.LT(1);
            if (cond.shouldStrip(t.getText())) {
                rew.delete(t);
            }
            tokens.consume();
        }

        out.write(rew.getText().getBytes(cs));
        out.flush();
    }
}

---

From source file:sonar.AntlrMumpsTokenizer.java

License:Open Source License

@Override
public void tokenize(SourceCode source, Tokens pmdTokens) throws IOException {
    LOG.info("Tokenizing file: " + source.getFileName());
    String fileName = source.getFileName();
    ANTLRInputStream input = new ANTLRFileStream(fileName);
    MLexer lexer = new MLexer(input);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    final int eof = -1;
    Token antlrToken = tokenStream.LT(1);

    while (antlrToken.getType() != eof) {
        tokenStream.consume();
        TokenEntry pmdToken = new TokenEntry(MLexer.tokenNames[antlrToken.getType()], fileName,
                antlrToken.getLine());/* w  ww .  ja  v  a  2 s .  com*/
        pmdTokens.add(pmdToken);
        antlrToken = tokenStream.LT(1);
    }

    pmdTokens.add(TokenEntry.getEOF());
}

---