Example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream.

Prototype

public CommonTokenStream(TokenSource tokenSource)

Source Link

Document

Constructs a new CommonTokenStream using the specified token source and the default token channel ( Token#DEFAULT_CHANNEL ).

Usage

From source file:io.proleap.cobol.preprocessor.sub.document.impl.CobolDocumentParserImpl.java

License:Open Source License

protected String processWithParser(final String code, final List<File> copyFiles,
        final CobolSourceFormatEnum format, final CobolDialect dialect) {
    // run the lexer
    final Cobol85PreprocessorLexer lexer = new Cobol85PreprocessorLexer(CharStreams.fromString(code));

    // get a list of matched tokens
    final CommonTokenStream tokens = new CommonTokenStream(lexer);

    // pass the tokens to the parser
    final Cobol85PreprocessorParser parser = new Cobol85PreprocessorParser(tokens);

    // register an error listener, so that preprocessing stops on errors
    parser.removeErrorListeners();/*  w  w w .  j  a  va  2 s.c o m*/
    parser.addErrorListener(new ThrowingErrorListener());

    // specify our entry point
    final StartRuleContext startRule = parser.startRule();

    // analyze contained copy books
    final CobolDocumentParserListenerImpl listener = new CobolDocumentParserListenerImpl(copyFiles, format,
            dialect, tokens);
    final ParseTreeWalker walker = new ParseTreeWalker();

    walker.walk(listener, startRule);

    final String result = listener.context().read();
    return result;
}

From source file:io.proleap.cobol.preprocessor.sub.parser.impl.CobolParserPreprocessorImpl.java

License:Open Source License

protected String processWithParser(final String program, final File libDirectory,
        final CobolSourceFormatEnum format, final CobolDialect dialect) {
    // run the lexer
    final Cobol85PreprocessorLexer lexer = new Cobol85PreprocessorLexer(CharStreams.fromString(program));

    // get a list of matched tokens
    final CommonTokenStream tokens = new CommonTokenStream(lexer);

    // pass the tokens to the parser
    final Cobol85PreprocessorParser parser = new Cobol85PreprocessorParser(tokens);

    // register an error listener, so that preprocessing stops on errors
    parser.removeErrorListeners();/*  www .  ja va2s  .co m*/
    parser.addErrorListener(new ThrowingErrorListener());

    // specify our entry point
    final StartRuleContext startRule = parser.startRule();

    // analyze contained copy books
    final CobolParserPreprocessorListenerImpl listener = new CobolParserPreprocessorListenerImpl(libDirectory,
            format, dialect, tokens);
    final ParseTreeWalker walker = new ParseTreeWalker();

    walker.walk(listener, startRule);

    final String result = listener.context().read();
    return result;
}

From source file:io.proleap.cobol.runner.impl.CobolParseTestRunnerImpl.java

License:Open Source License

protected void doParse(final String preProcessedInput, final File inputFile, final CobolParserParams params)
        throws IOException {
    final CobolLexer lexer = new CobolLexer(CharStreams.fromString(preProcessedInput));

    if (!params.getIgnoreSyntaxErrors()) {
        lexer.removeErrorListeners();/*from ww w .  j a v a2s. co  m*/
        lexer.addErrorListener(new ThrowingErrorListener());
    }

    final CommonTokenStream tokens = new CommonTokenStream(lexer);
    final CobolParser parser = new CobolParser(tokens);

    if (!params.getIgnoreSyntaxErrors()) {
        parser.removeErrorListeners();
        parser.addErrorListener(new ThrowingErrorListener());
    }

    final StartRuleContext startRule = parser.startRule();
    final File treeFile = new File(inputFile.getAbsolutePath() + TREE_SUFFIX);

    if (treeFile.exists()) {
        doCompareParseTree(treeFile, startRule, parser);
    }
}

From source file:io.proleap.vb6.asg.runner.impl.VbParserRunnerImpl.java

License:Open Source License

protected void parseFile(final File inputFile, final Charset charset, final Program program)
        throws IOException {
    if (!inputFile.isFile()) {
        LOG.warn("Could not find file {}", inputFile.getAbsolutePath());
    } else {/* w w w .  j a  v a2  s .c  o  m*/
        final String input = FileUtils.readFileToString(inputFile, charset);

        LOG.info("Parsing file {}.", inputFile.getName());

        final InputStream inputStream = new FileInputStream(inputFile);

        final VisualBasic6Lexer lexer = new VisualBasic6Lexer(CharStreams.fromStream(inputStream, charset));

        // get a list of matched tokens
        final CommonTokenStream tokens = new CommonTokenStream(lexer);

        // pass the tokens to the parser
        final VisualBasic6Parser parser = new VisualBasic6Parser(tokens);

        // specify our entry point
        final StartRuleContext ctx = parser.startRule();

        // determine the module name
        final String declaredModuleName = analyzeDeclaredModuleName(ctx);
        final String moduleName;

        if (declaredModuleName != null && !declaredModuleName.isEmpty()) {
            moduleName = declaredModuleName;
        } else {
            moduleName = getModuleName(inputFile);
        }

        // analyze contained modules and types
        final boolean isClazzModule = isClazzModule(inputFile);
        final boolean isStandardModule = isStandardModule(inputFile);

        final List<String> lines = splitLines(input);
        final ParserVisitor visitor = new VbModuleVisitorImpl(moduleName, lines, isClazzModule,
                isStandardModule, tokens, program);

        LOG.info("Collecting types in file {}.", inputFile.getName());
        visitor.visit(ctx);
    }
}

From source file:io.proleap.vb6.runner.impl.VbParseTestRunnerImpl.java

License:Open Source License

protected void doParse(final File inputFile) throws IOException {
    LOG.info("Parsing file {}.", inputFile.getName());

    final InputStream inputStream = new FileInputStream(inputFile);
    final VisualBasic6Lexer lexer = new VisualBasic6Lexer(new ANTLRInputStream(inputStream));

    lexer.removeErrorListeners();//from   w ww .  j  a v  a2 s. c o m
    lexer.addErrorListener(ThrowingErrorListener.INSTANCE);

    final CommonTokenStream tokens = new CommonTokenStream(lexer);
    final VisualBasic6Parser parser = new VisualBasic6Parser(tokens);

    parser.removeErrorListeners();
    parser.addErrorListener(ThrowingErrorListener.INSTANCE);

    final StartRuleContext startRule = parser.startRule();
    final File treeFile = new File(inputFile.getAbsolutePath() + TREE_SUFFIX);

    if (treeFile.exists()) {
        doCompareParseTree(treeFile, startRule, parser);
    }
}

From source file:io.siddhi.query.compiler.SiddhiCompiler.java

License:Open Source License

public static SiddhiApp parse(String source) {

    ANTLRInputStream input = new ANTLRInputStream(source);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();//from w  ww.  java 2  s. c o  m
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    //            parser.setErrorHandler(new BailErrorStrategy());
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.parse();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (SiddhiApp) eval.visit(tree);
}

From source file:io.siddhi.query.compiler.SiddhiCompiler.java

License:Open Source License

public static StreamDefinition parseStreamDefinition(String source) {

    ANTLRInputStream input = new ANTLRInputStream(source);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();/* www .  j  ava2  s . c  o m*/
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.definition_stream_final();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (StreamDefinition) eval.visit(tree);
}

From source file:io.siddhi.query.compiler.SiddhiCompiler.java

License:Open Source License

public static TableDefinition parseTableDefinition(String source) throws SiddhiParserException {

    ANTLRInputStream input = new ANTLRInputStream(source);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();//from   w w  w  . ja v  a 2 s .  co m
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.definition_table_final();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (TableDefinition) eval.visit(tree);
}

From source file:io.siddhi.query.compiler.SiddhiCompiler.java

License:Open Source License

public static AggregationDefinition parseAggregationDefinition(String source) throws SiddhiParserException {

    ANTLRInputStream input = new ANTLRInputStream(source);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();/*from   ww  w  .ja  v  a 2  s  .  c o m*/
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.definition_aggregation_final();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (AggregationDefinition) eval.visit(tree);
}

From source file:io.siddhi.query.compiler.SiddhiCompiler.java

License:Open Source License

public static Partition parsePartition(String source) throws SiddhiParserException {

    ANTLRInputStream input = new ANTLRInputStream(source);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();//  w  ww  . j  a v a 2  s  .com
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.partition_final();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (Partition) eval.visit(tree);
}