Example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

List of usage examples for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

  1. HOME
  2. Java
  3. org.antlr
  4. org.antlr.v4.runtime.*
  5. CommonTokenStream
  6. CommonTokenStream

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream.

Prototype

public CommonTokenStream(TokenSource tokenSource)

Source Link

Document

Constructs a new CommonTokenStream using the specified token source and the default token channel ( Token#DEFAULT_CHANNEL ).

Usage

From source file:org.apache.sysml.parser.dml.DMLParserWrapper.java

License:Apache License

/**
 * This function is supposed to be called directly only from DmlSyntacticValidator when it encounters 'import'
 * @param fileName script file name// w  w  w  .j  av  a  2s.c  o m
 * @param dmlScript script file contents
 * @param sourceNamespace namespace from source statement
 * @param argVals script arguments
 * @return dml program, or null if at least one error
 * @throws ParseException if ParseException occurs
 */
public DMLProgram doParse(String fileName, String dmlScript, String sourceNamespace,
        Map<String, String> argVals) throws ParseException {
    DMLProgram dmlPgm = null;

    ANTLRInputStream in;
    try {
        if (dmlScript == null) {
            dmlScript = readDMLScript(fileName, LOG);
        }

        InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
        in = new ANTLRInputStream(stream);
    } catch (FileNotFoundException e) {
        throw new ParseException("Cannot find file/resource: " + fileName, e);
    } catch (IOException e) {
        throw new ParseException("Cannot open file: " + fileName, e);
    } catch (LanguageException e) {
        throw new ParseException(e.getMessage(), e);
    }

    ProgramrootContext ast = null;
    CustomErrorListener errorListener = new CustomErrorListener();

    try {
        DmlLexer lexer = new DmlLexer(in);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        DmlParser antlr4Parser = new DmlParser(tokens);

        boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value. 

        if (tryOptimizedParsing) {
            // Try faster and simpler SLL
            antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            antlr4Parser.removeErrorListeners();
            antlr4Parser.setErrorHandler(new BailErrorStrategy());
            try {
                ast = antlr4Parser.programroot();
                // If successful, no need to try out full LL(*) ... SLL was enough
            } catch (ParseCancellationException ex) {
                // Error occurred, so now try full LL(*) for better error messages
                tokens.reset();
                antlr4Parser.reset();
                if (fileName != null) {
                    errorListener.setCurrentFileName(fileName);
                } else {
                    errorListener.setCurrentFileName("MAIN_SCRIPT");
                }
                // Set our custom error listener
                antlr4Parser.addErrorListener(errorListener);
                antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
                antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
                ast = antlr4Parser.programroot();
            }
        } else {
            // Set our custom error listener
            antlr4Parser.removeErrorListeners();
            antlr4Parser.addErrorListener(errorListener);
            errorListener.setCurrentFileName(fileName);

            // Now do the parsing
            ast = antlr4Parser.programroot();
        }
    } catch (Exception e) {
        throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
    }

    // Now convert the parse tree into DMLProgram
    // Do syntactic validation while converting 
    ParseTree tree = ast;
    // And also do syntactic validation
    ParseTreeWalker walker = new ParseTreeWalker();
    // Get list of function definitions which take precedence over built-in functions if same name
    DmlPreprocessor prep = new DmlPreprocessor(errorListener);
    walker.walk(prep, tree);
    // Syntactic validation
    DmlSyntacticValidator validator = new DmlSyntacticValidator(errorListener, argVals, sourceNamespace,
            prep.getFunctionDefs());
    walker.walk(validator, tree);
    errorListener.unsetCurrentFileName();
    this.parseIssues = errorListener.getParseIssues();
    this.atLeastOneWarning = errorListener.isAtLeastOneWarning();
    this.atLeastOneError = errorListener.isAtLeastOneError();
    if (atLeastOneError) {
        throw new ParseException(parseIssues, dmlScript);
    }
    if (atLeastOneWarning) {
        LOG.warn(CustomErrorListener.generateParseIssuesMessage(dmlScript, parseIssues));
    }
    dmlPgm = createDMLProgram(ast, sourceNamespace);

    return dmlPgm;
}

From source file:org.apache.sysml.parser.pydml.PyDMLParserWrapper.java

License:Apache License

/**
 * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import'
 * @param fileName script file name// www. j ava  2  s. c om
 * @param dmlScript script file contents
 * @param sourceNamespace namespace from source statement
 * @param argVals script arguments
 * @return dml program, or null if at least one error
 * @throws ParseException if ParseException occurs
 */
public DMLProgram doParse(String fileName, String dmlScript, String sourceNamespace,
        Map<String, String> argVals) throws ParseException {
    DMLProgram dmlPgm = null;

    ANTLRInputStream in;
    try {
        if (dmlScript == null) {
            dmlScript = readDMLScript(fileName, LOG);
        }

        InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
        in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
    } catch (FileNotFoundException e) {
        throw new ParseException("Cannot find file/resource: " + fileName, e);
    } catch (IOException e) {
        throw new ParseException("Cannot open file: " + fileName, e);
    } catch (LanguageException e) {
        throw new ParseException(e.getMessage(), e);
    }

    ProgramrootContext ast = null;
    CustomErrorListener errorListener = new CustomErrorListener();

    try {
        PydmlLexer lexer = new PydmlLexer(in);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        PydmlParser antlr4Parser = new PydmlParser(tokens);

        boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value. 

        if (tryOptimizedParsing) {
            // Try faster and simpler SLL
            antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            antlr4Parser.removeErrorListeners();
            antlr4Parser.setErrorHandler(new BailErrorStrategy());
            try {
                ast = antlr4Parser.programroot();
                // If successful, no need to try out full LL(*) ... SLL was enough
            } catch (ParseCancellationException ex) {
                // Error occurred, so now try full LL(*) for better error messages
                tokens.reset();
                antlr4Parser.reset();
                if (fileName != null) {
                    errorListener.setCurrentFileName(fileName);
                } else {
                    errorListener.setCurrentFileName("MAIN_SCRIPT");
                }
                // Set our custom error listener
                antlr4Parser.addErrorListener(errorListener);
                antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
                antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
                ast = antlr4Parser.programroot();
            }
        } else {
            // Set our custom error listener
            antlr4Parser.removeErrorListeners();
            antlr4Parser.addErrorListener(errorListener);
            errorListener.setCurrentFileName(fileName);

            // Now do the parsing
            ast = antlr4Parser.programroot();
        }
    } catch (Exception e) {
        throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
    }

    // Now convert the parse tree into DMLProgram
    // Do syntactic validation while converting 
    ParseTree tree = ast;
    // And also do syntactic validation
    ParseTreeWalker walker = new ParseTreeWalker();
    // Get list of function definitions which take precedence over built-in functions if same name
    PydmlPreprocessor prep = new PydmlPreprocessor(errorListener);
    walker.walk(prep, tree);
    // Syntactic validation
    PydmlSyntacticValidator validator = new PydmlSyntacticValidator(errorListener, argVals, sourceNamespace,
            prep.getFunctionDefs());
    walker.walk(validator, tree);
    errorListener.unsetCurrentFileName();
    this.parseIssues = errorListener.getParseIssues();
    this.atLeastOneWarning = errorListener.isAtLeastOneWarning();
    this.atLeastOneError = errorListener.isAtLeastOneError();
    if (atLeastOneError) {
        throw new ParseException(parseIssues, dmlScript);
    }
    if (atLeastOneWarning) {
        LOG.warn(CustomErrorListener.generateParseIssuesMessage(dmlScript, parseIssues));
    }
    dmlPgm = createDMLProgram(ast, sourceNamespace);

    return dmlPgm;
}

From source file:org.apache.sysml.parser.python.PyDMLParserWrapper.java

License:Apache License

/**
 * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import'
 * @param fileName//  w ww.  ja  v  a  2  s .co  m
 * @return null if atleast one error
 */
public DMLProgram doParse(String fileName, String dmlScript, HashMap<String, String> argVals)
        throws ParseException {
    DMLProgram dmlPgm = null;

    ANTLRInputStream in;
    try {
        if (dmlScript == null) {
            dmlScript = DMLParserWrapper.readDMLScript(fileName);
        }

        InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
        in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
    } catch (FileNotFoundException e) {
        throw new ParseException("ERROR: Cannot find file:" + fileName, e);
    } catch (IOException e) {
        throw new ParseException("ERROR: Cannot open file:" + fileName, e);
    } catch (LanguageException e) {
        throw new ParseException("ERROR: " + e.getMessage(), e);
    }

    PmlprogramContext ast = null;
    CustomDmlErrorListener errorListener = new CustomDmlErrorListener();

    try {
        PydmlLexer lexer = new PydmlLexer(in);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        PydmlParser antlr4Parser = new PydmlParser(tokens);

        boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value. 

        if (tryOptimizedParsing) {
            // Try faster and simpler SLL
            antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            antlr4Parser.removeErrorListeners();
            antlr4Parser.setErrorHandler(new BailErrorStrategy());
            try {
                ast = antlr4Parser.pmlprogram();
                // If successful, no need to try out full LL(*) ... SLL was enough
            } catch (ParseCancellationException ex) {
                // Error occurred, so now try full LL(*) for better error messages
                tokens.reset();
                antlr4Parser.reset();
                if (fileName != null) {
                    errorListener.pushCurrentFileName(fileName);
                } else {
                    errorListener.pushCurrentFileName("MAIN_SCRIPT");
                }
                // Set our custom error listener
                antlr4Parser.addErrorListener(errorListener);
                antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
                antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
                ast = antlr4Parser.pmlprogram();
            }
        } else {
            // Set our custom error listener
            antlr4Parser.removeErrorListeners();
            antlr4Parser.addErrorListener(errorListener);
            errorListener.pushCurrentFileName(fileName);

            // Now do the parsing
            ast = antlr4Parser.pmlprogram();
        }
    } catch (Exception e) {
        throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
    }

    try {
        // Now convert the parse tree into DMLProgram
        // Do syntactic validation while converting 
        ParseTree tree = ast;
        // And also do syntactic validation
        ParseTreeWalker walker = new ParseTreeWalker();
        PydmlSyntacticValidatorHelper helper = new PydmlSyntacticValidatorHelper(errorListener);
        PydmlSyntacticValidator validator = new PydmlSyntacticValidator(helper, fileName, argVals);
        walker.walk(validator, tree);
        errorListener.popFileName();
        if (errorListener.isAtleastOneError()) {
            return null;
        }
        dmlPgm = createDMLProgram(ast);
    } catch (Exception e) {
        throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram" + e.getMessage(), e);
    }

    return dmlPgm;
}

From source file:org.apache.tajo.engine.parser.HiveConverter.java

License:Apache License

public Expr parse(String sql) {
    HiveLexer lexer = new HiveLexer(new ANTLRNoCaseStringStream(sql));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    parser = new HiveParser(tokens);
    parser.setBuildParseTree(true);//from w  ww .  j a  v  a  2 s  . c  om

    HiveParser.StatementContext context;
    try {
        context = parser.statement();
    } catch (SQLParseError e) {
        throw new SQLSyntaxError(e);
    }

    return visit(context);
}

From source file:org.apache.tajo.engine.parser.HiveQLAnalyzer.java

License:Apache License

public Expr parse(String sql) {
    HiveQLLexer lexer = new HiveQLLexer(new ANTLRNoCaseStringStream(sql));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    parser = new HiveQLParser(tokens);
    parser.setBuildParseTree(true);/*from  w w w.j a va2s.c  o  m*/

    HiveQLParser.StatementContext context;
    try {
        context = parser.statement();
    } catch (SQLParseError e) {
        throw new SQLSyntaxError(e);
    }

    return visit(context);
}

From source file:org.apache.tajo.engine.parser.SQLAnalyzer.java

License:Apache License

public Expr parse(String sql) {
    ANTLRInputStream input = new ANTLRInputStream(sql);
    SQLLexer lexer = new SQLLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    this.parser = new SQLParser(tokens);
    parser.setBuildParseTree(true);/*from   w  ww .  j  a  v  a  2 s  .co m*/
    parser.removeErrorListeners();

    parser.setErrorHandler(new SQLErrorStrategy());
    parser.addErrorListener(new SQLErrorListener());

    SqlContext context;
    try {
        context = parser.sql();
    } catch (SQLParseError e) {
        e.printStackTrace();
        throw new SQLSyntaxError(e);
    }
    return visitSql(context);
}

From source file:org.apache.tajo.parser.sql.SQLAnalyzer.java

License:Apache License

public Expr parse(String sql) throws SQLSyntaxError {

    final ANTLRInputStream input = new ANTLRInputStream(sql);
    final SQLLexer lexer = new SQLLexer(input);
    lexer.removeErrorListeners();//from w w  w.ja v  a  2s .c om
    lexer.addErrorListener(new SQLErrorListener());

    final CommonTokenStream tokens = new CommonTokenStream(lexer);

    final SQLParser parser = new SQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(new SQLErrorListener());
    parser.setBuildParseTree(true);

    SqlContext context;
    try {
        context = parser.sql();
    } catch (SQLParseError e) {
        throw new SQLSyntaxError(e.getMessage());
    } catch (Throwable t) {
        throw new TajoInternalError(t.getMessage());
    }

    return visitSql(context);
}

From source file:org.babyfish.persistence.path.QueryPaths.java

License:Open Source License

private static List<QueryPath> compileViaCache(String queryPath) {

    Lock lock;//w ww.  jav  a 2 s  .c  o m
    List<QueryPath> compileResult;

    (lock = CACHE_LOCK.readLock()).lock(); //1st locking
    try {
        compileResult = LEVEL_ONE_CACHE.get(queryPath); //1st level-1 reading
        if (compileResult == null) { //1st level-1 checking
            compileResult = LEVEL_TWO_CACHE.access(queryPath); //1st level-2 reading
        }
    } finally {
        lock.unlock();
    }

    if (compileResult == null) { //1st level-2 checking
        (lock = CACHE_LOCK.writeLock()).lock(); //2nd locking
        try {
            compileResult = LEVEL_ONE_CACHE.get(queryPath); //2nd level-1 reading
            if (compileResult == null) { //2nd level-1 checking
                compileResult = LEVEL_TWO_CACHE.access(queryPath); //2nd level-2 reading
                if (compileResult == null) { //2nd level-2 checking
                    ANTLRInputStream input = new ANTLRInputStream(queryPath);
                    QueryPathLexer lexer = new QueryPathLexer(input);
                    CommonTokenStream tokens = new CommonTokenStream(lexer);
                    QueryPathParser parser = new QueryPathParser(tokens);
                    QueryPathErrorListener queryPathErrorListener = new QueryPathErrorListener();
                    lexer.removeErrorListeners();
                    lexer.addErrorListener(queryPathErrorListener);
                    parser.removeErrorListeners();
                    parser.addErrorListener(queryPathErrorListener);
                    VisitorImpl visitor = new VisitorImpl();
                    parser.main().accept(visitor);
                    compileResult = visitor.getQueryPaths();

                    //save to level-2 cache
                    for (int i = LEVEL_TWO_CACHE.size() - LEVEL_2_CACHE_MAX_SIZE; i >= 0; i--) {
                        LEVEL_TWO_CACHE.pollFirstEntry();
                    }
                    LEVEL_TWO_CACHE.put(queryPath, compileResult);
                }
                //save to level-1 cache
                LEVEL_ONE_CACHE.put(queryPath, compileResult);
            }
        } finally {
            lock.unlock();
        }
    }
    return compileResult;
}

From source file:org.ballerinalang.composer.service.workspace.rest.datamodel.BLangFileRestService.java

License:Open Source License

/**
 * Parses an input stream into a json model. During this parsing we are compiling the code as well.
 * @param stream - The input stream./*w w  w. j a va  2  s.  c o m*/
 * @return A string which contains a json model.
 * @throws IOException
 */
private String parseJsonDataModel(InputStream stream) throws IOException {

    ANTLRInputStream antlrInputStream = new ANTLRInputStream(stream);
    BallerinaLexer ballerinaLexer = new BallerinaLexer(antlrInputStream);
    CommonTokenStream ballerinaToken = new CommonTokenStream(ballerinaLexer);

    BallerinaParser ballerinaParser = new BallerinaParser(ballerinaToken);
    BallerinaComposerErrorStrategy errorStrategy = new BallerinaComposerErrorStrategy();
    ballerinaParser.setErrorHandler(errorStrategy);

    GlobalScope globalScope = GlobalScope.getInstance();
    BTypes.loadBuiltInTypes(globalScope);
    BLangPackage bLangPackage = new BLangPackage(globalScope);
    BLangPackage.PackageBuilder packageBuilder = new BLangPackage.PackageBuilder(bLangPackage);
    BallerinaComposerModelBuilder bLangModelBuilder = new BallerinaComposerModelBuilder(packageBuilder,
            StringUtils.EMPTY);
    BLangAntlr4Listener ballerinaBaseListener = new BLangAntlr4Listener(bLangModelBuilder);
    ballerinaParser.addParseListener(ballerinaBaseListener);
    ballerinaParser.compilationUnit();
    BallerinaFile bFile = bLangModelBuilder.build();

    BuiltInNativeConstructLoader.loadConstructs(globalScope);

    JsonObject response = new JsonObject();
    BLangJSONModelBuilder jsonModelBuilder = new BLangJSONModelBuilder(response);
    bFile.accept(jsonModelBuilder);

    return response.toString();
}

From source file:org.ballerinalang.composer.service.workspace.rest.datamodel.BLangFileRestService.java

License:Open Source License

/**
 * Validates a given ballerina input// w  w w  .j av a 2 s.c  o  m
 * @param stream - The input stream.
 * @return List of errors if any
 * @throws IOException
 */
private JsonObject validate(InputStream stream) throws IOException {

    ANTLRInputStream antlrInputStream = new ANTLRInputStream(stream);
    BallerinaLexer ballerinaLexer = new BallerinaLexer(antlrInputStream);
    CommonTokenStream ballerinaToken = new CommonTokenStream(ballerinaLexer);

    BallerinaParser ballerinaParser = new BallerinaParser(ballerinaToken);
    BallerinaComposerErrorStrategy errorStrategy = new BallerinaComposerErrorStrategy();
    ballerinaParser.setErrorHandler(errorStrategy);

    GlobalScope globalScope = GlobalScope.getInstance();
    BTypes.loadBuiltInTypes(globalScope);
    BLangPackage bLangPackage = new BLangPackage(globalScope);
    BLangPackage.PackageBuilder packageBuilder = new BLangPackage.PackageBuilder(bLangPackage);

    BallerinaComposerModelBuilder bLangModelBuilder = new BallerinaComposerModelBuilder(packageBuilder,
            StringUtils.EMPTY);
    BLangAntlr4Listener ballerinaBaseListener = new BLangAntlr4Listener(bLangModelBuilder);
    ballerinaParser.addParseListener(ballerinaBaseListener);
    ballerinaParser.compilationUnit();

    JsonArray errors = new JsonArray();

    for (SyntaxError error : errorStrategy.getErrorTokens()) {
        errors.add(error.toJson());
    }

    JsonObject result = new JsonObject();
    result.add("errors", errors);

    return result;
}