List of usage examples for org.antlr.v4.runtime CommonTokenStream reset
@Deprecated public void reset()
From source file:boa.compiler.BoaCompiler.java
License:Apache License
private static Start parse(final CommonTokenStream tokens, final BoaParser parser, final BoaErrorListener parserErrorListener) { parser.setBuildParseTree(false);/*ww w .j a v a 2 s . c o m*/ parser.getInterpreter().setPredictionMode(PredictionMode.SLL); try { return parser.start().ast; } catch (final ParseCancellationException e) { // fall-back to LL mode parsing if SLL fails tokens.reset(); parser.reset(); parser.removeErrorListeners(); parser.addErrorListener(parserErrorListener); parser.getInterpreter().setPredictionMode(PredictionMode.LL); return parser.start().ast; } }
From source file:com.facebook.presto.type.TypeCalculation.java
License:Apache License
private static ParserRuleContext parseTypeCalculation(String calculation) { TypeCalculationLexer lexer = new TypeCalculationLexer( new CaseInsensitiveStream(new ANTLRInputStream(calculation))); CommonTokenStream tokenStream = new CommonTokenStream(lexer); TypeCalculationParser parser = new TypeCalculationParser(tokenStream); lexer.removeErrorListeners();/*from w w w . j a v a 2 s . c om*/ lexer.addErrorListener(ERROR_LISTENER); parser.removeErrorListeners(); parser.addErrorListener(ERROR_LISTENER); ParserRuleContext tree; try { // first, try parsing with potentially faster SLL mode parser.getInterpreter().setPredictionMode(PredictionMode.SLL); tree = parser.typeCalculation(); } catch (ParseCancellationException ex) { // if we fail, parse with LL mode tokenStream.reset(); // rewind input stream parser.reset(); parser.getInterpreter().setPredictionMode(PredictionMode.LL); tree = parser.typeCalculation(); } return tree; }
From source file:com.ibm.bi.dml.parser.python.PyDMLParserWrapper.java
License:Open Source License
/** * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import' * @param fileName//www . j a v a 2 s . c om * @return null if atleast one error */ public DMLProgram doParse(String fileName, String dmlScript, HashMap<String, String> argVals) throws ParseException { DMLProgram dmlPgm = null; ANTLRInputStream in; try { if (dmlScript == null) { dmlScript = DMLParserWrapper.readDMLScript(fileName); } InputStream stream = new ByteArrayInputStream(dmlScript.getBytes()); in = new org.antlr.v4.runtime.ANTLRInputStream(stream); // else { // if(!(new File(fileName)).exists()) { // throw new ParseException("ERROR: Cannot open file:" + fileName); // } // in = new ANTLRInputStream(new FileInputStream(fileName)); // } } catch (FileNotFoundException e) { throw new ParseException("ERROR: Cannot find file:" + fileName); } catch (IOException e) { throw new ParseException("ERROR: Cannot open file:" + fileName); } catch (LanguageException e) { throw new ParseException("ERROR: " + e.getMessage()); } PmlprogramContext ast = null; CustomDmlErrorListener errorListener = new CustomDmlErrorListener(); try { PydmlLexer lexer = new PydmlLexer(in); CommonTokenStream tokens = new CommonTokenStream(lexer); PydmlParser antlr4Parser = new PydmlParser(tokens); boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value. if (tryOptimizedParsing) { // Try faster and simpler SLL antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL); antlr4Parser.removeErrorListeners(); antlr4Parser.setErrorHandler(new BailErrorStrategy()); try { ast = antlr4Parser.pmlprogram(); // If successful, no need to try out full LL(*) ... SLL was enough } catch (ParseCancellationException ex) { // Error occurred, so now try full LL(*) for better error messages tokens.reset(); antlr4Parser.reset(); if (fileName != null) { errorListener.pushCurrentFileName(fileName); } else { errorListener.pushCurrentFileName("MAIN_SCRIPT"); } // Set our custom error listener antlr4Parser.addErrorListener(errorListener); antlr4Parser.setErrorHandler(new DefaultErrorStrategy()); antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL); ast = antlr4Parser.pmlprogram(); } } else { // Set our custom error listener antlr4Parser.removeErrorListeners(); antlr4Parser.addErrorListener(errorListener); errorListener.pushCurrentFileName(fileName); // Now do the parsing ast = antlr4Parser.pmlprogram(); } } catch (Exception e) { throw new ParseException("ERROR: Cannot parse the program:" + fileName); } try { // Now convert the parse tree into DMLProgram // Do syntactic validation while converting ParseTree tree = ast; // And also do syntactic validation ParseTreeWalker walker = new ParseTreeWalker(); PydmlSyntacticValidatorHelper helper = new PydmlSyntacticValidatorHelper(errorListener); PydmlSyntacticValidator validator = new PydmlSyntacticValidator(helper, fileName, argVals); walker.walk(validator, tree); errorListener.popFileName(); if (errorListener.isAtleastOneError()) { return null; } dmlPgm = createDMLProgram(ast); } catch (Exception e) { throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram:" + e.getMessage()); } return dmlPgm; }
From source file:edu.iastate.cs.boa.ui.errors.FetchCompilerError.java
License:Apache License
protected StartContext parse(final String input, final String[] errors) throws IOException { final CommonTokenStream tokens = lex(input); final BoaParser parser = new BoaParser(tokens); final List<String> foundErr = new ArrayList<String>(); parser.removeErrorListeners();/*w w w . ja v a2s . c o m*/ parser.addErrorListener(new BaseErrorListener() { @Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) throws ParseCancellationException { throw new ParseCancellationException(e); } }); parseErrorListener = new ParserErrorListener(); parser.setBuildParseTree(false); parser.getInterpreter().setPredictionMode(PredictionMode.SLL); StartContext p; try { p = parser.start(); } catch (final Exception e) { // fall-back to LL mode parsing if SLL fails tokens.reset(); parser.reset(); parser.removeErrorListeners(); parser.addErrorListener(new BaseErrorListener() { @Override public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, final int charPositionInLine, final String msg, final RecognitionException e) { foundErr.add(line + "," + charPositionInLine + ": " + msg); } }); parser.getInterpreter().setPredictionMode(PredictionMode.LL); p = parser.start(); } return p; }
From source file:io.prestosql.sql.parser.SqlParser.java
License:Apache License
private Node invokeParser(String name, String sql, Function<SqlBaseParser, ParserRuleContext> parseFunction, ParsingOptions parsingOptions) { try {/*w w w. j a va 2s. c o m*/ SqlBaseLexer lexer = new SqlBaseLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); CommonTokenStream tokenStream = new CommonTokenStream(lexer); SqlBaseParser parser = new SqlBaseParser(tokenStream); // Override the default error strategy to not attempt inserting or deleting a token. // Otherwise, it messes up error reporting parser.setErrorHandler(new DefaultErrorStrategy() { @Override public Token recoverInline(Parser recognizer) throws RecognitionException { if (nextTokensContext == null) { throw new InputMismatchException(recognizer); } else { throw new InputMismatchException(recognizer, nextTokensState, nextTokensContext); } } }); parser.addParseListener(new PostProcessor(Arrays.asList(parser.getRuleNames()))); lexer.removeErrorListeners(); lexer.addErrorListener(LEXER_ERROR_LISTENER); parser.removeErrorListeners(); if (enhancedErrorHandlerEnabled) { parser.addErrorListener(PARSER_ERROR_HANDLER); } else { parser.addErrorListener(LEXER_ERROR_LISTENER); } ParserRuleContext tree; try { // first, try parsing with potentially faster SLL mode parser.getInterpreter().setPredictionMode(PredictionMode.SLL); tree = parseFunction.apply(parser); } catch (ParseCancellationException ex) { // if we fail, parse with LL mode tokenStream.reset(); // rewind input stream parser.reset(); parser.getInterpreter().setPredictionMode(PredictionMode.LL); tree = parseFunction.apply(parser); } return new AstBuilder(parsingOptions).visit(tree); } catch (StackOverflowError e) { throw new ParsingException(name + " is too large (stack overflow while parsing)"); } }
From source file:net.openchrom.xxd.processor.supplier.rscripting.ui.editor.RenameProperty.java
License:Open Source License
private void applySelection(final ITextSelection textSelection) { info.setOldName(textSelection.getText()); info.setNewName(textSelection.getText()); info.addOffset(textSelection.getOffset()); /* Add AST manipulation here! */ IEditorPart editor = (IEditorPart) PlatformUI.getWorkbench().getActiveWorkbenchWindow().getActivePage() .getActiveEditor();//from w w w . j a v a 2s. c o m if (editor instanceof REditor) { IDocument doc = ((ITextEditor) editor).getDocumentProvider().getDocument(editor.getEditorInput()); ANTLRInputStream input = new ANTLRInputStream(doc.get()); RLexer lexer = new RLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); RFilter filter = new RFilter(tokens); filter.stream(); // call start rule: stream tokens.reset(); RParser parser = new RParser(tokens); RuleContext tree = parser.prog(); /* * ParseTreePattern p = parser.compileParseTreePattern("ID", RParser.RULE_expr); * ParseTreeMatch m = p.match(tree); * if ( m.succeeded() ) { * System.out.println("succeed!"); * } */ info.addOffset(textSelection.getOffset() + 150); info.addOffset(textSelection.getOffset() + 250); info.setSourceFile(getFile()); } }
From source file:net.openchrom.xxd.processor.supplier.rscripting.ui.editor.RReconcilingStrategy.java
License:Open Source License
/** * next character position - used locally and only valid while {@link #calculatePositions()} is in progress. */// www . j a v a 2 s . c om protected void calculatePositions() { Vector<REditorOutlineNode> editorOldNodes = editor.nodes; /* Create the category base node for the outline! */ editor.createNodes(); if (editor != null) { IResource resource = (IResource) editor.getEditorInput().getAdapter(IResource.class); if (resource != null) { try { resource.deleteMarkers(IMarker.PROBLEM, true, IResource.DEPTH_INFINITE); } catch (CoreException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } } } IDocument doc = fDocument; ANTLRInputStream input = new ANTLRInputStream(doc.get()); RLexer lexer = new RLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); RFilter filter = new RFilter(tokens); filter.stream(); // call start rule: stream tokens.reset(); RParser parser = new RParser(tokens); // parser.setErrorHandler(new RErrorStrategy()); parser.setBuildParseTree(true); UnderlineListener li = new UnderlineListener(editor); lexer.removeErrorListeners(); lexer.addErrorListener(li); parser.removeErrorListeners(); // parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION); parser.addErrorListener(li); // Token to= parser.match(0); // System.out.println("Errors: " + parser.getNumberOfSyntaxErrors()); ParseTreeWalker walker = new ParseTreeWalker(); RuleContext tree = parser.prog(); /* Create the listener to create the outline, etc. */ RBaseListen list = new RBaseListen(tokens, editor, parser); list.startStop.clear(); walker.walk(list, tree); /* * long startTime = System.currentTimeMillis(); * long stopTime = System.currentTimeMillis(); * long elapsedTime = stopTime - startTime; * System.out.println(elapsedTime); */ fPositions.clear(); for (int i = 0; i < list.startStop.size(); i++) { String pos = (String) list.startStop.get(i); String[] val = pos.split(","); fPositions.add(new Position(Integer.parseInt(val[0]), Integer.parseInt(val[1]))); } /* Update the outline! */ Display.getDefault().asyncExec(new Runnable() { public void run() { editor.updateFoldingStructure(fPositions); editor.outlineInputChanged(editorOldNodes, editor.nodes); } }); }
From source file:org.apache.sysml.parser.dml.DMLParserWrapper.java
License:Apache License
/** * This function is supposed to be called directly only from DmlSyntacticValidator when it encounters 'import' * @param fileName script file name/* ww w . ja va 2s. com*/ * @param dmlScript script file contents * @param sourceNamespace namespace from source statement * @param argVals script arguments * @return dml program, or null if at least one error * @throws ParseException if ParseException occurs */ public DMLProgram doParse(String fileName, String dmlScript, String sourceNamespace, Map<String, String> argVals) throws ParseException { DMLProgram dmlPgm = null; ANTLRInputStream in; try { if (dmlScript == null) { dmlScript = readDMLScript(fileName, LOG); } InputStream stream = new ByteArrayInputStream(dmlScript.getBytes()); in = new ANTLRInputStream(stream); } catch (FileNotFoundException e) { throw new ParseException("Cannot find file/resource: " + fileName, e); } catch (IOException e) { throw new ParseException("Cannot open file: " + fileName, e); } catch (LanguageException e) { throw new ParseException(e.getMessage(), e); } ProgramrootContext ast = null; CustomErrorListener errorListener = new CustomErrorListener(); try { DmlLexer lexer = new DmlLexer(in); CommonTokenStream tokens = new CommonTokenStream(lexer); DmlParser antlr4Parser = new DmlParser(tokens); boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value. if (tryOptimizedParsing) { // Try faster and simpler SLL antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL); antlr4Parser.removeErrorListeners(); antlr4Parser.setErrorHandler(new BailErrorStrategy()); try { ast = antlr4Parser.programroot(); // If successful, no need to try out full LL(*) ... SLL was enough } catch (ParseCancellationException ex) { // Error occurred, so now try full LL(*) for better error messages tokens.reset(); antlr4Parser.reset(); if (fileName != null) { errorListener.setCurrentFileName(fileName); } else { errorListener.setCurrentFileName("MAIN_SCRIPT"); } // Set our custom error listener antlr4Parser.addErrorListener(errorListener); antlr4Parser.setErrorHandler(new DefaultErrorStrategy()); antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL); ast = antlr4Parser.programroot(); } } else { // Set our custom error listener antlr4Parser.removeErrorListeners(); antlr4Parser.addErrorListener(errorListener); errorListener.setCurrentFileName(fileName); // Now do the parsing ast = antlr4Parser.programroot(); } } catch (Exception e) { throw new ParseException("ERROR: Cannot parse the program:" + fileName, e); } // Now convert the parse tree into DMLProgram // Do syntactic validation while converting ParseTree tree = ast; // And also do syntactic validation ParseTreeWalker walker = new ParseTreeWalker(); // Get list of function definitions which take precedence over built-in functions if same name DmlPreprocessor prep = new DmlPreprocessor(errorListener); walker.walk(prep, tree); // Syntactic validation DmlSyntacticValidator validator = new DmlSyntacticValidator(errorListener, argVals, sourceNamespace, prep.getFunctionDefs()); walker.walk(validator, tree); errorListener.unsetCurrentFileName(); this.parseIssues = errorListener.getParseIssues(); this.atLeastOneWarning = errorListener.isAtLeastOneWarning(); this.atLeastOneError = errorListener.isAtLeastOneError(); if (atLeastOneError) { throw new ParseException(parseIssues, dmlScript); } if (atLeastOneWarning) { LOG.warn(CustomErrorListener.generateParseIssuesMessage(dmlScript, parseIssues)); } dmlPgm = createDMLProgram(ast, sourceNamespace); return dmlPgm; }
From source file:org.apache.sysml.parser.pydml.PyDMLParserWrapper.java
License:Apache License
/** * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import' * @param fileName script file name//from w ww . ja v a 2s . c om * @param dmlScript script file contents * @param sourceNamespace namespace from source statement * @param argVals script arguments * @return dml program, or null if at least one error * @throws ParseException if ParseException occurs */ public DMLProgram doParse(String fileName, String dmlScript, String sourceNamespace, Map<String, String> argVals) throws ParseException { DMLProgram dmlPgm = null; ANTLRInputStream in; try { if (dmlScript == null) { dmlScript = readDMLScript(fileName, LOG); } InputStream stream = new ByteArrayInputStream(dmlScript.getBytes()); in = new org.antlr.v4.runtime.ANTLRInputStream(stream); } catch (FileNotFoundException e) { throw new ParseException("Cannot find file/resource: " + fileName, e); } catch (IOException e) { throw new ParseException("Cannot open file: " + fileName, e); } catch (LanguageException e) { throw new ParseException(e.getMessage(), e); } ProgramrootContext ast = null; CustomErrorListener errorListener = new CustomErrorListener(); try { PydmlLexer lexer = new PydmlLexer(in); CommonTokenStream tokens = new CommonTokenStream(lexer); PydmlParser antlr4Parser = new PydmlParser(tokens); boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value. if (tryOptimizedParsing) { // Try faster and simpler SLL antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL); antlr4Parser.removeErrorListeners(); antlr4Parser.setErrorHandler(new BailErrorStrategy()); try { ast = antlr4Parser.programroot(); // If successful, no need to try out full LL(*) ... SLL was enough } catch (ParseCancellationException ex) { // Error occurred, so now try full LL(*) for better error messages tokens.reset(); antlr4Parser.reset(); if (fileName != null) { errorListener.setCurrentFileName(fileName); } else { errorListener.setCurrentFileName("MAIN_SCRIPT"); } // Set our custom error listener antlr4Parser.addErrorListener(errorListener); antlr4Parser.setErrorHandler(new DefaultErrorStrategy()); antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL); ast = antlr4Parser.programroot(); } } else { // Set our custom error listener antlr4Parser.removeErrorListeners(); antlr4Parser.addErrorListener(errorListener); errorListener.setCurrentFileName(fileName); // Now do the parsing ast = antlr4Parser.programroot(); } } catch (Exception e) { throw new ParseException("ERROR: Cannot parse the program:" + fileName, e); } // Now convert the parse tree into DMLProgram // Do syntactic validation while converting ParseTree tree = ast; // And also do syntactic validation ParseTreeWalker walker = new ParseTreeWalker(); // Get list of function definitions which take precedence over built-in functions if same name PydmlPreprocessor prep = new PydmlPreprocessor(errorListener); walker.walk(prep, tree); // Syntactic validation PydmlSyntacticValidator validator = new PydmlSyntacticValidator(errorListener, argVals, sourceNamespace, prep.getFunctionDefs()); walker.walk(validator, tree); errorListener.unsetCurrentFileName(); this.parseIssues = errorListener.getParseIssues(); this.atLeastOneWarning = errorListener.isAtLeastOneWarning(); this.atLeastOneError = errorListener.isAtLeastOneError(); if (atLeastOneError) { throw new ParseException(parseIssues, dmlScript); } if (atLeastOneWarning) { LOG.warn(CustomErrorListener.generateParseIssuesMessage(dmlScript, parseIssues)); } dmlPgm = createDMLProgram(ast, sourceNamespace); return dmlPgm; }
From source file:org.apache.sysml.parser.python.PyDMLParserWrapper.java
License:Apache License
/** * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import' * @param fileName/* w ww .ja v a 2 s. c o m*/ * @return null if atleast one error */ public DMLProgram doParse(String fileName, String dmlScript, HashMap<String, String> argVals) throws ParseException { DMLProgram dmlPgm = null; ANTLRInputStream in; try { if (dmlScript == null) { dmlScript = DMLParserWrapper.readDMLScript(fileName); } InputStream stream = new ByteArrayInputStream(dmlScript.getBytes()); in = new org.antlr.v4.runtime.ANTLRInputStream(stream); } catch (FileNotFoundException e) { throw new ParseException("ERROR: Cannot find file:" + fileName, e); } catch (IOException e) { throw new ParseException("ERROR: Cannot open file:" + fileName, e); } catch (LanguageException e) { throw new ParseException("ERROR: " + e.getMessage(), e); } PmlprogramContext ast = null; CustomDmlErrorListener errorListener = new CustomDmlErrorListener(); try { PydmlLexer lexer = new PydmlLexer(in); CommonTokenStream tokens = new CommonTokenStream(lexer); PydmlParser antlr4Parser = new PydmlParser(tokens); boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value. if (tryOptimizedParsing) { // Try faster and simpler SLL antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL); antlr4Parser.removeErrorListeners(); antlr4Parser.setErrorHandler(new BailErrorStrategy()); try { ast = antlr4Parser.pmlprogram(); // If successful, no need to try out full LL(*) ... SLL was enough } catch (ParseCancellationException ex) { // Error occurred, so now try full LL(*) for better error messages tokens.reset(); antlr4Parser.reset(); if (fileName != null) { errorListener.pushCurrentFileName(fileName); } else { errorListener.pushCurrentFileName("MAIN_SCRIPT"); } // Set our custom error listener antlr4Parser.addErrorListener(errorListener); antlr4Parser.setErrorHandler(new DefaultErrorStrategy()); antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL); ast = antlr4Parser.pmlprogram(); } } else { // Set our custom error listener antlr4Parser.removeErrorListeners(); antlr4Parser.addErrorListener(errorListener); errorListener.pushCurrentFileName(fileName); // Now do the parsing ast = antlr4Parser.pmlprogram(); } } catch (Exception e) { throw new ParseException("ERROR: Cannot parse the program:" + fileName, e); } try { // Now convert the parse tree into DMLProgram // Do syntactic validation while converting ParseTree tree = ast; // And also do syntactic validation ParseTreeWalker walker = new ParseTreeWalker(); PydmlSyntacticValidatorHelper helper = new PydmlSyntacticValidatorHelper(errorListener); PydmlSyntacticValidator validator = new PydmlSyntacticValidator(helper, fileName, argVals); walker.walk(validator, tree); errorListener.popFileName(); if (errorListener.isAtleastOneError()) { return null; } dmlPgm = createDMLProgram(ast); } catch (Exception e) { throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram" + e.getMessage(), e); } return dmlPgm; }