List of usage examples for org.antlr.v4.runtime CommonTokenStream CommonTokenStream
public CommonTokenStream(TokenSource tokenSource)
From source file:boa.compiler.BoaCompiler.java
License:Apache License
public static void main(final String[] args) throws IOException { CommandLine cl = processCommandLineOptions(args); if (cl == null) return;/*from w ww . java 2 s . c o m*/ final ArrayList<File> inputFiles = BoaCompiler.inputFiles; // get the name of the generated class final String className = getGeneratedClass(cl); // get the filename of the jar we will be writing final String jarName; if (cl.hasOption('o')) jarName = cl.getOptionValue('o'); else jarName = className + ".jar"; // make the output directory final File outputRoot = new File(new File(System.getProperty("java.io.tmpdir")), UUID.randomUUID().toString()); final File outputSrcDir = new File(outputRoot, "boa"); if (!outputSrcDir.mkdirs()) throw new IOException("unable to mkdir " + outputSrcDir); // find custom libs to load final List<URL> libs = new ArrayList<URL>(); if (cl.hasOption('l')) for (final String lib : cl.getOptionValues('l')) libs.add(new File(lib).toURI().toURL()); final File outputFile = new File(outputSrcDir, className + ".java"); final BufferedOutputStream o = new BufferedOutputStream(new FileOutputStream(outputFile)); try { final List<String> jobnames = new ArrayList<String>(); final List<String> jobs = new ArrayList<String>(); boolean isSimple = true; final List<Program> visitorPrograms = new ArrayList<Program>(); SymbolTable.initialize(libs); for (int i = 0; i < inputFiles.size(); i++) { final File f = inputFiles.get(i); try { final BoaLexer lexer = new BoaLexer(new ANTLRFileStream(f.getAbsolutePath())); lexer.removeErrorListeners(); lexer.addErrorListener(new LexerErrorListener()); final CommonTokenStream tokens = new CommonTokenStream(lexer); final BoaParser parser = new BoaParser(tokens); parser.removeErrorListeners(); parser.addErrorListener(new BaseErrorListener() { @Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) throws ParseCancellationException { throw new ParseCancellationException(e); } }); final BoaErrorListener parserErrorListener = new ParserErrorListener(); final Start p = parse(tokens, parser, parserErrorListener); if (cl.hasOption("ast")) new ASTPrintingVisitor().start(p); final String jobName = "" + i; try { if (!parserErrorListener.hasError) { new TypeCheckingVisitor().start(p, new SymbolTable()); final TaskClassifyingVisitor simpleVisitor = new TaskClassifyingVisitor(); simpleVisitor.start(p); LOG.info(f.getName() + ": task complexity: " + (!simpleVisitor.isComplex() ? "simple" : "complex")); isSimple &= !simpleVisitor.isComplex(); new InheritedAttributeTransformer().start(p); new LocalAggregationTransformer().start(p); // if a job has no visitor, let it have its own method // also let jobs have own methods if visitor merging is disabled if (!simpleVisitor.isComplex() || cl.hasOption("nv") || inputFiles.size() == 1) { new VisitorOptimizingTransformer().start(p); if (cl.hasOption("pp")) new PrettyPrintVisitor().start(p); if (cl.hasOption("ast")) new ASTPrintingVisitor().start(p); final CodeGeneratingVisitor cg = new CodeGeneratingVisitor(jobName); cg.start(p); jobs.add(cg.getCode()); jobnames.add(jobName); } // if a job has visitors, fuse them all together into a single program else { p.getProgram().jobName = jobName; visitorPrograms.add(p.getProgram()); } } } catch (final TypeCheckException e) { parserErrorListener.error("typecheck", lexer, null, e.n.beginLine, e.n.beginColumn, e.n2.endColumn - e.n.beginColumn + 1, e.getMessage(), e); } } catch (final Exception e) { System.err.print(f.getName() + ": compilation failed: "); e.printStackTrace(); } } final int maxVisitors; if (cl.hasOption('v')) maxVisitors = Integer.parseInt(cl.getOptionValue('v')); else maxVisitors = Integer.MAX_VALUE; if (!visitorPrograms.isEmpty()) try { for (final Program p : new VisitorMergingTransformer().mergePrograms(visitorPrograms, maxVisitors)) { new VisitorOptimizingTransformer().start(p); if (cl.hasOption("pp")) new PrettyPrintVisitor().start(p); if (cl.hasOption("ast")) new ASTPrintingVisitor().start(p); final CodeGeneratingVisitor cg = new CodeGeneratingVisitor(p.jobName); cg.start(p); jobs.add(cg.getCode()); jobnames.add(p.jobName); } } catch (final Exception e) { System.err.println("error fusing visitors - falling back: " + e); e.printStackTrace(); for (final Program p : visitorPrograms) { new VisitorOptimizingTransformer().start(p); if (cl.hasOption("pp")) new PrettyPrintVisitor().start(p); if (cl.hasOption("ast")) new ASTPrintingVisitor().start(p); final CodeGeneratingVisitor cg = new CodeGeneratingVisitor(p.jobName); cg.start(p); jobs.add(cg.getCode()); jobnames.add(p.jobName); } } if (jobs.size() == 0) throw new RuntimeException("no files compiled without error"); final ST st = AbstractCodeGeneratingVisitor.stg.getInstanceOf("Program"); st.add("name", className); st.add("numreducers", inputFiles.size()); st.add("jobs", jobs); st.add("jobnames", jobnames); st.add("combineTables", CodeGeneratingVisitor.combineAggregatorStrings); st.add("reduceTables", CodeGeneratingVisitor.reduceAggregatorStrings); st.add("splitsize", isSimple ? 64 * 1024 * 1024 : 10 * 1024 * 1024); o.write(st.render().getBytes()); } finally { o.close(); } compileGeneratedSrc(cl, jarName, outputRoot, outputFile); }
From source file:boa.compiler.BoaCompiler.java
License:Apache License
public static void parseOnly(final String[] args) throws IOException { CommandLine cl = processParseCommandLineOptions(args); if (cl == null) return;/*from ww w .ja v a 2 s .co m*/ final ArrayList<File> inputFiles = BoaCompiler.inputFiles; // find custom libs to load final List<URL> libs = new ArrayList<URL>(); if (cl.hasOption('l')) for (final String lib : cl.getOptionValues('l')) libs.add(new File(lib).toURI().toURL()); final List<String> jobnames = new ArrayList<String>(); final List<String> jobs = new ArrayList<String>(); boolean isSimple = true; final List<Program> visitorPrograms = new ArrayList<Program>(); SymbolTable.initialize(libs); for (int i = 0; i < inputFiles.size(); i++) { final File f = inputFiles.get(i); try { final BoaLexer lexer = new BoaLexer(new ANTLRFileStream(f.getAbsolutePath())); lexer.removeErrorListeners(); lexer.addErrorListener(new LexerErrorListener()); final CommonTokenStream tokens = new CommonTokenStream(lexer); final BoaParser parser = new BoaParser(tokens); parser.removeErrorListeners(); parser.addErrorListener(new BaseErrorListener() { @Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) throws ParseCancellationException { throw new ParseCancellationException(e); } }); final BoaErrorListener parserErrorListener = new ParserErrorListener(); final Start p = parse(tokens, parser, parserErrorListener); if (cl.hasOption("ast")) new ASTPrintingVisitor().start(p); final String jobName = "" + i; try { if (!parserErrorListener.hasError) { new TypeCheckingVisitor().start(p, new SymbolTable()); final TaskClassifyingVisitor simpleVisitor = new TaskClassifyingVisitor(); simpleVisitor.start(p); LOG.info(f.getName() + ": task complexity: " + (!simpleVisitor.isComplex() ? "simple" : "complex")); isSimple &= !simpleVisitor.isComplex(); new InheritedAttributeTransformer().start(p); new LocalAggregationTransformer().start(p); // if a job has no visitor, let it have its own method // also let jobs have own methods if visitor merging is disabled if (!simpleVisitor.isComplex() || cl.hasOption("nv") || inputFiles.size() == 1) { new VisitorOptimizingTransformer().start(p); if (cl.hasOption("pp")) new PrettyPrintVisitor().start(p); if (cl.hasOption("ast")) new ASTPrintingVisitor().start(p); final CodeGeneratingVisitor cg = new CodeGeneratingVisitor(jobName); cg.start(p); jobs.add(cg.getCode()); jobnames.add(jobName); } // if a job has visitors, fuse them all together into a single program else { p.getProgram().jobName = jobName; visitorPrograms.add(p.getProgram()); } } } catch (final TypeCheckException e) { parserErrorListener.error("typecheck", lexer, null, e.n.beginLine, e.n.beginColumn, e.n2.endColumn - e.n.beginColumn + 1, e.getMessage(), e); } } catch (final Exception e) { System.err.print(f.getName() + ": parsing failed: "); e.printStackTrace(); } } }
From source file:br.beholder.memelang.model.executor.MemeLanguageCompiler.java
public void realizarCompilacao(String txt) { this.ais = new ANTLRInputStream(txt); this.lexer = new MemelangLexer(ais); this.stream = new CommonTokenStream(lexer); this.parser = new MemelangParser(stream); this.erroLexico = new ErroLexico(); this.lexer.removeErrorListeners(); this.parser.removeErrorListeners(); this.lexer.addErrorListener(erroLexico); this.parser.addErrorListener(erroLexico); this.warnings.clear(); this.tree = parser.prog(); if (erroLexico.getErrors().isEmpty()) { SemanticVisitor semantic = new SemanticVisitor(new ArrayList<Identificador>()); try {//ww w . ja va 2 s . c o m semantic.visit(tree); ids = semantic.getTabelaSimbolos(); if (!semantic.getSemanticWarnings().isEmpty()) { for (String war : semantic.getSemanticWarnings()) { this.warnings.add(war); } } model = getModel(ids); if (!semantic.getSemanticErrors().isEmpty()) { for (ParseCancellationException err : semantic.getSemanticErrors()) { erroLexico.getErrors().add(err.getMessage()); } } else { BipGeneratorVisitor generatorVisitor = new BipGeneratorVisitor(ids); generatorVisitor.visit(tree); codigoGerado = generatorVisitor.getCodigo(); // System.out.println(codigoGerado); } } catch (Exception e) { ids = semantic.getTabelaSimbolos(); model = getModel(ids); erroLexico.getErrors().add(e.getLocalizedMessage()); Logger.getLogger(Application.class.getName()).log(Level.SEVERE, null, e); } } }
From source file:br.edu.ifrn.potigol.Principal.java
License:Open Source License
private static Listener getListner(final String arq) throws IOException { String r = lerArquivo(arq);//from w w w . j av a 2 s . c o m final ANTLRInputStream input = new ANTLRInputStream(r); final potigolLexer lexer = new potigolLexer(input); final CommonTokenStream tokens = new CommonTokenStream(lexer); final potigolParser parser = new potigolParser(tokens); final ParseTree tree = parser.prog(); final ParseTreeWalker walker = new ParseTreeWalker(); final Listener listener = new Listener(); try { // if (tree.toString() != "[]") walker.walk(listener, tree); } catch (Exception e) { } return listener; }
From source file:br.unicamp.cst.util.SimulateConfiguration.java
License:Open Source License
private void jButtonApplyActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_jButtonApplyActionPerformed // TODO add your handling code here: String s = jTextAreaComand.getText(); //String s = jTextAreaComand.getText(); jTextAreaComand.setText(null);/*from w w w . j av a 2s . c o m*/ // jTextAreaComand.setText(null); s = s + "\n"; /** * ******************** to Parse ******************* */ ANTLRInputStream input = new ANTLRInputStream(s); // create a lexer that feeds off of input CharStream OwrlLexer lexer = new OwrlLexer(input); // create a buffer of tokens pulled from the lexer CommonTokenStream tokens = new CommonTokenStream(lexer); // create a parser that feeds off the tokens buffer OwrlParser parser = new OwrlParser(tokens); ParseTree tree = parser.conf(); // begin parsing at init rule // Walk it and attach our listener ParseTreeWalker walker = new ParseTreeWalker(); CustomizedListener listener = new CustomizedListener(); for (int i = 1; i < (tree.getChildCount() - 2); i++) { walker.walk(listener, tree.getChild(i)); } List<Pair<String, List<AbstractObject>>> memory = listener.getMemory(); // // if (!configurations.applyCommands(memory)) { // jTextAreaComand.setText(jTextAreaComand.getText().concat("\nErro: Falha ao aplicar instrues. Verifique seu comando.\n")); // } buildTree(); }
From source file:ca.nines.ise.dom.DOMBuilder.java
License:Open Source License
/** * Fire off the full parse of the input and return the resulting DOM object. * Any errors encountered during the parse will be logged in the Log class. * <p>// ww w.j ava 2 s. c o m * @see Log * <p> * @return the DOM built by parsing the string or file. */ @Override public DOM build() { ParserErrorListener parserListener = new ParserErrorListener(dom); ISELexer lexer = new ISELexer(ais); lexer.removeErrorListeners(); lexer.addErrorListener(parserListener); CommonTokenStream tokenStream = new CommonTokenStream(lexer); ISEParser parser = new ISEParser(tokenStream); parser.removeErrorListeners(); parser.addErrorListener(parserListener); ParseTreeWalker ptw = new ParseTreeWalker(); tokens = parser.getTokenStream(); ParseTree pt = parser.document(); ptw.walk(this, pt); dom.index(); return dom; }
From source file:carbon.compiler.CarbonCompiler.java
License:BSD License
public List<ParseTree> getTrees() { List<ParseTree> roots = new ArrayList<ParseTree>(); for (String fileName : targetFiles) { try {/*from www .j a v a2s . co m*/ File file = new File(fileName); if (!file.isAbsolute()) { file = new File(System.getProperty("user.dir"), fileName); } ANTLRFileStream input = new ANTLRFileStream(file.getAbsolutePath()); CarbonLexer lexer = new CarbonLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); CarbonParser parser = new CarbonParser(tokens); roots.add(parser.file()); } catch (IOException ioe) { //errorManager.toolError(ErrorKind.CANNOT_OPEN_FILE, ""); throw new RuntimeException(ioe); } } return roots; }
From source file:cdql.cqe.ContextQueryExecutor.java
@Override public Graph execute(String query) throws CDQLSyntaxtErrorException { StringBuilder res = new StringBuilder(); ANTLRInputStream input = new ANTLRInputStream(query); // create a lexer that feeds off of input CharStream CDQLLexer lexer = new CDQLLexer(input); // create a buffer of tokens pulled from the lexer CommonTokenStream tokens = new CommonTokenStream(lexer); // create a parser that feeds off the tokens buffer CDQLParser parser = new CDQLParser(tokens); ParseTree tree = parser.cdql(); // begin parsing at init rule if (parser.getNumberOfSyntaxErrors() != 0) { throw new CDQLSyntaxtErrorException("syntaxt error"); }// www . j av a 2 s .co m CDQLVisitor stringCDQLBaseVisitor = new CDQLVisitor(); // Walk the tree created during the parse, trigger callbacks res.append(tree.toStringTree(parser)); stringCDQLBaseVisitor.visit(tree); String error = ""; for (String s : stringCDQLBaseVisitor.getUsedContextEntities()) { if (!stringCDQLBaseVisitor.getDefinedEntities().containsKey(s)) { error += ("\n" + s + " not defined"); } } if (!error.isEmpty()) { throw new CDQLSyntaxtErrorException(error); } error = ""; for (String s : stringCDQLBaseVisitor.getCalledFunctions()) { if (!stringCDQLBaseVisitor.getDefinedFunctions().containsKey(s)) { error += ("\n" + s + " not defined"); } else if (stringCDQLBaseVisitor.getDefinedFunctions().get(s) instanceof AggregationFunction) { //stringCDQLBaseVisitor.getDefinedFunctions().get(s) } } if (!error.isEmpty()) { throw new CDQLSyntaxtErrorException(error); } error = ""; Map<String, Vertex> parsedEntites = new HashMap<>(); List<ContextEntity> executionPlan = new ArrayList<>(); Graph graph = new Graph(); boolean flag = true; while (flag) { flag = false; for (Map.Entry<String, ContextEntity> entry : stringCDQLBaseVisitor.getDefinedEntities().entrySet()) { ContextEntity value = entry.getValue(); if (value.getCondition() == null || value.getCondition().getDependency() == null || value.getCondition().getDependency().isEmpty()) { if (!parsedEntites.containsKey(entry.getKey())) { Vertex vertex = new Vertex(value); graph.addVertex(vertex, false); executionPlan.add(value); parsedEntites.put(entry.getKey(), vertex); res.append("\n" + entry.getKey()); flag = true; } } else { boolean flag2 = true; Vertex vertex = new Vertex(value); List<Edge> edges = new ArrayList<>(); for (Map.Entry<String, List<String>> s : value.getCondition().getDependency().entrySet()) { if (!parsedEntites.containsKey(s.getKey())) { flag2 = false; break; } else { edges.add(new Edge(vertex, parsedEntites.get(s.getKey()))); } } if (flag2) { if (!parsedEntites.containsKey(entry.getKey())) { graph.addVertex(vertex, false); for (Edge e : edges) { graph.addEdge(e.getOne(), e.getTwo()); } executionPlan.add(value); parsedEntites.put(entry.getKey(), vertex); flag = true; res.append("\n" + entry.getKey()); } } } } } if (parsedEntites.size() != stringCDQLBaseVisitor.getDefinedEntities().size()) { throw new CDQLSyntaxtErrorException("the relationship between entites cannot be solved"); } Query q = new Query(); q.setQuery(query); q.setQuerystatus("IN_PROGRESS"); q = em.merge(q); em.flush(); try { for (ContextEntity e : executionPlan) { switch (e.getType()) { case DEVICE: String where = ""; if (e.getCondition() != null && !e.getCondition().getRPNCondition().isEmpty()) { where = " where " + e.getCondition().toJPAWhereClause(); } List<Device> result = em .createQuery("SELECT " + e.getEntityID() + " from Device as " + e.getEntityID() + where) .getResultList(); HashSet<String> requestedEntities = stringCDQLBaseVisitor.getDefinedContextAttributes() .get(e.getEntityID()); JSONArray requesteContext = new JSONArray(); ObjectsType objectsType = new ObjectsType(); Map<String, ObjectType> objs = new HashMap<>(); for (String re : requestedEntities) { try { String replaceFirst = re.replaceFirst("\\.", ""); requesteContext.put(replaceFirst); String[] split = replaceFirst.split("\\."); InfoItemType infoItem = new InfoItemType(); if (split.length < 2) { infoItem = null; } else { infoItem.setNameAttribute(split[1]); } if (objs.containsKey(split[0]) && infoItem != null) { objs.get(split[0]).getInfoItem().add(infoItem); } else { ObjectType obj = new ObjectType(); QlmID qlmID = new QlmID(); qlmID.setValue(split[0]); obj.getId().add(qlmID); if (infoItem != null) { obj.getInfoItem().add(infoItem); } objs.put(split[0], obj); } } catch (Exception es) { } } for (Map.Entry<String, ObjectType> obj : objs.entrySet()) { objectsType.getObject().add(obj.getValue()); } String ipAddress = ""; try { InetAddress find = IpAddressFinder.find(); ipAddress = find.getHostAddress(); } catch (UnknownHostException ex) { Logger.getLogger(ContextQueryExecutor.class.getName()).log(Level.SEVERE, null, ex); } OmiEnvelope createReadRequestEnvelope = OmiFactory.createReadRequestEnvelope(100, objectsType, "v1", "http://" + ipAddress + ":59646/CoaaS-war/webresources/generic"); IdType idType = new IdType(); idType.setFormat("db"); idType.setValue(q.getQueryid().toString()); createReadRequestEnvelope.getRead().getRequestID().add(idType); Gson gson = new Gson(); String toJson = gson.toJson(createReadRequestEnvelope); JSONObject msg = null; try { msg = new JSONObject(toJson); } catch (JSONException ex) { Logger.getLogger(ContextQueryExecutor.class.getName()).log(Level.SEVERE, null, ex); } // infoItem.set for (Device d : result) { try { FCM.pushFCMNotification(d.getFcmid(), msg); } catch (Exception ex) { Logger.getLogger(ContextQueryExecutor.class.getName()).log(Level.SEVERE, null, ex); } } break; case PERSON: break; } } } catch (Exception e) { } return graph; }
From source file:ch.raffael.contracts.devtools.ast.AstInspector.java
License:Apache License
public void parse(final String source) { final StringBuilder errors = new StringBuilder(); final StringBuilder trace = new StringBuilder("Log:\n"); ANTLRErrorListener errorListener = new BaseErrorListener() { @Override//from w w w . ja v a2s. c om public void syntaxError(Recognizer<?, ?> recognizer, @Nullable Object offendingSymbol, int line, int charPositionInLine, String msg, @Nullable RecognitionException e) { errors.append("Line ").append(line).append(':').append(charPositionInLine).append(": ").append(msg) .append('\n'); } }; final CelLexer lexer = new CelLexer(new ANTLRInputStream(source)); final CelParser parser = new CelParser(new CommonTokenStream(lexer)); lexer.addErrorListener(errorListener); parser.addErrorListener(errorListener); parser.addParseListener(new ParseTreeListener() { @Override public void enterEveryRule(ParserRuleContext ctx) { trace.append(" enter\t").append(parser.getRuleNames()[ctx.getRuleIndex()]).append(", LT(1)=") .append(parser.getTokenStream().LT(1).getText()).append('\n'); } @Override public void visitTerminal(TerminalNode node) { trace.append(" consume\t").append(node.getSymbol()).append(" rule ") .append(parser.getRuleNames()[parser.getContext().getRuleIndex()]).append('\n'); } @Override public void visitErrorNode(ErrorNode node) { } @Override public void exitEveryRule(ParserRuleContext ctx) { trace.append(" exit\t").append(parser.getRuleNames()[ctx.getRuleIndex()]).append(", LT(1)=") .append(parser.getTokenStream().LT(1).getText()).append('\n'); } }); parser.setBuildParseTree(true); AstBuilder builder = new AstBuilder(); builder.install(parser); final CelParser.ClauseContext rootContext = parser.clause(); if (errors.length() != 0) { errors.append('\n'); } Runnable guiUpdate = new Runnable() { @Override public void run() { log.setText(errors.toString() + trace.toString()); log.setSelectionStart(0); log.setSelectionEnd(0); if (rootContext == null || rootContext.node == null) { syntaxTree.setModel(new DefaultTreeModel(AstTreeNode.empty())); updateParseTree(null); } else { syntaxTree.setModel(new DefaultTreeModel(AstTreeNode.from(rootContext.node))); updateParseTree(new TreeViewer(Arrays.asList(parser.getRuleNames()), rootContext)); } for (int i = 0; i < syntaxTree.getRowCount(); i++) { syntaxTree.expandRow(i); } } }; if (SwingUtilities.isEventDispatchThread()) { guiUpdate.run(); } else { SwingUtilities.invokeLater(guiUpdate); } }
From source file:ch.raffael.contracts.processor.cel.Compiler.java
License:Apache License
public Clause parse() throws ParseException { ANTLRErrorListener errorListener = new BaseErrorListener() { @Override/* ww w . j a v a 2 s . c o m*/ public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { addANTLRError(e, msg); } }; CelLexer lexer = new CelLexer(new ANTLRInputStream(expression)); lexer.addErrorListener(errorListener); CelParser parser = new CelParser(new CommonTokenStream(lexer)); parser.addErrorListener(errorListener); try { Clause ast = new AstBuilder().install(parser).clause().node; if (ast == null) { if (errors.isEmpty()) { throw new IllegalStateException("No AST returned, but no errors reported"); } throw new ParseException(errors); } return ast; } catch (RecognitionException e) { // should not happen addANTLRError(e, "Unexpected: " + e.toString()); throw new ParseException(errors); } }