Example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream.

Prototype

public CommonTokenStream(TokenSource tokenSource) 

Source Link

Document

Constructs a new CommonTokenStream using the specified token source and the default token channel ( Token#DEFAULT_CHANNEL ).

Usage

From source file:com.twosigma.beakerx.javash.autocomplete.JavaAutocomplete.java

License:Apache License

private AutocompleteResult find(String txt, int cur, ClassLoader l, Imports imports) {
    registry = AutocompleteRegistryFactory.createRegistry(cps);
    ClassUtils cu = createClassUtils(l);
    setup(cu, registry);/*  w  w  w  .  j  av  a  2s  .  c  o  m*/
    AutocompleteRegistryFactory.addDefaultImports(cu, registry, imports.toListOfStrings(), cps);

    Lexer lexer = new JavaLexer(new ANTLRInputStream(txt));
    CommonTokenStream tokens = new CommonTokenStream(lexer);

    // Create a parser that reads from the scanner
    JavaParser parser = new JavaParser(tokens);
    parser.removeErrorListeners();

    // start parsing at the compilationUnit rule
    ParserRuleContext t = parser.compilationUnit();
    ParseTreeWalker walker = new ParseTreeWalker();
    List<AutocompleteCandidate> q = new ArrayList<AutocompleteCandidate>();

    JavaImportDeclarationCompletion extractor = new JavaImportDeclarationCompletion(txt, cur, registry, cps,
            cu);
    JavaNameBuilder extractor2 = new JavaNameBuilder(registry, cu);
    JavaNodeCompletion extractor3 = new JavaNodeCompletion(txt, cur, registry, cu);
    walker.walk(extractor, t);
    if (extractor.getQuery() != null)
        q.addAll(extractor.getQuery());
    walker.walk(extractor2, t);
    walker.walk(extractor3, t);
    if (extractor3.getQuery() != null)
        q.addAll(extractor3.getQuery());
    List<String> ret = registry.searchCandidates(q);

    if (!ret.isEmpty()) {
        return new AutocompleteResult(ret, getStartIndex(extractor, extractor2, extractor3));
    }
    return findAutocompleteResult(txt, cur, cu);
}

---

From source file:com.wandrell.tabletop.dice.parser.DefaultDiceNotationParser.java

License:Apache License

/**
 * Creates the ANTLR4 parser to be used for processing the dice expression.
 * <p>/*from  w ww. j ava2  s  .  com*/
 * This parser will be tailored to the received expression, but it will
 * still need a listener which, using the visitor pattern, will create the
 * final object.
 * 
 * @param expression
 *            expression used to generate the parser
 * @return an ANTLR4 parser tailored for the expression
 */
private final DiceNotationGrammarParser buildDiceNotationParser(final String expression) {
    final ANTLRInputStream input; // Input stream for the expression
    final DiceNotationGrammarLexer lexer; // Lexer for the expression tokens
    final TokenStream tokens; // Expression tokens

    input = new ANTLRInputStream(expression);
    lexer = new DiceNotationGrammarLexer(input);
    tokens = new CommonTokenStream(lexer);

    return new DiceNotationGrammarParser(tokens);
}

---

From source file:com.wandrell.tabletop.dice.test.integration.generated.grammar.ITDiceGrammar.java

License:Apache License

/**
 * Builds a grammar parser for the specified notation.
 * /*from w  w w . j a  va  2s .c  o m*/
 * @param notation
 *            notation to parse
 * @return a parse tailored the the notation
 */
private final DiceNotationGrammarParser getParser(final String notation) {
    final CharStream in; // Characters stream
    final DiceNotationGrammarLexer lexer; // Lexer
    final TokenStream tokens; // Lexical tokens
    final DiceNotationGrammarParser parser; // Parser

    in = new ANTLRInputStream(notation);
    lexer = new DiceNotationGrammarLexer(in);
    tokens = new CommonTokenStream(lexer);

    parser = new DiceNotationGrammarParser(tokens);

    parser.addErrorListener(new DefaultErrorListener());

    return parser;
}

---

From source file:com.wordpress.thesolidsnake.bahasaku.Main.java

License:Apache License

public static void main(String[] args) {
    if (args.length == 0) {
        System.out.println("Tool ini dipakai untuk menjalankan program BahasaKu\n\n" + "Contoh penggunaan:\n"
                + "bahasaku namafilesource.baku\n" + "bahasaku source1.baku source2.baku\n"
                + "bahasaku -trace namafilesource.baku\n");
        return;/*from w w  w .  j a  va 2 s.co m*/
    }

    boolean enableTrace = false;
    List<String> sourceFiles = new ArrayList<>();

    // Melakukan parsing argumen
    // Dapat melakukan kompilasi lebih dari satu file
    for (String arg : args) {
        if ("-trace".equals(arg)) {
            enableTrace = true;
        } else {
            sourceFiles.add(arg);
        }
    }

    // Memulai proses kompilasi
    for (String sourceFile : sourceFiles) {
        Path file = Paths.get(sourceFile);
        if (!file.toFile().exists()) {
            System.out.println("Tidak dapat menemukan file: [" + file + "]");
            continue;
        }

        try {

            // Front-end
            ANTLRInputStream antlrInputStream = new ANTLRInputStream(Files.newInputStream(file));
            BahasaKuLexer lexer = new BahasaKuLexer(antlrInputStream);
            CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
            BahasaKuParser parser = new BahasaKuParser(commonTokenStream);
            ParseTree parseTree = parser.file();

            // Back-end
            BahasaKuVisitor jvmVisitor = new BahasaKuVisitor(file.toFile(), enableTrace);
            jvmVisitor.visit(parseTree);
            jvmVisitor.generateOutput();

            // Menjalankan Program
            String namaFileLengkap = file.toFile().getName();
            String namaClass = namaFileLengkap.substring(0, namaFileLengkap.lastIndexOf('.'));
            ProcessBuilder pb = new ProcessBuilder(System.getProperty("java.home") + "/bin/java", "-classpath",
                    System.getProperty("java.class.path") + File.pathSeparatorChar
                            + file.getParent().toAbsolutePath().toFile().toString(),
                    namaClass);
            if (enableTrace) {
                System.out.println("Mengerjakan " + pb.command());
            }
            pb.inheritIO();
            pb.start().waitFor();

        } catch (Exception ex) {
            System.out.println("Terjadi kesalahan [" + ex.getMessage() + "]");
        }
    }

}

---

From source file:com.xiaomi.linden.bql.BQLCompiler.java

License:Apache License

public LindenRequest compile(String bqlStmt) {
    // Lexer splits input into tokens
    ANTLRInputStream input = new ANTLRInputStream(bqlStmt);
    BQLLexer lexer = new BQLLexer(input);
    lexer.removeErrorListeners();//from  w w  w.j  a v a  2s  .  co  m
    TokenStream tokens = new CommonTokenStream(lexer);

    // Parser generates abstract syntax tree
    BQLParser parser = new BQLParser(tokens);
    parser.removeErrorListeners();
    BQLCompilerAnalyzer analyzer = new BQLCompilerAnalyzer(parser, lindenSchema);

    /*You can save a great deal of time on correct inputs by using a two-stage parsing strategy.
      1. Attempt to parse the input using BailErrorStrategy and PredictionMode.SLL.
         If no exception is thrown, you know the answer is correct.
      2. If a ParseCancellationException is thrown, retry the parse using the
         default settings (DefaultErrorStrategy and PredictionMode.LL).
    */
    parser.setErrorHandler(new BailErrorStrategy());
    parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
    try {
        BQLParser.StatementContext ret = parser.statement();
        ParseTreeWalker.DEFAULT.walk(analyzer, ret);
        return analyzer.getLindenRequest(ret);
    } catch (ParseCancellationException e) {
        try {
            parser.reset();
            parser.getInterpreter().setPredictionMode(PredictionMode.LL);
            BQLParser.StatementContext ret = parser.statement();
            ParseTreeWalker.DEFAULT.walk(analyzer, ret);
            return analyzer.getLindenRequest(ret);
        } catch (Exception e2) {
            throw new RuntimeException(getErrorMessage(e2));
        }
    } catch (Exception e) {
        throw new RuntimeException(getErrorMessage(e));
    }
}

---

From source file:com.xylocore.copybook.generator.parser.CopybookLexerTestApp.java

License:Apache License

/**
 * FILLIN//from w  ww .  java  2s  .  c  o m
 * 
 * @param       args
 */
public static void main(String[] args) {
    try {
        FileInputStream myIS = new FileInputStream(args[0]);
        CopybookInputStream myCIS = new CopybookInputStream(myIS, 72);
        ANTLRInputStream myInput = new ANTLRInputStream(myCIS);
        CopybookLexer myLexer = new CopybookLexer(myInput);

        CommonTokenStream myTokenStream = new CommonTokenStream(myLexer);
        CopybookParser myParser = new CopybookParser(myTokenStream);

        myParser.copybook();

        //            while ( (myToken = myLexer.nextToken()).getType() != CopybookLexer.EOF )
        //            {
        //                System.out.println( myToken.toString() );
        //            }
    } catch (Exception myException) {
        myException.printStackTrace();
    }
}

---

From source file:com.xylocore.copybook.generator.parser.CopybookProcessor.java

License:Apache License

/**
 * FILLIN//ww w.  j  a v a 2s.  c  o  m
 * 
 * @param       aEnvironment
 * 
 * @return
 */
public Copybook process(Environment aEnvironment) {
    try (InputStream myInputStream = openCopybookSource(aEnvironment)) {
        Metadata myMetadata = aEnvironment.getMetadata();
        CopybookInputStream myCIS = new CopybookInputStream(myInputStream, myMetadata.getRightMarginLimit());
        ANTLRInputStream myInput = new ANTLRInputStream(myCIS);
        CopybookLexer myLexer = new CopybookLexer(myInput);
        CommonTokenStream myTokenStream = new CommonTokenStream(myLexer);
        CopybookParser myParser = new CopybookParser(myTokenStream);
        ParseTree myParseTree = myParser.copybook();
        ParseTreeWalker myWalker = new ParseTreeWalker();
        CopybookTransformationListener myListener = new CopybookTransformationListener(aEnvironment);

        myWalker.walk(myListener, myParseTree);

        return myListener.getCopybook();
    } catch (IOException myIOException) {
        // TODO: implement better exception processing
        throw new RuntimeException(myIOException);
    }
}

---

From source file:com.yahoo.elide.core.EntityPermissions.java

License:Apache License

private ParseTree parseExpression(String expression) {
    ANTLRInputStream is = new ANTLRInputStream(expression);
    ExpressionLexer lexer = new ExpressionLexer(is);
    lexer.removeErrorListeners();/*from   w  w  w . j  av a2 s  .  c o  m*/
    lexer.addErrorListener(new BaseErrorListener() {
        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
                int charPositionInLine, String msg, RecognitionException e) {
            throw new ParseCancellationException(msg, e);
        }
    });
    ExpressionParser parser = new ExpressionParser(new CommonTokenStream(lexer));
    lexer.reset();
    return parser.start();
}

---

From source file:com.yahoo.elide.Elide.java

License:Apache License

/**
 * Compile request to AST./* w  w w.j  a v  a2 s  .  c  o  m*/
 *
 * @param path request
 * @return AST parse tree
 */
public static ParseTree parse(String path) {
    String normalizedPath = Paths.get(path).normalize().toString().replace(File.separatorChar, '/');
    if (normalizedPath.startsWith("/")) {
        normalizedPath = normalizedPath.substring(1);
    }
    ANTLRInputStream is = new ANTLRInputStream(normalizedPath);
    CoreLexer lexer = new CoreLexer(is);
    lexer.removeErrorListeners();
    lexer.addErrorListener(new BaseErrorListener() {
        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
                int charPositionInLine, String msg, RecognitionException e) {
            throw new ParseCancellationException(msg, e);
        }
    });
    CoreParser parser = new CoreParser(new CommonTokenStream(lexer));
    parser.setErrorHandler(new BailErrorStrategy());
    return parser.start();
}

---

From source file:com.yahoo.elide.parsers.JsonApiParser.java

License:Apache License

/**
 * Compile request to AST.//from   www  .  ja  v  a 2 s  . c o m
 * @param path request
 * @return AST
 */
public static ParseTree parse(String path) {
    ANTLRInputStream is = new ANTLRInputStream(path);
    CoreLexer lexer = new CoreLexer(is);
    lexer.removeErrorListeners();
    lexer.addErrorListener(new BaseErrorListener() {
        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
                int charPositionInLine, String msg, RecognitionException e) {
            throw new ParseCancellationException(e);
        }
    });
    CoreParser parser = new CoreParser(new CommonTokenStream(lexer));
    parser.setErrorHandler(new BailErrorStrategy());
    return parser.start();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »