Example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream.

Prototype

public CommonTokenStream(TokenSource tokenSource) 

Source Link

Document

Constructs a new CommonTokenStream using the specified token source and the default token channel ( Token#DEFAULT_CHANNEL ).

Usage

From source file:net.sf.jame.contextfree.parser.ContextFreeParser.java

License:Open Source License

public ContextFreeConfig parseConfig(File baseDir, Reader reader) throws ContextFreeParserException {
    try {//www  .j  a v  a  2  s.c o m
        CharStream stream = new ANTLRInputStream(reader);
        CFDGLexer lexer = new CFDGLexer(stream);
        TokenStream tokenStream = new CommonTokenStream(lexer);
        CFDGParser parser = new CFDGParser(tokenStream);
        ContextFreeConfig config = new ContextFreeConfig();
        CFInterpreter interpreter = new CFInterpreter(baseDir, config);
        //         interpreter.inACfdg(parser.cfdg());
        //         parser.cfdg()parse().apply(interpreter);
        return config;
        //      } catch (ParserException e) {
        //         String line = null;
        //         try {
        //            BufferedReader reader2 = new BufferedReader(reader);
        //            reader.reset();
        //            int i = 0;
        //            while ((line = reader2.readLine()) != null) {
        //               if (i == e.getToken().getLine()) {
        //                  break;
        //               }
        //               i += 1;
        //            }
        //            System.out.println("[" + e.getToken().getLine() + "," +  e.getToken().getPos() + "] unexpected token " + e.getToken().getText() + " at line " + line);
        //         } catch (IOException x) {
        //            x.printStackTrace();
        //         }
        //         throw new ContextFreeParserException(e);
        //      } catch (LexerException e) {
        //         throw new ContextFreeParserException(e);
    } catch (IOException e) {
        throw new ContextFreeParserException(e);
    }
}

---

From source file:net.sourceforge.mavenhippo.parser.JavaAntlrParser.java

License:Apache License

@Override
public Map<String, HippoBeanClass> parse(File javaFile) {
    try {//from  w ww  .j a v  a2  s .  co m
        if (javaFile.exists()) {
            ANTLRInputStream input = new ANTLRFileStream(javaFile.getAbsolutePath());
            Java8Lexer lexer = new Java8Lexer(input);
            CommonTokenStream tokens = new CommonTokenStream(lexer);
            Java8Parser parser = new Java8Parser(tokens);
            ParserRuleContext tree = parser.compilationUnit();
            ParseTreeWalker walker = new ParseTreeWalker();
            walker.walk(this, tree);
        }
    } catch (Exception e) {
        log.warn(e.getMessage(), e);
    }
    return map;
}

---

From source file:net.team2xh.crt.gui.editor.EditorTextPane.java

License:Open Source License

private void highlightText() {

    getHighlighter().removeAllHighlights();
    colorize(COMMENT, 0, 0, getText().length());

    highlightCurrentLine();//from  w ww .  jav a  2 s. co  m

    CRTLexer lexer = new CRTLexer(new ANTLRInputStream(getText()));
    CRTParser parser = new CRTParser(new CommonTokenStream(lexer));
    ParserRuleContext tree = parser.script();
    ParseTreeWalker walker = new ParseTreeWalker();

    lexer = new CRTLexer(new ANTLRInputStream(getText()));

    for (Token token = lexer.nextToken(); token.getType() != Token.EOF; token = lexer.nextToken()) {
        int cursor = token.getCharPositionInLine();
        int line = token.getLine() - 1;
        int length = token.getText().length();

        switch (token.getType()) {
        case CRTLexer.ASSIGN:
        case CRTLexer.ATTRIBUTE:
        case CRTLexer.ADD:
        case CRTLexer.SUBTRACT:
        case CRTLexer.INTERSECTION:
        case CRTLexer.MULTIPLY:
        case CRTLexer.DIVIDE:
        case CRTLexer.MODULO:
        case CRTLexer.NOT:
        case CRTLexer.LESS:
        case CRTLexer.GREATER:
        case CRTLexer.LESS_EQUAL:
        case CRTLexer.GREATER_EQUAL:
        case CRTLexer.EQUAL:
        case CRTLexer.NOT_EQUAL:
        case CRTLexer.AND:
        case CRTLexer.OR:
        case CRTLexer.QUESTION:
        case CRTLexer.COLON:
            colorize(OPERATORS, line, cursor, length);
            break;

        case CRTLexer.TRANSLATE:
        case CRTLexer.SCALE:
        case CRTLexer.ROTATE:
            colorize(TRANSFORM, line, cursor, length);
            break;

        case CRTLexer.NAME:
            colorize(NAME, line, cursor, length);
            break;

        case CRTLexer.IDENTIFIER:
            colorize(IDENTIFIER, line, cursor, length);
            break;

        case CRTLexer.FLOAT:
        case CRTLexer.INTEGER:
        case CRTLexer.TRUE:
        case CRTLexer.FALSE:
            colorize(NUMBER, line, cursor, length);
            break;

        case CRTLexer.SCENE:
        case CRTLexer.SETTINGS:
        case CRTLexer.MACRO:
            colorize(BLOCK, line, cursor, length);
            break;

        case CRTLexer.STRING:
            colorize(STRING, line, cursor, length);
            break;

        // TODO: don't hide under line highlight
        case CRTLexer.INVALID:
            try {
                int index = getLineStartOffset(line) + cursor;
                int s = getLineStartOffset(index);
                int e = getLineEndOffset(index);
                highlight(cursor, cursor + length, eh);
                highlight(s, e, lh);
            } catch (BadLocationException e) {
            }
            break;

        default:
            colorize(NORMAL, line, cursor, length);
            break;
        }
    }

    // Reset lexer for errors because parser consumed it
    walker.walk(new SyntaxHighlightingListener(), tree);

}

---

From source file:net.team2xh.crt.language.compiler.Compiler.java

License:Open Source License

private static Script compile(String code, Compiler compiler) {
    CRTLexer lexer = new CRTLexer(new ANTLRInputStream(code));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    CRTParser parser = new CRTParser(tokens);
    ParseTree tree = parser.script();/*from   w  w  w.j a  v a2s  .  c  o  m*/

    compiler.visit(tree);

    return compiler.script;
}

---

From source file:net.udidb.expr.lang.c.CExpressionCompiler.java

License:Open Source License

private static CParser createParser(String expression) throws IOException {
    ANTLRInputStream inputStream = new ANTLRInputStream(
            new ByteArrayInputStream(expression.getBytes(StandardCharsets.UTF_8)));
    CLexer lexer = new CLexer(inputStream);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    return new CParser(tokens);
}

---

From source file:ngspipesengine.core.parsers.java.PipesJavaParser.java

License:Open Source License

public static StringBuilder getPipelineAsJava(String filePath, Log log)
        throws MalformedURLException, EngineException {

    log.debug(TAG, "Begin compiling pipes");

    PipelineLexer lexer = new PipelineLexer(new ANTLRInputStream(IO.readFile(filePath)));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    PipelineParser parser = new PipelineParser(tokens);

    PipelineParser.PipelineContext pipelineCtx;
    try {//ww w. j a va2s. c  o m
        pipelineCtx = parser.pipeline();
        log.debug(TAG, "End compiling pipes");
    } catch (RecognitionException e) {
        log.error(TAG, "pipes file is malformed");
        throw new EngineException("pipes file is malformed", e);
    }

    log.debug(TAG, "Parsing pipes to java");
    return getPipesContent(pipelineCtx);
}

---

From source file:ninja.leaping.permissionsex.util.glob.Globs.java

License:Apache License

private static GlobNode parse(ANTLRInputStream input) throws GlobParseException {
    GlobLexer lexer = new GlobLexer(input);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    GlobParser parser = new GlobParser(tokenStream);
    parser.setErrorHandler(new BailErrorStrategy());
    ParseTreeWalker walker = new ParseTreeWalker();
    GlobListener listener = new GlobListener();

    try {//from   w ww .  j a va 2s .c  o m
        walker.walk(listener, parser.rootGlob());
    } catch (ParseCancellationException e) {
        RecognitionException ex = ((RecognitionException) e.getCause());
        throw new GlobParseException("Unable to parse glob: Error at token " + ex.getOffendingToken().getText()
                + " at position " + ex.getOffendingToken().getLine() + ":"
                + ex.getOffendingToken().getCharPositionInLine(), ex);
    }

    return listener.popNode();
}

---

From source file:nl.basjes.parse.useragent.analyze.MatcherAction.java

License:Apache License

void init(String newMatchExpression, Matcher newMatcher) {
    this.matcher = newMatcher;
    this.matches = new ArrayList<>(16);
    this.isFixedString = false;
    this.matchExpression = newMatchExpression;
    setVerbose(newMatcher.getVerbose());

    InitErrorListener errorListener = new InitErrorListener();
    ANTLRInputStream input = new ANTLRInputStream(this.matchExpression);
    UserAgentTreeWalkerLexer lexer = new UserAgentTreeWalkerLexer(input);

    lexer.addErrorListener(errorListener);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    UserAgentTreeWalkerParser parser = new UserAgentTreeWalkerParser(tokens);

    parser.addErrorListener(errorListener);

    //        parser.setTrace(true);
    requiredPattern = parseWalkerExpression(parser); // parse

    // We couldn't ditch the double quotes around the fixed values in the parsing pase.
    // So we ditch them here. We simply walk the tree and modify some of the tokens.
    new UnQuoteValues().visit(requiredPattern);

    // Now we create an evaluator instance
    evaluator = new TreeExpressionEvaluator(requiredPattern, newMatcher.lookups, verbose);

    // Is a fixed value (i.e. no events will ever be fired)?
    String fixedValue = evaluator.getFixedValue();
    if (fixedValue != null) {
        setFixedValue(fixedValue);//  w  w  w .ja  va  2 s  .c  om
        isFixedString = true;
        return; // Not interested in any patterns
    }

    registerAllPossiblyInterestingPatterns();
}

---

From source file:nl.basjes.parse.useragent.parse.UserAgentTreeFlattener.java

License:Apache License

private UserAgentContext parseUserAgent(UserAgent userAgent) {
    String userAgentString = EvilManualUseragentStringHacks.fixIt(userAgent.getUserAgentString());

    ANTLRInputStream input = new ANTLRInputStream(userAgentString);
    UserAgentLexer lexer = new UserAgentLexer(input);

    CommonTokenStream tokens = new CommonTokenStream(lexer);

    UserAgentParser parser = new UserAgentParser(tokens);

    if (!verbose) {
        lexer.removeErrorListeners();/* w  w  w . j  a  v a 2 s.c o m*/
        parser.removeErrorListeners();
    }
    lexer.addErrorListener(userAgent);
    parser.addErrorListener(userAgent);

    return parser.userAgent();
}

---

From source file:nl.bigo.pp.Main.java

License:Open Source License

public static void main(String[] args) throws Exception {

    if (args.length == 0) {
        // When no parameter is provided, pick Karpov's PGN database file.
        args = new String[] { "src/resources/Karpov.pgn" };
    }//from  www  .  j  ava  2 s  .co m

    String fileName = args[0];
    System.out.printf("Parsing `%s`...", fileName);

    // Create the lexer and parser.
    PGNLexer lexer = new PGNLexer(new ANTLRFileStream(fileName));
    PGNParser parser = new PGNParser(new CommonTokenStream(lexer));

    // Use the demo listener which will print some info about the PGN file.
    ParseTreeWalker walker = new ParseTreeWalker();
    ParseTree tree = parser.parse();
    walker.walk(new DemoListener(), tree);

    System.out.println("\nDone!");
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »