Example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream.

Prototype

public CommonTokenStream(TokenSource tokenSource) 

Source Link

Document

Constructs a new CommonTokenStream using the specified token source and the default token channel ( Token#DEFAULT_CHANNEL ).

Usage

From source file:org.apache.drill.exec.record.metadata.schema.parser.SchemaExprParser.java

License:Apache License

private static SchemaParser initParser(String value) {
    CodePointCharStream stream = CharStreams.fromString(value);
    UpperCaseCharStream upperCaseStream = new UpperCaseCharStream(stream);

    SchemaLexer lexer = new SchemaLexer(upperCaseStream);
    lexer.removeErrorListeners();//  w ww.  j  a va2  s  . co  m
    lexer.addErrorListener(ErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);

    SchemaParser parser = new SchemaParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(ErrorListener.INSTANCE);

    return parser;
}

---

From source file:org.apache.hive.hplsql.Exec.java

License:Apache License

/**
 * Initialize PL/HQL//from www  .j av a 2 s. c o m
 */
Integer init(String[] args) throws Exception {
    if (!parseArguments(args)) {
        return 1;
    }
    // specify the default log4j2 properties file.
    System.setProperty("log4j.configurationFile", "hive-log4j2.properties");
    conf = new Conf();
    conf.init();
    conn = new Conn(this);
    meta = new Meta(this);
    initOptions();

    expr = new Expression(this);
    select = new Select(this);
    stmt = new Stmt(this);
    converter = new Converter(this);

    function = new Function(this);
    new FunctionDatetime(this).register(function);
    new FunctionMisc(this).register(function);
    new FunctionString(this).register(function);
    new FunctionOra(this).register(function);
    addVariable(new Var(ERRORCODE, Var.Type.BIGINT, 0L));
    addVariable(new Var(SQLCODE, Var.Type.BIGINT, 0L));
    addVariable(new Var(SQLSTATE, Var.Type.STRING, "00000"));
    addVariable(new Var(HOSTCODE, Var.Type.BIGINT, 0L));
    for (Map.Entry<String, String> v : arguments.getVars().entrySet()) {
        addVariable(new Var(v.getKey(), Var.Type.STRING, v.getValue()));
    }
    InputStream input = null;
    if (execString != null) {
        input = new ByteArrayInputStream(execString.getBytes("UTF-8"));
    } else {
        input = new FileInputStream(execFile);
    }
    HplsqlLexer lexer = new HplsqlLexer(new ANTLRInputStream(input));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    HplsqlParser parser = new HplsqlParser(tokens);
    tree = parser.program();
    if (trace) {
        System.err.println("Configuration file: " + conf.getLocation());
        System.err.println("Parser tree: " + tree.toStringTree(parser));
    }
    includeRcFile();
    return 0;
}

---

From source file:org.apache.hive.hplsql.Exec.java

License:Apache License

/**
 * Execute statements from an include file
 *///from  ww w.ja  va  2 s.  c o m
void include(String content) throws Exception {
    InputStream input = new ByteArrayInputStream(content.getBytes("UTF-8"));
    HplsqlLexer lexer = new HplsqlLexer(new ANTLRInputStream(input));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    HplsqlParser parser = new HplsqlParser(tokens);
    ParseTree tree = parser.program();
    visit(tree);
}

---

From source file:org.apache.metron.common.query.PredicateProcessor.java

License:Apache License

public boolean parse(String rule, VariableResolver resolver) {
    if (rule == null || isEmpty(rule.trim())) {
        return true;
    }//from ww w.  j  a  va2 s  .c  om
    ANTLRInputStream input = new ANTLRInputStream(rule);
    PredicateLexer lexer = new PredicateLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(new ErrorListener());
    TokenStream tokens = new CommonTokenStream(lexer);
    PredicateParser parser = new PredicateParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(new ErrorListener());
    QueryCompiler treeBuilder = new QueryCompiler(resolver);
    parser.removeParseListeners();
    parser.addParseListener(treeBuilder);
    parser.single_rule();
    return treeBuilder.getResult();
}

---

From source file:org.apache.metron.common.stellar.BaseStellarProcessor.java

License:Apache License

/**
 * Parses and evaluates the given Stellar expression, {@code rule}.
 * @param rule The Stellar expression to parse and evaluate.
 * @return The Expression, which can be reevaluated without reparsing in different Contexts and Resolvers.
 *//*w  w w.  java2 s.  c  om*/
public static StellarCompiler.Expression compile(final String rule) {
    if (rule == null || isEmpty(rule.trim())) {
        return null;
    }

    ANTLRInputStream input = new ANTLRInputStream(rule);
    StellarLexer lexer = new StellarLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(new ErrorListener());
    TokenStream tokens = new CommonTokenStream(lexer);
    StellarParser parser = new StellarParser(tokens);

    StellarCompiler treeBuilder = new StellarCompiler(ArithmeticEvaluator.INSTANCE,
            NumberLiteralEvaluator.INSTANCE, ComparisonExpressionWithOperatorEvaluator.INSTANCE);
    parser.addParseListener(treeBuilder);
    parser.removeErrorListeners();
    parser.addErrorListener(new ErrorListener());
    parser.transformation();
    return treeBuilder.getExpression();
}

---

From source file:org.apache.metron.common.transformation.TransformationProcessor.java

License:Apache License

public Object parse(String rule, VariableResolver resolver) {
    if (rule == null || isEmpty(rule.trim())) {
        return null;
    }/*from  www  .  j  a va  2  s.com*/
    ANTLRInputStream input = new ANTLRInputStream(rule);
    TransformationLexer lexer = new TransformationLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(new ErrorListener());
    TokenStream tokens = new CommonTokenStream(lexer);
    TransformationParser parser = new TransformationParser(tokens);

    TransformationCompiler treeBuilder = new TransformationCompiler(resolver);
    parser.addParseListener(treeBuilder);
    parser.removeErrorListeners();
    parser.addErrorListener(new ErrorListener());
    parser.transformation();
    return treeBuilder.getResult();
}

---

From source file:org.apache.metron.profiler.client.window.WindowProcessor.java

License:Apache License

private static TokenStream createTokenStream(String statement) {
    if (statement == null || isEmpty(statement.trim())) {
        return null;
    }//from   ww  w. j  ava 2  s. co  m
    statement = statement.trim();
    ANTLRInputStream input = new ANTLRInputStream(statement);
    WindowLexer lexer = new WindowLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(new ErrorListener());
    TokenStream tokens = new CommonTokenStream(lexer);
    return tokens;
}

---

From source file:org.apache.olingo.server.core.uri.parser.Parser.java

License:Apache License

private ParserRuleContext parseRule(final String input, final ParserEntryRules entryPoint)
        throws UriParserSyntaxException {
    UriParserParser parser = null;/*from   w  w w  . j a  v a2 s.  c om*/
    UriLexer lexer = null;
    ParserRuleContext ret = null;

    // Use 2 stage approach to improve performance
    // see https://github.com/antlr/antlr4/issues/192

    // stage = 1
    try {

        // create parser
        if (logLevel > 0) {
            //TODO: Discuss if we should keep this code
            lexer = new UriLexer(new ANTLRInputStream(input));
            showTokens(input, lexer.getAllTokens());
        }

        lexer = new UriLexer(new ANTLRInputStream(input));
        parser = new UriParserParser(new CommonTokenStream(lexer));

        // Set error strategy
        addStage1ErrorStategy(parser);

        // Set error collector
        addStage1ErrorListener(parser);

        // user the faster LL parsing
        parser.getInterpreter().setPredictionMode(PredictionMode.SLL);

        // parse
        switch (entryPoint) {
        case All:
            ret = parser.allEOF();
            break;
        case Batch:
            ret = parser.batchEOF();
            break;
        case CrossJoin:
            ret = parser.crossjoinEOF();
            break;
        case Metadata:
            ret = parser.metadataEOF();
            break;
        case PathSegment:
            ret = parser.pathSegmentEOF();
            break;
        case FilterExpression:
            lexer.mode(Lexer.DEFAULT_MODE);
            ret = parser.filterExpressionEOF();
            break;
        case Orderby:
            lexer.mode(Lexer.DEFAULT_MODE);
            ret = parser.orderByEOF();
            break;
        case ExpandItems:
            lexer.mode(Lexer.DEFAULT_MODE);
            ret = parser.expandItemsEOF();
            break;
        case Entity:
            ret = parser.entityEOF();
            break;
        case Select:
            ret = parser.selectEOF();
            break;
        default:
            break;

        }

    } catch (ParseCancellationException hardException) {
        // stage = 2
        try {

            // create parser
            lexer = new UriLexer(new ANTLRInputStream(input));
            parser = new UriParserParser(new CommonTokenStream(lexer));

            // Set error strategy
            addStage2ErrorStategy(parser);

            // Set error collector
            addStage2ErrorListener(parser);

            // Use the slower SLL parsing
            parser.getInterpreter().setPredictionMode(PredictionMode.LL);

            // parse
            switch (entryPoint) {
            case All:
                ret = parser.allEOF();
                break;
            case Batch:
                ret = parser.batchEOF();
                break;
            case CrossJoin:
                ret = parser.crossjoinEOF();
                break;
            case Metadata:
                ret = parser.metadataEOF();
                break;
            case PathSegment:
                ret = parser.pathSegmentEOF();
                break;
            case FilterExpression:
                lexer.mode(Lexer.DEFAULT_MODE);
                ret = parser.filterExpressionEOF();
                break;
            case Orderby:
                lexer.mode(Lexer.DEFAULT_MODE);
                ret = parser.orderByEOF();
                break;
            case ExpandItems:
                lexer.mode(Lexer.DEFAULT_MODE);
                ret = parser.expandItemsEOF();
                break;
            case Entity:
                ret = parser.entityEOF();
                break;
            case Select:
                ret = parser.selectEOF();
                break;
            default:
                break;
            }

        } catch (final RecognitionException weakException) {
            throw new UriParserSyntaxException("Error in syntax", weakException,
                    UriParserSyntaxException.MessageKeys.SYNTAX);

            // exceptionOnStage = 2;
        }
    } catch (final RecognitionException hardException) {
        throw new UriParserSyntaxException("Error in syntax", hardException,
                UriParserSyntaxException.MessageKeys.SYNTAX);
    }

    return ret;
}

---

From source file:org.apache.sling.scripting.sightly.impl.compiler.frontend.ExpressionParser.java

License:Apache License

private SightlyParser createParser(String string) {
    SightlyLexer lexer = new SightlyLexer(new ANTLRInputStream(string));
    lexer.removeErrorListeners();//  w w w  .  j a va2  s.co  m
    lexer.addErrorListener(new SightlyParserErrorListener());
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    SightlyParser parser = new SightlyParser(tokenStream);
    parser.removeErrorListeners();
    parser.addErrorListener(new SightlyParserErrorListener());
    return parser;
}

---

From source file:org.apache.syncope.core.logic.scim.SearchCondConverter.java

License:Apache License

public static SearchCond convert(final SearchCondVisitor visitor, final String filter) {
    SCIMFilterParser parser = new SCIMFilterParser(
            new CommonTokenStream(new SCIMFilterLexer(CharStreams.fromString(filter))));
    parser.setBuildParseTree(true);//from   w ww .  j  a v a 2 s.  com
    parser.setTrimParseTree(true);
    parser.setProfile(true);
    parser.removeErrorListeners();
    parser.setErrorHandler(new SCIMFilterErrorHandler());

    try {
        return visitor.visit(parser.scimFilter());
    } catch (Exception e) {
        LOG.error("Could not parse '{}'", filter, e);
        throw new BadRequestException(ErrorType.invalidFilter, "Could not parse '" + filter + "'");
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »