Example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream.

Prototype

public CommonTokenStream(TokenSource tokenSource) 

Source Link

Document

Constructs a new CommonTokenStream using the specified token source and the default token channel ( Token#DEFAULT_CHANNEL ).

Usage

From source file:amulet.translator.runtimecheck.RuntimeCheck.java

public RuntimeCheck(QMClass qmclass, String code, boolean toggle, Vector<String> global_arrays,
        ResourceProfiler resourceProfiler) {
    ANTLRInputStream input = new ANTLRInputStream(code);
    CLexer lexer = new CLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    CParser parser = new CParser(tokens);
    ParseTree tree;/*from   w  w  w  .  jav a  2 s  .  c om*/
    if (code.trim().length() > 0 && code.indexOf(";") == -1 && code.indexOf("//") == -1
            && code.indexOf("/*") == -1) {
        //Guard Code
        tree = parser.expression();
    } else {
        tree = parser.blockItemList();
    }

    ParseTreeWalker walker = new ParseTreeWalker(); // create standard walker
    CExtractor extractor = new CExtractor(qmclass, tokens, parser, toggle, global_arrays, resourceProfiler);
    walker.walk(extractor, tree); // initiate walk of tree with listener
    if (toggle) {
        checkedCode = togglePinCode + "\n";
    }
    checkedCode += extractor.rewriter.getText();
    if (toggle) {
        checkedCode += "\n" + togglePinCode;
    }
}

---

From source file:android.databinding.tool.ExpressionParser.java

License:Apache License

public Expr parse(String input, @Nullable Location locationInFile) {
    ANTLRInputStream inputStream = new ANTLRInputStream(input);
    BindingExpressionLexer lexer = new BindingExpressionLexer(inputStream);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    final BindingExpressionParser parser = new BindingExpressionParser(tokenStream);
    BindingExpressionParser.BindingSyntaxContext root = parser.bindingSyntax();
    try {//  w  w  w  .j  ava2s  .  c  om
        mModel.setCurrentLocationInFile(locationInFile);
        visitor.setParseTreeListener(new ParseTreeListener() {
            List<ParserRuleContext> mStack = new ArrayList<ParserRuleContext>();

            @Override
            public void visitTerminal(TerminalNode node) {
            }

            @Override
            public void visitErrorNode(ErrorNode node) {
            }

            @Override
            public void enterEveryRule(ParserRuleContext ctx) {
                mStack.add(ctx);
                mModel.setCurrentParserContext(ctx);
            }

            @Override
            public void exitEveryRule(ParserRuleContext ctx) {
                Preconditions.check(ctx == mStack.get(mStack.size() - 1),
                        "Inconsistent exit from context. Received %s, expecting %s", ctx.toInfoString(parser),
                        mStack.get(mStack.size() - 1).toInfoString(parser));
                mStack.remove(mStack.size() - 1);
                if (mStack.size() > 0) {
                    mModel.setCurrentParserContext(mStack.get(mStack.size() - 1));
                } else {
                    mModel.setCurrentParserContext(null);
                }
            }
        });
        return root.accept(visitor);
    } finally {
        mModel.setCurrentLocationInFile(null);
    }
}

---

From source file:android.databinding.tool.store.LayoutFileParser.java

License:Apache License

private ResourceBundle.LayoutFileBundle parseXml(final File original, String pkg) throws IOException {
    try {//  w ww . jav  a2s  .  co m
        Scope.enter(new FileScopeProvider() {
            @Override
            public String provideScopeFilePath() {
                return original.getAbsolutePath();
            }
        });
        final String xmlNoExtension = ParserHelper.stripExtension(original.getName());
        ANTLRInputStream inputStream = new ANTLRInputStream(new FileReader(original));
        XMLLexer lexer = new XMLLexer(inputStream);
        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
        XMLParser parser = new XMLParser(tokenStream);
        XMLParser.DocumentContext expr = parser.document();
        XMLParser.ElementContext root = expr.element();
        if (!"layout".equals(root.elmName.getText())) {
            return null;
        }
        XMLParser.ElementContext data = getDataNode(root);
        XMLParser.ElementContext rootView = getViewNode(original, root);

        if (hasMergeInclude(rootView)) {
            L.e(ErrorMessages.INCLUDE_INSIDE_MERGE);
            return null;
        }
        boolean isMerge = "merge".equals(rootView.elmName.getText());

        ResourceBundle.LayoutFileBundle bundle = new ResourceBundle.LayoutFileBundle(original, xmlNoExtension,
                original.getParentFile().getName(), pkg, isMerge);
        final String newTag = original.getParentFile().getName() + '/' + xmlNoExtension;
        parseData(original, data, bundle);
        parseExpressions(newTag, rootView, isMerge, bundle);
        return bundle;
    } finally {
        Scope.exit();
    }
}

---

From source file:android.databinding.tool.util.XmlEditor.java

License:Apache License

public static String strip(File f, String newTag) throws IOException {
    ANTLRInputStream inputStream = new ANTLRInputStream(new FileReader(f));
    XMLLexer lexer = new XMLLexer(inputStream);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    XMLParser parser = new XMLParser(tokenStream);
    XMLParser.DocumentContext expr = parser.document();
    XMLParser.ElementContext root = expr.element();

    if (root == null || !"layout".equals(nodeName(root))) {
        return null; // not a binding layout
    }//from  w w  w  .  j  a  va 2s .c  o m

    List<? extends ElementContext> childrenOfRoot = elements(root);
    List<? extends XMLParser.ElementContext> dataNodes = filterNodesByName("data", childrenOfRoot);
    if (dataNodes.size() > 1) {
        L.e("Multiple binding data tags in %s. Expecting a maximum of one.", f.getAbsolutePath());
    }

    ArrayList<String> lines = new ArrayList<>();
    lines.addAll(FileUtils.readLines(f, "utf-8"));

    for (android.databinding.parser.XMLParser.ElementContext it : dataNodes) {
        replace(lines, toPosition(it.getStart()), toEndPosition(it.getStop()), "");
    }
    List<? extends XMLParser.ElementContext> layoutNodes = excludeNodesByName("data", childrenOfRoot);
    if (layoutNodes.size() != 1) {
        L.e("Only one layout element and one data element are allowed. %s has %d", f.getAbsolutePath(),
                layoutNodes.size());
    }

    final XMLParser.ElementContext layoutNode = layoutNodes.get(0);

    ArrayList<Pair<String, android.databinding.parser.XMLParser.ElementContext>> noTag = new ArrayList<>();

    recurseReplace(layoutNode, lines, noTag, newTag, 0);

    // Remove the <layout>
    Position rootStartTag = toPosition(root.getStart());
    Position rootEndTag = toPosition(root.content().getStart());
    replace(lines, rootStartTag, rootEndTag, "");

    // Remove the </layout>
    ImmutablePair<Position, Position> endLayoutPositions = findTerminalPositions(root, lines);
    replace(lines, endLayoutPositions.left, endLayoutPositions.right, "");

    StringBuilder rootAttributes = new StringBuilder();
    for (AttributeContext attr : attributes(root)) {
        rootAttributes.append(' ').append(attr.getText());
    }
    Pair<String, XMLParser.ElementContext> noTagRoot = null;
    for (Pair<String, XMLParser.ElementContext> pair : noTag) {
        if (pair.getRight() == layoutNode) {
            noTagRoot = pair;
            break;
        }
    }
    if (noTagRoot != null) {
        ImmutablePair<String, XMLParser.ElementContext> newRootTag = new ImmutablePair<>(
                noTagRoot.getLeft() + rootAttributes.toString(), layoutNode);
        int index = noTag.indexOf(noTagRoot);
        noTag.set(index, newRootTag);
    } else {
        ImmutablePair<String, XMLParser.ElementContext> newRootTag = new ImmutablePair<>(
                rootAttributes.toString(), layoutNode);
        noTag.add(newRootTag);
    }
    //noinspection NullableProblems
    Collections.sort(noTag, new Comparator<Pair<String, XMLParser.ElementContext>>() {
        @Override
        public int compare(Pair<String, XMLParser.ElementContext> o1,
                Pair<String, XMLParser.ElementContext> o2) {
            Position start1 = toPosition(o1.getRight().getStart());
            Position start2 = toPosition(o2.getRight().getStart());
            int lineCmp = Integer.compare(start2.line, start1.line);
            if (lineCmp != 0) {
                return lineCmp;
            }
            return Integer.compare(start2.charIndex, start1.charIndex);
        }
    });
    for (Pair<String, android.databinding.parser.XMLParser.ElementContext> it : noTag) {
        XMLParser.ElementContext element = it.getRight();
        String tag = it.getLeft();
        Position endTagPosition = endTagPosition(element);
        fixPosition(lines, endTagPosition);
        String line = lines.get(endTagPosition.line);
        String newLine = line.substring(0, endTagPosition.charIndex) + " " + tag
                + line.substring(endTagPosition.charIndex);
        lines.set(endTagPosition.line, newLine);
    }
    return StringUtils.join(lines, System.getProperty("line.separator"));
}

---

From source file:android.databinding.tool.util.XmlEditor.java

License:Apache License

private static String defaultReplacement(XMLParser.AttributeContext attr) {
    String textWithQuotes = attr.attrValue.getText();
    String escapedText = textWithQuotes.substring(1, textWithQuotes.length() - 1);
    if (!escapedText.startsWith("@{") || !escapedText.endsWith("}")) {
        return null;
    }/*from w  w  w .  ja  va 2  s.c o  m*/
    String text = StringEscapeUtils.unescapeXml(escapedText.substring(2, escapedText.length() - 1));
    ANTLRInputStream inputStream = new ANTLRInputStream(text);
    BindingExpressionLexer lexer = new BindingExpressionLexer(inputStream);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    BindingExpressionParser parser = new BindingExpressionParser(tokenStream);
    BindingExpressionParser.BindingSyntaxContext root = parser.bindingSyntax();
    BindingExpressionParser.DefaultsContext defaults = root.defaults();
    if (defaults != null) {
        BindingExpressionParser.ConstantValueContext constantValue = defaults.constantValue();
        BindingExpressionParser.LiteralContext literal = constantValue.literal();
        if (literal != null) {
            BindingExpressionParser.StringLiteralContext stringLiteral = literal.stringLiteral();
            if (stringLiteral != null) {
                TerminalNode doubleQuote = stringLiteral.DoubleQuoteString();
                if (doubleQuote != null) {
                    String quotedStr = doubleQuote.getText();
                    String unquoted = quotedStr.substring(1, quotedStr.length() - 1);
                    return StringEscapeUtils.escapeXml10(unquoted);
                } else {
                    String quotedStr = stringLiteral.SingleQuoteString().getText();
                    String unquoted = quotedStr.substring(1, quotedStr.length() - 1);
                    String unescaped = unquoted.replace("\"", "\\\"").replace("\\`", "`");
                    return StringEscapeUtils.escapeXml10(unescaped);
                }
            }
        }
        return constantValue.getText();
    }
    return null;
}

---

From source file:anndl.Anndl.java

private static void buildModel(InputStream input) throws Exception {
    ANNDLLexer lexer = new ANNDLLexer(new ANTLRInputStream(input));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    ANNDLParser parser = new ANNDLParser(tokens);
    ParseTree tree = parser.model();/*from  www .j a v a 2 s  . c o m*/

    ModelVisitor visitor = new ModelVisitor();

    ModelClassifier themodel = (ModelClassifier) visitor.visit(tree);
    //themodel.PrintInfo();
    themodel.extracthidden();

    System.out.println("Membaca File Training...");
    DataSource trainingsoure = new DataSource(themodel.filetraining);
    Instances trainingdata = trainingsoure.getDataSet();
    if (trainingdata.classIndex() == -1) {
        trainingdata.setClassIndex(trainingdata.numAttributes() - 1);
    }

    System.out.println("Melakukan konfigurasi ANN ... ");
    MultilayerPerceptron mlp = new MultilayerPerceptron();
    mlp.setLearningRate(themodel.learningrate);
    mlp.setMomentum(themodel.momentum);
    mlp.setTrainingTime(themodel.epoch);
    mlp.setHiddenLayers(themodel.hidden);

    System.out.println("Melakukan Training data ...");
    mlp.buildClassifier(trainingdata);

    Debug.saveToFile(themodel.namamodel + ".model", mlp);

    System.out.println("\n~~ .. ~~ .. ~~ .. ~~ .. ~~ .. ~~ .. ~~ .. ~~ .. ~~ ..");
    System.out.println("Model ANN Berhasil Diciptakan dengan nama file : " + themodel.namamodel + ".model");
    System.out.println("~~ .. ~~ .. ~~ .. ~~ .. ~~ .. ~~ .. ~~ .. ~~ .. ~~ .. \n");

}

---

From source file:annis.ql.parser.AnnisParserAntlr.java

License:Apache License

public QueryData parse(String aql, List<Long> corpusList) {
    final List<AqlParseError> errors = new LinkedList<>();

    AqlLexer lexerNonDNF = new AqlLexer(new ANTLRInputStream(aql));
    lexerNonDNF.removeErrorListeners();/*  www. j av a 2s.  c o  m*/
    lexerNonDNF.addErrorListener(new AqlLexerErrorListener(errors));

    // bring first into DNF
    RawAqlPreParser rawParser = new RawAqlPreParser(new CommonTokenStream(lexerNonDNF));
    rawParser.removeErrorListeners();
    rawParser.addErrorListener(new AqlParseErrorListener(errors));

    RawAqlPreParser.StartContext treeRaw = rawParser.start();
    if (!errors.isEmpty()) {
        throw new AnnisQLSyntaxException(Joiner.on("\n").join(errors), errors);
    }
    //treeRaw.inspect(rawParser);

    ParseTreeWalker walkerRaw = new ParseTreeWalker();
    RawAqlListener listenerRaw = new RawAqlListener();
    walkerRaw.walk(listenerRaw, treeRaw);

    LogicClause topNode = listenerRaw.getRoot();

    DNFTransformer.toDNF(topNode);

    // use the DNF form and parse it again
    TokenSource source = new ListTokenSource(topNode.getCoveredToken());
    AqlParser parserDNF = new AqlParser(new CommonTokenStream(source));

    parserDNF.removeErrorListeners();
    parserDNF.addErrorListener(new AqlParseErrorListener(errors));

    AqlParser.StartContext treeDNF = parserDNF.start();

    //treeDNF.inspect(parserDNF);

    if (!errors.isEmpty()) {
        throw new AnnisQLSyntaxException(Joiner.on("\n").join(errors), errors);
    }

    ParseTreeWalker walker = new ParseTreeWalker();
    NodeIDListener idListener = new NodeIDListener();
    walker.walk(idListener, treeDNF);

    QueryNodeListener nodeListener = new QueryNodeListener(idListener.getNodeIntervalToID());

    try {
        walker.walk(nodeListener, treeDNF);

        QueryData data = nodeListener.getQueryData();

        data.setCorpusList(corpusList);
        data.addMetaAnnotations(nodeListener.getMetaData());

        JoinListener joinListener = new JoinListener(data, precedenceBound, nodeListener.getTokenPositions());
        walker.walk(joinListener, treeDNF);

        if (postProcessors != null) {
            for (QueryDataTransformer transformer : postProcessors) {
                data = transformer.transform(data);
            }
        }
        return data;

    } catch (NullPointerException ex) {
        log.warn("Null pointer exception occured during parsing", ex);
        throw new AnnisQLSemanticsException(ex.getMessage());
    } catch (IllegalArgumentException ex) {
        throw new AnnisQLSemanticsException(ex.getMessage());
    }

}

---

From source file:annis.ql.parser.AnnisParserAntlr.java

License:Apache License

public String dumpTree(String aql) {
    AqlLexer lexer = new AqlLexer(new ANTLRInputStream(aql));
    AqlParser parser = new AqlParser(new CommonTokenStream(lexer));

    final List<AqlParseError> errors = new LinkedList<>();

    parser.removeErrorListeners();//w w  w  .j a  v a 2s.  c  om
    parser.addErrorListener(new AqlParseErrorListener(errors));

    ParseTree tree = parser.start();

    if (errors.isEmpty()) {
        return tree.toStringTree();
    } else {
        throw new AnnisQLSyntaxException(Joiner.on("\n").join(errors), errors);
    }
}

---

From source file:annis.visualizers.htmlvis.VisParser.java

License:Apache License

public VisParser(InputStream inStream) throws IOException, VisParserException {
    this.definitions = new LinkedList<VisualizationDefinition>();

    HTMLVisConfigLexer lexer = new HTMLVisConfigLexer(new ANTLRInputStream(inStream));
    HTMLVisConfigParser parser = new HTMLVisConfigParser(new CommonTokenStream(lexer));

    final List<String> errors = new LinkedList<String>();

    parser.removeErrorListeners();/*from  w  w w  .  java 2  s . c o  m*/
    parser.addErrorListener(new BaseErrorListener() {

        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
                int charPositionInLine, String msg, RecognitionException e) {
            errors.add("line " + line + ":" + charPositionInLine + " " + msg);
        }

    });

    ParseTree tree = parser.start();
    if (errors.isEmpty()) {
        ParseTreeWalker walker = new ParseTreeWalker();
        walker.walk((VisParser) this, tree);
    } else {
        throw new VisParserException("Parser error:\n" + StringUtils.join(errors, "\n"));
    }
}

---

From source file:aqua.nix.cli.dsl.SemanticGenerator.java

License:Open Source License

public List<CommandAttribute> interpret(String input) {
    List<CommandAttribute> commands = new ArrayList<>();

    CharStream stream = new ANTLRInputStream(input);
    InputLexer lexer = new InputLexer(stream);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    InputParser parser = new InputParser(tokens);
    parser.addParseListener(new InputListenerImpl(commands));

    parser.input();// w w  w  . j  a  va  2  s . c om
    return commands;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »