Example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

List of usage examples for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

  1. HOME
  2. Java
  3. org.antlr
  4. org.antlr.v4.runtime.*
  5. CommonTokenStream
  6. CommonTokenStream

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream.

Prototype

public CommonTokenStream(TokenSource tokenSource)

Source Link

Document

Constructs a new CommonTokenStream using the specified token source and the default token channel ( Token#DEFAULT_CHANNEL ).

Usage

From source file:com.oracle.truffle.dsl.processor.expression.DSLExpression.java

License:Open Source License

public static DSLExpression parse(String input) {
    ExpressionLexer lexer = new ExpressionLexer(CharStreams.fromString(input));
    TokenStream tokens = new CommonTokenStream(lexer);
    ExpressionParser parser = new ExpressionParser(tokens);
    lexer.removeErrorListeners();//from www  .  j av a 2s . c o m
    parser.removeErrorListeners();
    lexer.addErrorListener(DSLErrorListener.INSTANCE);
    parser.addErrorListener(DSLErrorListener.INSTANCE);
    try {
        return parser.expression().result;
    } catch (RecognitionException e) {
        throw new InvalidExpressionException(e.getMessage());
    }
}

From source file:com.oracle.truffle.llvm.LlvmMain.java

License:Open Source License

/**
 * Parse and run the specified Llvm source. Factored out in a separate method so that it can
 * also be used by the unit test harness.
 *///from  w  ww.  j av  a  2  s.  c  o m

public static void runTwo(String in, LlvmContext context, Source source) throws Exception {
    // ANTLRFileStream input = new ANTLRFileStream(in);
    // MethodSLLexer lexer = new MethodSLLexer(input);
    // CommonTokenStream tokens = new CommonTokenStream(lexer);
    // MethodSLParser parser = new MethodSLParser(tokens, context, source);
    // parser.prog();
    ANTLRFileStream input = new ANTLRFileStream(in);
    LLVM_IRLexer lexer = new LLVM_IRLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    LLVM_IRParser parser = new LLVM_IRParser(tokens, context, source);
    parser.model();
}

From source file:com.oracle.truffle.sl.parser.SimpleLanguageParser.java

License:Open Source License

public static Map<String, RootCallTarget> parseSL(SLLanguage language, Source source) {
    SimpleLanguageLexer lexer = new SimpleLanguageLexer(
            CharStreams.fromString(source.getCharacters().toString()));
    SimpleLanguageParser parser = new SimpleLanguageParser(new CommonTokenStream(lexer));
    lexer.removeErrorListeners();/*from w  w  w  .j  a v a 2s.  co m*/
    parser.removeErrorListeners();
    BailoutErrorListener listener = new BailoutErrorListener(source);
    lexer.addErrorListener(listener);
    parser.addErrorListener(listener);
    parser.factory = new SLNodeFactory(language, source);
    parser.source = source;
    parser.simplelanguage();
    return parser.factory.getAllFunctions();
}

From source file:com.parser.java.ParserHelper.java

public void initWithFileName(String str) throws IOException {
    CharStream c = new ANTLRFileStream(str);

    strFilePath = str;/*from   w ww .j ava 2  s. c  o m*/
    // create the lexer attached to stdin
    lexer = new JavaLexer(c);
    // create the buffer of tokens between the lexer and parser
    tokens = new CommonTokenStream(lexer);
    parser = new JavaParser(tokens);
    parser.compilationUnit();
    parser.compilationUnit();
    FileReader fReader;
    fReader = new FileReader(str);
    BufferedReader reader = new BufferedReader(fReader);
    String cursor; // 
    String content = "";

    while ((cursor = reader.readLine()) != null) {

        content += cursor;

    }

    Set<String> keywords = new HashSet<String>();
    Set<String> specialCharacters = new HashSet<String>();
    parser.compilationUnit();

    for (String str1 : lexer.arrSpecialCharacter) {
        if (specialCharacters.add(str1)) {
            spcialCharCt++;
        }
    }

    for (String str2 : lexer.arrKeywords) {
        if (keywords.add(str2)) {
            keywordCtUnique++;
        }
    }
    chars = content.length();
    System.out.println("_---------------------------------------------------------------");

    Set<String> ConstArr = new HashSet<String>();
    for (String strConst : parser.arrConst) {
        if (ConstArr.add(strConst)) {
            //System.out.println("keyword is " + token.getText());
            //spcialCharCt++;
        }
    }

    chars = content.length();
    System.out.println("_---------------------------------------------------------------");

    ParserModel modelPackageName = new ParserModel();
    //PackageName
    modelPackageName.setParserElement("PackageName");
    modelPackageName.setParserContentAndCount(parser.strPackageName);
    arrHelper.add(modelPackageName);

    ParserModel modelClassName = new ParserModel();
    modelClassName.setParserElement("ClassName");
    modelClassName.setParserContentAndCount(parser.strClassName);
    arrHelper.add(modelClassName);

    ParserModel modelUniqueUID = new ParserModel();
    modelUniqueUID.setParserElement("Total UID Count");
    modelUniqueUID.setParserContentAndCount(String.valueOf(parser.UIDTTTT));
    arrHelper.add(modelUniqueUID);

    ParserModel modelUID = new ParserModel();
    modelUID.setParserElement("Unique UID Count");
    modelUID.setParserContentAndCount(String.valueOf(parser.uniqueUIDCount));
    arrHelper.add(modelUID);

    ParserModel modelSpecialCharCount = new ParserModel();
    modelSpecialCharCount.setParserElement("Special Character Count");
    modelSpecialCharCount.setParserContentAndCount(String.valueOf(lexer.arrSpecialCharacter.size()));
    arrHelper.add(modelSpecialCharCount);

    ParserModel modelUniqueSpecialChar = new ParserModel();
    modelUniqueSpecialChar.setParserElement("Unique Special Character Count");
    modelUniqueSpecialChar.setParserContentAndCount(
            String.valueOf(spcialCharCt).concat(" -> ").concat(specialCharacters.toString()));
    arrHelper.add(modelUniqueSpecialChar);

    ParserModel modelConstCount = new ParserModel();
    modelConstCount.setParserElement("Constant Counts");
    modelConstCount.setParserContentAndCount(String.valueOf(parser.constCount));
    arrHelper.add(modelConstCount);

    ParserModel modelUniqueConstCount = new ParserModel();
    modelUniqueConstCount.setParserElement("Unique Constant Counts");
    modelUniqueConstCount.setParserContentAndCount(
            String.valueOf(ConstArr.size()).concat(" -> ").concat(ConstArr.toString()));
    arrHelper.add(modelUniqueConstCount);

    ParserModel modelKeywordCount = new ParserModel();
    int count = lexer.arrKeywords.size() + parser.arrKeyword.size();
    modelKeywordCount.setParserElement("Keyword count and Content");
    ArrayList<String> strArray = new ArrayList();

    strArray.addAll(lexer.arrKeywords);
    strArray.addAll(parser.arrKeyword);

    modelKeywordCount.setParserContentAndCount(String.valueOf(
            String.valueOf(strArray.size()).concat(" -> ").concat(String.valueOf(strArray.toString()))));
    arrHelper.add(modelKeywordCount);

    for (String str2 : strArray) {
        if (keywords.add(str2)) {
            keywordCtUnique++;
        }
    }
    ParserModel modelKeywordUnique = new ParserModel();
    modelKeywordUnique.setParserElement("Keyword Unique Count and Content");
    modelKeywordUnique.setParserContentAndCount(
            String.valueOf(keywordCtUnique).concat(" -> ").concat(String.valueOf(keywords.toString())));
    arrHelper.add(modelKeywordUnique);

    ParserModel modelCommentCount1 = new ParserModel();
    //modelWhiteSpace.setParserElement("WhiteSpace Char Count");
    modelCommentCount1.setParserElement("Comment to total characters ");
    modelCommentCount1.setParserContentAndCount(String.valueOf(lexer.lineCommentCount));
    arrHelper.add(modelCommentCount1);

    ParserModel modelCommentCount = new ParserModel();
    //modelWhiteSpace.setParserElement("WhiteSpace Char Count");
    float whiteSpacePerc1 = (float) (lexer.lineCommentCount) / chars * 100;
    modelCommentCount.setParserElement("Comment to total characters percentage");
    modelCommentCount.setParserContentAndCount(String.valueOf(whiteSpacePerc1).concat("%"));
    arrHelper.add(modelCommentCount);

    /*ParserModel modelTotalChar = new ParserModel();
    modelTotalChar.setParserElement("Total Char Count");
    modelTotalChar.setParserContentAndCount(String.valueOf(chars));
    arrHelper.add(modelTotalChar);*/
    ParserModel modelWhiteSpace1 = new ParserModel();
    modelWhiteSpace1.setParserElement("WhiteSpace total count");
    modelWhiteSpace1.setParserContentAndCount(String.valueOf(lexer.whiteSpaceCount));
    arrHelper.add(modelWhiteSpace1);
    System.out.println("---------------------------- " + arrHelper.size());
    ParserModel modelWhiteSpace = new ParserModel();
    modelWhiteSpace.setParserElement("WhiteSpace to total characters percentages");
    float whiteSpacePerc = (float) (lexer.whiteSpaceCount) / chars * 100;
    modelWhiteSpace.setParserContentAndCount(String.valueOf(whiteSpacePerc).concat("%"));
    arrHelper.add(modelWhiteSpace);
    System.out.println("---------------------------- " + arrHelper.size());

    System.out.println("Package name is " + parser.strPackageName);
    System.out.println("Class name is " + parser.strClassName);
    System.out.println("unique uid count is " + parser.uniqueUIDCount);
    System.out.println("uid count is " + parser.uidCount);
    System.out.println("spacial Character Count is " + lexer.specialCharCount);
    System.out.println("Unique Special Character " + spcialCharCt);

    System.out.println("Constant Count " + parser.constCount);
    System.out.println("keywordCt is " + lexer.arrKeywords.size());
    System.out.println("Unique keywordCt is " + keywordCtUnique);
    System.out.println("Keywords are :" + lexer.arrKeywords);
    System.out.println("Line Comment :" + lexer.lineCommentCount);
    System.out.println("Character count :" + chars);

}

From source file:com.paypal.digraph.parser.GraphParser.java

License:BSD License

public GraphParser(InputStream is) throws GraphParserException {
    DOTLexer lexer = null;//from  ww w .j a  v  a  2 s  .  co  m
    DOTParser parser = null;
    try {
        lexer = new DOTLexer(new ANTLRInputStream(is));
        lexer.addErrorListener(new ErrorListener());

        parser = new DOTParser(new CommonTokenStream(lexer));
        parser.setErrorHandler(new ExceptionErrorStrategy());
        parser.addErrorListener(new ErrorListener());

        ParseTree tree = parser.graph();
        ParseTreeWalker.DEFAULT.walk(new NodeListener(), tree);
        ParseTreeWalker.DEFAULT.walk(new EdgeListener(), tree);
    } catch (Throwable t) {
        if (mErrMsg != null)
            throw new GraphParserException(mErrMsg, t);
        if (lexer != null) {
            mErrMsg = "at line " + lexer.getLine() + ":" + lexer.getCharPositionInLine();
            throw new GraphParserException(mErrMsg, t);
        }
        throw new GraphParserException(t);
    }

    if (mErrMsg != null)
        throw new GraphParserException(mErrMsg);
}

From source file:com.project.NLP.SourceCodeToXML.AST.java

/**
 * creates the tree for walking/*from   w  w w . j av a  2  s .c  o  m*/
 * @param fileName
 */
public void sourceCodeTreeWalker(String fileName) {
    try {
        Java8Lexer lexer = new Java8Lexer(new ANTLRFileStream(fileName.trim()));
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        Java8Parser parser = new Java8Parser(tokens);
        ParserRuleContext tree = parser.compilationUnit();

        ParseTreeWalker walker = new ParseTreeWalker(); // create standard walker
        ExtractInterfaceListener extractor = new ExtractInterfaceListener(parser);
        walker.walk(extractor, tree);
    } catch (ParserConfigurationException ex) {
        JOptionPane.showMessageDialog(null, "Parser problem while parsing the source code files.",
                "Source-code Conversion", JOptionPane.ERROR_MESSAGE);
    } catch (IOException ex) {
        JOptionPane.showMessageDialog(null, "Problem occured in the given file.", "Source-code Conversion",
                JOptionPane.ERROR_MESSAGE);
    }
}

From source file:com.project.staticss.java8.AST.java

/**
 * creates the tree for walking//from   w  ww. ja v  a2 s . c o m
 * 
 * @param fileName
 */
public void sourceCodeTreeWalker(String fileName) {
    try {
        Java8Lexer lexer = new Java8Lexer(new ANTLRFileStream(fileName.trim()));
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        Java8Parser parser = new Java8Parser(tokens);
        ParserRuleContext tree = parser.compilationUnit();

        ParseTreeWalker walker = new ParseTreeWalker(); // create standard
        // walker
        ExtractInterfaceListener extractor = new ExtractInterfaceListener(parser);
        walker.walk(extractor, tree);
    } catch (ParserConfigurationException ex) {
        JOptionPane.showMessageDialog(null, "Parser problem while parsing the source code files.",
                "Source-code Conversion", JOptionPane.ERROR_MESSAGE);
    } catch (IOException ex) {
        JOptionPane.showMessageDialog(null, "Problem occured in the given file.", "Source-code Conversion",
                JOptionPane.ERROR_MESSAGE);
    }
}

From source file:com.puppycrawl.tools.checkstyle.checks.javadoc.AbstractJavadocCheck.java

License:Open Source License

/**
 * Parses block comment content as javadoc comment.
 * @param blockComment/*from  w w w .j a v a2 s  . c  o  m*/
 *        block comment content.
 * @return parse tree
 */
private ParseTree parseJavadocAsParseTree(String blockComment) {
    final ANTLRInputStream input = new ANTLRInputStream(blockComment);

    final JavadocLexer lexer = new JavadocLexer(input);

    // remove default error listeners
    lexer.removeErrorListeners();

    // add custom error listener that logs parsing errors
    lexer.addErrorListener(errorListener);

    final CommonTokenStream tokens = new CommonTokenStream(lexer);

    final JavadocParser parser = new JavadocParser(tokens);

    // remove default error listeners
    parser.removeErrorListeners();

    // add custom error listener that logs syntax errors
    parser.addErrorListener(errorListener);

    // This strategy stops parsing when parser error occurs.
    // By default it uses Error Recover Strategy which is slow and useless.
    parser.setErrorHandler(new BailErrorStrategy());

    return parser.javadoc();
}

From source file:com.puppycrawl.tools.checkstyle.checks.javadoc.ExpectedParseTreeGenerator.java

License:Open Source License

private static ParseTree parseJavadocFromFile(File file) throws IOException {
    final String content = Files.toString(file, Charsets.UTF_8);
    final InputStream in = new ByteArrayInputStream(content.getBytes(Charsets.UTF_8));

    final ANTLRInputStream input = new ANTLRInputStream(in);
    final JavadocLexer lexer = new JavadocLexer(input);
    lexer.removeErrorListeners();/*from  w  w w  .  ja va2s. com*/

    final BaseErrorListener errorListener = new FailOnErrorListener();
    lexer.addErrorListener(errorListener);

    final CommonTokenStream tokens = new CommonTokenStream(lexer);

    final JavadocParser parser = new JavadocParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(errorListener);

    return parser.javadoc();
}

From source file:com.qosdev.QPL.Main.java

License:Open Source License

public static void main(String[] args) throws IOException {
    if (args.length != 1) {
        throw new RuntimeException("Please supply file as argument");
    }//from   ww w . jav a  2  s .co  m
    ANTLRInputStream ais = new ANTLRFileStream(args[0]);
    QPLLexer lex = new QPLLexer(ais);
    TokenStream toks = new CommonTokenStream(lex);
    QPLParser parse = new QPLParser(toks);
    ParseTree tree = parse.prog();
    System.out.println(new ImpVisitor(Paths.get(args[0]).getParent().normalize().toString()).visit(tree));
}