Example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

List of usage examples for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

  1. HOME
  2. Java
  3. org.antlr
  4. org.antlr.v4.runtime.*
  5. CommonTokenStream
  6. CommonTokenStream

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream.

Prototype

public CommonTokenStream(TokenSource tokenSource)

Source Link

Document

Constructs a new CommonTokenStream using the specified token source and the default token channel ( Token#DEFAULT_CHANNEL ).

Usage

From source file:edu.clemson.cs.rsrg.init.Controller.java

License:Open Source License

/**
 * <p>This method uses the {@link ResolveFile} provided
 * to construct a parser and create an ANTLR4 module AST.</p>
 *
 * @param file The RESOLVE file that we are going to compile.
 *
 * @return The inner representation for a module. See {@link ModuleDec}.
 *
 * @throws MiscErrorException Some how we couldn't instantiate an {@link CharStream}.
 * @throws SourceErrorException There are errors in the source file.
 *///w  w  w  . j a v a  2 s.  com
private ModuleDec createModuleAST(ResolveFile file) {
    CharStream input = file.getInputStream();
    if (input == null) {
        throw new MiscErrorException("CharStream null", new IllegalArgumentException());
    }

    // Create a RESOLVE language lexer
    ResolveLexer lexer = new ResolveLexer(input);
    ResolveTokenFactory factory = new ResolveTokenFactory(file);
    lexer.removeErrorListeners();
    lexer.addErrorListener(myAntlrLexerErrorListener);
    lexer.setTokenFactory(factory);

    // Create a RESOLVE language parser
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    ResolveParser parser = new ResolveParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(myAntlrParserErrorListener);
    parser.setTokenFactory(factory);

    // Two-Stage Parsing
    // Reason: We might not need the full power of LL.
    // The solution proposed by the ANTLR folks (found here:
    // https://github.com/antlr/antlr4/blob/master/doc/faq/general.md)
    // is to use SLL prediction mode first and switch to LL if it fails.
    ParserRuleContext rootModuleCtx;
    parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
    try {
        rootModuleCtx = parser.module();
    } catch (Exception ex) {
        tokens.seek(0);
        parser.reset();
        parser.getInterpreter().setPredictionMode(PredictionMode.LL);
        rootModuleCtx = parser.module();
    }

    // Check for any parsing errors
    int numParserErrors = parser.getNumberOfSyntaxErrors();
    if (numParserErrors != 0) {
        throw new MiscErrorException("Found " + numParserErrors + " errors while parsing " + file.toString(),
                new IllegalStateException());
    }

    // Build the intermediate representation
    TreeBuildingListener v = new TreeBuildingListener(file, myCompileEnvironment.getTypeGraph());
    ParseTreeWalker.DEFAULT.walk(v, rootModuleCtx);

    return v.getModule();
}

From source file:edu.cmu.cs.vbc.prog.checkstyle.checks.javadoc.AbstractJavadocCheck.java

License:Open Source License

/**
 * Parses block comment content as javadoc comment.
 * @param blockComment/*www  .j av  a  2  s  .  com*/
 *        block comment content.
 * @return parse tree
 * @throws IOException
 *         errors in ANTLRInputStream
 */
private ParseTree parseJavadocAsParseTree(String blockComment) throws IOException {
    final Charset utf8Charset = Charset.forName("UTF-8");
    final InputStream in = new ByteArrayInputStream(blockComment.getBytes(utf8Charset));

    final ANTLRInputStream input = new ANTLRInputStream(in);

    final JavadocLexer lexer = new JavadocLexer(input);

    // remove default error listeners
    lexer.removeErrorListeners();

    // add custom error listener that logs parsing errors
    lexer.addErrorListener(errorListener);

    final CommonTokenStream tokens = new CommonTokenStream(lexer);

    final JavadocParser parser = new JavadocParser(tokens);

    // remove default error listeners
    parser.removeErrorListeners();

    // add custom error listener that logs syntax errors
    parser.addErrorListener(errorListener);

    // This strategy stops parsing when parser error occurs.
    // By default it uses Error Recover Strategy which is slow and useless.
    parser.setErrorHandler(new BailErrorStrategy());

    return parser.javadoc();
}

From source file:edu.iastate.cs.boa.ui.errors.FetchCompilerError.java

License:Apache License

protected CommonTokenStream lex(final String input, final int[] ids, final String[] strings,
        final String[] errors) throws IOException {
    lexer = new BoaLexer(new ANTLRInputStream(new StringReader(input)));
    lexer.removeErrorListeners();//from w w  w .ja va2s  .c om
    lexer.addErrorListener(new BaseErrorListener() {
        @Override
        public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
                final int charPositionInLine, final String msg, final RecognitionException e) {
            error("lexer", (BoaLexer) recognizer, offendingSymbol, line, charPositionInLine, 1, msg, e);
        }
    });

    final CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();

    return tokens;
}

From source file:edu.odu.cs.cs350.yellow1.mutationgeneration.CFile.java

License:Open Source License

private boolean applyTokenTransformation(String original, String by, String operation) throws Exception {
    ANTLRInputStream input = new ANTLRInputStream(fileContents.toString());
    CLexer lexer = new CLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    CParser parser = new CParser(tokens);
    ParseTree tree = parser.compilationUnit();
    ParseTreeWalker walker = new ParseTreeWalker(); // create standard

    for (Integer i = 0; i < tokens.size(); i++) {
        System.out.println("Tokens: " + tokens.get(i).getText().toString());
        if (tokens.get(i).getText().toString().equals(original)) {
            String s1 = ((Integer) tokens.get(i).getStartIndex()).toString();
            String s2 = ((Integer) tokens.get(i).getStopIndex()).toString();
            String uniqId = operation;
            uniqId = uniqId.concat(s1).concat(s2);

            String newName = fileContents.toString();
            if (mutationsAppliedContains(uniqId) < 0) {

                newName = newName.substring(0, tokens.get(i).getStartIndex()) + by
                        + newName.substring(tokens.get(i).getStopIndex() + by.length());
                writeMutation(this.fileName + "." + uniqId, newName);
                this.addToApplied(uniqId);
            }/*from  www .ja v a  2  s  . co m*/
        }
    }
    return true;
}

From source file:edu.odu.cs.cs350.yellow1.mutationgeneration.JavaFile.java

License:Open Source License

/**
 * change tokens and add the mutuant cases to the mutant case vector
 * //  w w w . j  a va 2  s. c o  m
 * @param originalthe
 *            original token
 * 
 * @param by
 *            the replacement token
 * 
 * @param operation
 *            the operation name
 * 
 * 
 *            it is acceptable int y = +10; to be converted to int y = /10;
 *            because it will make the mutant project compilation fail and
 *            this mutation will be discarded because the compilation has
 *            failed
 * 
 * 
 */
private boolean applyTokenTransformation(String original, String by, String operation) throws Exception {
    ANTLRInputStream input = new ANTLRInputStream(fileContents.toString());
    JavaLexer lexer = new JavaLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);

    JavaParser parser = new JavaParser(tokens);

    ParseTree tree = parser.compilationUnit();

    TokenRewriteStream trs = new TokenRewriteStream();

    ParseTreeWalker walker = new ParseTreeWalker(); // create standard

    for (Integer i = 0; i < tokens.size(); i++) {
        if (tokens.get(i).getText().toString().equals(original)) {
            int startIndex = tokens.get(i).getStartIndex();
            int stopIndex = tokens.get(i).getStopIndex() + 1;
            mvc.add(startIndex, stopIndex, by);

        }
    }
    return true;
}

From source file:edu.odu.cs.cs350.yellow1.mutationgeneration.JavaFile.java

License:Open Source License

/**
 * @param blockstart/*  w  w  w.ja v  a 2  s  .c  o  m*/
 *            accepts if or while
 * 
 * @param set
 *            accepts true or false
 * 
 *            Replaces the condition after [blockstart] by [set]
 */
private void applyConditionTransformation(String blockstart, String set) {
    int lparen = 0;
    int rparen = 0;
    int startIndex = -1;
    int stopIndex = -1;
    boolean ifLock = false;

    ANTLRInputStream input = new ANTLRInputStream(fileContents.toString());
    JavaLexer lexer = new JavaLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    JavaParser parser = new JavaParser(tokens);
    ParseTree tree = parser.compilationUnit();
    ParseTreeWalker walker = new ParseTreeWalker(); // create standard
    for (Integer i = 0; i < tokens.size(); i++) {

        if (tokens.get(i).getText().toString().equals(blockstart)) {
            ifLock = true;
        }

        if (ifLock == true && tokens.get(i).getText().toString().equals("(")) {
            lparen++;

            if (startIndex == -1)
                startIndex = tokens.get(i).getStartIndex();
        }

        if (ifLock == true && tokens.get(i).getText().toString().equals(")"))

        {

            rparen++;

        }

        if (lparen != 0 && rparen != 0 && lparen == rparen && ifLock == true) {
            ifLock = false;
            lparen = 0;
            rparen = 0;

            stopIndex = tokens.get(i).getStopIndex() + 1;

            String by = "(" + set + ")";

            this.mvc.add(startIndex, stopIndex, by);
            startIndex = -1;
            stopIndex = -1;
        }

    }

}

From source file:edu.odu.cs.cs350.yellow1.mutationgeneration.JavaFile.java

License:Open Source License

private void constantOperations(String required) {
    ANTLRInputStream input = new ANTLRInputStream(fileContents.toString());
    JavaLexer lexer = new JavaLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);

    JavaParser parser = new JavaParser(tokens);

    ParseTree tree = parser.compilationUnit();

    TokenRewriteStream trs = new TokenRewriteStream();

    ParseTreeWalker walker = new ParseTreeWalker(); // create standard

    for (Integer i = 0; i < tokens.size(); i++) {
        if (NumberUtils.isNumber(tokens.get(i).getText().toString())) {
            int startIndex = tokens.get(i).getStartIndex();
            int stopIndex = tokens.get(i).getStopIndex() + 1;
            String by = fileContents.substring(startIndex, stopIndex) + required;
            mvc.add(startIndex, stopIndex, by);
        }/*from  w  w w  . j a v a 2  s. co m*/
    }
}

From source file:edu.psu.swe.scim.spec.protocol.data.PatchOperationPath.java

License:Apache License

protected void parsePatchPath(String patchPath) throws FilterParseException {
    FilterLexer l = new FilterLexer(new ANTLRInputStream(patchPath));
    FilterParser p = new FilterParser(new CommonTokenStream(l));
    p.setBuildParseTree(true);/*w  w  w  .ja  v  a 2 s. c o m*/

    p.addErrorListener(new BaseErrorListener() {
        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
                int charPositionInLine, String msg, RecognitionException e) {
            throw new IllegalStateException("failed to parse at line " + line + " due to " + msg, e);
        }
    });

    try {
        ParseTree tree = p.patchPath();
        PatchPathListener patchPathListener = new PatchPathListener();
        ParseTreeWalker.DEFAULT.walk(patchPathListener, tree);

        this.valuePathExpression = patchPathListener.getValuePathExpression();
    } catch (IllegalStateException e) {
        throw new FilterParseException(e);
    }
}

From source file:edu.psu.swe.scim.spec.protocol.search.Filter.java

License:Apache License

protected FilterExpression parseFilter(String filter) throws FilterParseException {
    FilterLexer l = new FilterLexer(new ANTLRInputStream(filter));
    FilterParser p = new FilterParser(new CommonTokenStream(l));
    p.setBuildParseTree(true);/*from   w  w  w .ja v  a2  s .com*/

    p.addErrorListener(new BaseErrorListener() {
        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
                int charPositionInLine, String msg, RecognitionException e) {
            throw new IllegalStateException(
                    "failed to parse at line " + line + ":" + charPositionInLine + " due to " + msg, e);
        }
    });

    try {
        ParseTree tree = p.filter();
        ExpressionBuildingListener expListener = new ExpressionBuildingListener();
        ParseTreeWalker.DEFAULT.walk(expListener, tree);

        return expListener.getFilterExpression();
    } catch (IllegalStateException e) {
        throw new FilterParseException("Failed to parse filter: " + filter, e);
    }
}

From source file:edu.psu.swe.scim.spec.resources.PhoneNumber.java

License:Apache License

public void setValue(String value) throws PhoneNumberParseException {
    if (value == null) {
        throw new PhoneNumberParseException("null values are illegal for phone numbers");
    }// w w w  .  j a va 2s.c  o  m

    if (strict) {
        PhoneNumberLexer phoneNumberLexer = new PhoneNumberLexer(new ANTLRInputStream(value));
        PhoneNumberParser p = new PhoneNumberParser(new CommonTokenStream(phoneNumberLexer));
        p.setBuildParseTree(true);

        p.addErrorListener(new BaseErrorListener() {
            @Override
            public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
                    int charPositionInLine, String msg, RecognitionException e) {
                throw new IllegalStateException("failed to parse at line " + line + " due to " + msg, e);
            }
        });

        PhoneNumberParseTreeListener tpl = new PhoneNumberParseTreeListener();
        try {
            ParseTree tree = p.phoneNumber();
            ParseTreeWalker.DEFAULT.walk(tpl, tree);
        } catch (IllegalStateException e) {
            throw new PhoneNumberParseException(e);
        }

        PhoneNumber parsedPhoneNumber = tpl.getPhoneNumber();

        this.value = parsedPhoneNumber.getValue();
        this.number = parsedPhoneNumber.getNumber();
        this.extension = parsedPhoneNumber.getExtension();
        this.subAddress = parsedPhoneNumber.getSubAddress();
        this.phoneContext = parsedPhoneNumber.getPhoneContext();
        this.params = parsedPhoneNumber.getParams();
        this.isGlobalNumber = parsedPhoneNumber.isGlobalNumber();
        this.isDomainPhoneContext = parsedPhoneNumber.isDomainPhoneContext();
    } else {
        this.value = value;
    }
}