Example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream

Introduction

In this page you can find the example usage for org.antlr.v4.runtime CommonTokenStream CommonTokenStream.

Prototype

public CommonTokenStream(TokenSource tokenSource)

Source Link

Document

Constructs a new CommonTokenStream using the specified token source and the default token channel ( Token#DEFAULT_CHANNEL ).

Usage

From source file:org.ballerinalang.composer.service.workspace.rest.datamodel.BLangFragmentParser.java

License:Open Source License

protected static JsonObject getJsonModel(String source) throws IOException {
    ByteArrayInputStream inputStream = new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8));
    ANTLRInputStream antlrInputStream = new ANTLRInputStream(inputStream);
    BallerinaLexer ballerinaLexer = new BallerinaLexer(antlrInputStream);
    CommonTokenStream ballerinaToken = new CommonTokenStream(ballerinaLexer);

    BallerinaParser ballerinaParser = new BallerinaParser(ballerinaToken);
    BallerinaComposerErrorStrategy errorStrategy = new BallerinaComposerErrorStrategy();
    ballerinaParser.setErrorHandler(errorStrategy);

    GlobalScope globalScope = GlobalScope.getInstance();
    BTypes.loadBuiltInTypes(globalScope);
    BLangPackage bLangPackage = new BLangPackage(globalScope);
    BLangPackage.PackageBuilder packageBuilder = new BLangPackage.PackageBuilder(bLangPackage);
    BallerinaComposerModelBuilder bLangModelBuilder = new BallerinaComposerModelBuilder(packageBuilder,
            StringUtils.EMPTY);//from  w  w w . jav  a 2  s. c  o m
    BLangAntlr4Listener ballerinaBaseListener = new BLangAntlr4Listener(true, ballerinaToken, bLangModelBuilder,
            new File(TEMP_UNTITLED).toPath());
    ballerinaParser.addParseListener(ballerinaBaseListener);
    ballerinaParser.compilationUnit();
    BallerinaFile bFile = bLangModelBuilder.build();

    JsonObject jsonModelRoot = new JsonObject();
    JsonArray errors = new JsonArray();
    for (SyntaxError error : errorStrategy.getErrorTokens()) {
        // reduce number of lines in wrapper function from row count
        error.setRow(error.getRow() - 1);
        errors.add(error.toJson());
    }

    if (errorStrategy.getErrorTokens().isEmpty()
            /* if parser recovered the error */
            || (bFile.getCompilationUnits().length > 0 && (((BallerinaFunction) bFile.getCompilationUnits()[0])
                    .getCallableUnitBody().getStatements().length == 1)
            /* if the only recovered error is the additional semicolon error*/
                    && errorStrategy.getErrorTokens().size() == 1
                    && errorStrategy.getErrorTokens().get(0).getText().contains("unwanted token ';'"))) {
        BLangJSONModelBuilder jsonModelBuilder = new BLangJSONModelBuilder(jsonModelRoot);
        bFile.accept(jsonModelBuilder);
    } else {
        jsonModelRoot.add(SYNTAX_ERRORS, errors);
    }

    return jsonModelRoot;
}

From source file:org.ballerinalang.composer.service.workspace.suggetions.AutoCompleteSuggesterImpl.java

License:Open Source License

@Override
public BallerinaFile getBallerinaFile(BFile bFile, Position cursorPosition,
        CapturePossibleTokenStrategy capturePossibleTokenStrategy) throws IOException {
    ByteArrayInputStream inputStream = new ByteArrayInputStream(
            bFile.getContent().getBytes(StandardCharsets.UTF_8));
    ANTLRInputStream antlrInputStream = new ANTLRInputStream(inputStream);
    BallerinaLexer ballerinaLexer = new BallerinaLexer(antlrInputStream);
    CommonTokenStream ballerinaToken = new CommonTokenStream(ballerinaLexer);

    BallerinaParser ballerinaParser = new BallerinaParser(ballerinaToken);
    ballerinaParser.setErrorHandler(capturePossibleTokenStrategy);

    GlobalScope globalScope = GlobalScope.getInstance();
    BTypes.loadBuiltInTypes(globalScope);
    BLangPackage bLangPackage = new BLangPackage(globalScope);
    BLangPackage.PackageBuilder packageBuilder = new BLangPackage.PackageBuilder(bLangPackage);
    BallerinaComposerModelBuilder bLangModelBuilder = new BallerinaComposerModelBuilder(packageBuilder,
            StringUtils.EMPTY);//from w w w  .  ja  va 2  s. c  o m
    BLangAntlr4Listener ballerinaBaseListener = new BLangAntlr4Listener(true, ballerinaToken, bLangModelBuilder,
            new File(bFile.getFileName()).toPath());
    ballerinaParser.addParseListener(ballerinaBaseListener);
    ballerinaParser.compilationUnit();
    BallerinaFile ballerinaFile = bLangModelBuilder.build();
    return ballerinaFile;
}

From source file:org.ballerinalang.composer.service.workspace.swagger.SwaggerConverterUtils.java

License:Open Source License

/**
 * Generate ballerina fine from the String definition
 *
 * @param ballerinaDefinition//from ww  w.ja  v  a2  s.  co m
 * @return
 * @throws IOException
 */
public static BallerinaFile getBFileFromBallerinaDefinition(String ballerinaDefinition) throws IOException {
    //TODO this method need to replaced with the utility provided by ballerina core.
    ANTLRInputStream antlrInputStream = new ANTLRInputStream(ballerinaDefinition);
    BallerinaLexer ballerinaLexer = new BallerinaLexer(antlrInputStream);
    CommonTokenStream ballerinaToken = new CommonTokenStream(ballerinaLexer);
    BallerinaParser ballerinaParser = new BallerinaParser(ballerinaToken);
    ballerinaParser.setErrorHandler(new BallerinaComposerErrorStrategy());
    GlobalScope globalScope = GlobalScope.getInstance();
    BTypes.loadBuiltInTypes(globalScope);
    BLangPackage bLangPackage = new BLangPackage(globalScope);
    BLangPackage.PackageBuilder packageBuilder = new BLangPackage.PackageBuilder(bLangPackage);
    BallerinaComposerModelBuilder bLangModelBuilder = new BallerinaComposerModelBuilder(packageBuilder,
            StringUtils.EMPTY);
    BLangAntlr4Listener ballerinaBaseListener = new BLangAntlr4Listener(bLangModelBuilder);
    ballerinaParser.addParseListener(ballerinaBaseListener);
    ballerinaParser.compilationUnit();
    BallerinaFile bFile = bLangModelBuilder.build();
    return bFile;
}

From source file:org.ballerinalang.langserver.completions.CompletionSubRuleParser.java

License:Open Source License

private static BallerinaParser getParser(LSContext context, String rule) {
    ANTLRInputStream inputStream = new ANTLRInputStream(rule);
    BallerinaLexer lexer = new BallerinaLexer(inputStream);
    CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
    BallerinaParser parser = new BallerinaParser(commonTokenStream);
    parser.setErrorHandler(new SubRuleParserErrorStrategy(context));

    return parser;
}

From source file:org.ballerinalang.langserver.util.references.ReferencesSubRuleParser.java

License:Open Source License

public static void parserCompilationUnit(String content, LSContext context, Position position) {

    // TODO: 1/23/19 Check what happens when the content is not a valid compilation unit and when there are errors
    ANTLRInputStream inputStream = new ANTLRInputStream(content);
    BallerinaLexer lexer = new BallerinaLexer(inputStream);
    CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
    BallerinaParser parser = new BallerinaParser(commonTokenStream);
    parser.setErrorHandler(new ReferenceFindTokenErrorStrategy(context, position));

    parser.compilationUnit();/*from ww  w .  j  a v  a2  s.co  m*/
}

From source file:org.ballerinalang.launcher.LauncherUtils.java

License:Open Source License

static BallerinaFile buildLangModel(Path sourceFilePath) {
    ANTLRInputStream antlrInputStream = getAntlrInputStream(sourceFilePath);

    try {/*from ww w.  j  a  va  2 s.c  om*/
        // Setting the name of the source file being parsed, to the ANTLR input stream.
        // This is required by the parser-error strategy.
        antlrInputStream.name = getFileName(sourceFilePath);

        BallerinaLexer ballerinaLexer = new BallerinaLexer(antlrInputStream);
        CommonTokenStream ballerinaToken = new CommonTokenStream(ballerinaLexer);

        BallerinaParser ballerinaParser = new BallerinaParser(ballerinaToken);
        ballerinaParser.setErrorHandler(new BallerinaParserErrorStrategy());

        GlobalScope globalScope = GlobalScope.getInstance();
        loadGlobalSymbols(globalScope);
        BLangPackage bLangPackage = new BLangPackage(globalScope);

        BLangModelBuilder bLangModelBuilder = new BLangModelBuilder(bLangPackage);
        BLangAntlr4Listener ballerinaBaseListener = new BLangAntlr4Listener(bLangModelBuilder);
        ballerinaParser.addParseListener(ballerinaBaseListener);
        ballerinaParser.compilationUnit();
        BallerinaFile balFile = bLangModelBuilder.build();

        BuiltInNativeConstructLoader.loadConstructs(globalScope);

        SemanticAnalyzer semanticAnalyzer = new SemanticAnalyzer(balFile, bLangPackage);
        balFile.accept(semanticAnalyzer);

        return balFile;
    } catch (ParseCancellationException | SemanticException | LinkerException e) {
        throw createLauncherException(makeFirstLetterUpperCase(e.getMessage()));
    } catch (Throwable e) {
        throw createLauncherException(
                getFileName(sourceFilePath) + ": " + makeFirstLetterUpperCase(e.getMessage()));
    }
}

From source file:org.ballerinalang.toml.util.TomlProcessor.java

License:Open Source License

/**
 * Generate the proxy object by passing in the toml file.
 *
 * @param stream charstream object containing the content
 * @return proxy object//  ww  w.  ja  v  a 2s. c  o m
 */
public static ParseTree parseTomlContent(CharStream stream) {
    TomlLexer lexer = new TomlLexer(stream);

    // Get a list of matched tokens
    CommonTokenStream tokens = new CommonTokenStream(lexer);

    // Pass the tokens to the parser
    TomlParser parser = new TomlParser(tokens);
    return parser.toml();
}

From source file:org.ballerinalang.util.program.BLangFiles.java

License:Open Source License

public static BallerinaFile loadFile(String sourceFileName, Path packagePath, InputStream inputStream,
        BLangPackage.PackageBuilder packageBuilder) {

    Path sourceFilePath;//from  w w w.j  a  v a 2 s  .  c  o  m
    if (packagePath == null || packagePath.toString().equals(".")) {
        sourceFilePath = Paths.get(sourceFileName);
    } else {
        sourceFilePath = packagePath.resolve(sourceFileName);
    }

    try {
        ANTLRInputStream antlrInputStream = new ANTLRInputStream(inputStream);

        // Setting the name of the source file being parsed, to the ANTLR input stream.
        // This is required by the parser-error strategy.
        antlrInputStream.name = sourceFilePath.toString();

        BallerinaLexer ballerinaLexer = new BallerinaLexer(antlrInputStream);
        CommonTokenStream ballerinaToken = new CommonTokenStream(ballerinaLexer);

        BallerinaParser ballerinaParser = new BallerinaParser(ballerinaToken);
        ballerinaParser.setErrorHandler(new BallerinaParserErrorStrategy());

        BLangModelBuilder bLangModelBuilder = new BLangModelBuilder(packageBuilder, sourceFileName);
        BLangAntlr4Listener antlr4Listener = new BLangAntlr4Listener(bLangModelBuilder, sourceFilePath);
        ballerinaParser.addParseListener(antlr4Listener);
        ballerinaParser.compilationUnit();
        return bLangModelBuilder.build();

    } catch (IOException e) {
        throw new IllegalStateException(
                "error in reading source file '" + sourceFilePath + "': " + e.getMessage());

    } catch (ParseCancellationException e) {
        throw new ParserException(e.getMessage(), e);
    }
}