Example usage for org.antlr.v4.runtime TokenStream get

Introduction

In this page you can find the example usage for org.antlr.v4.runtime TokenStream get.

Prototype

public Token get(int index);

Source Link

Document

Gets the Token at the specified index in the stream.

Usage

From source file:com.huawei.streaming.cql.semanticanalyzer.parser.CQLErrorStrategy.java

License:Apache License

@NotNull
private String getText(TokenStream tokens, Interval interval) {
    int start = interval.a;
    int stop = interval.b;
    if (start < 0 || stop < 0)
        return "";

    if (stop >= tokens.size())
        stop = tokens.size() - 1;// www. ja  va  2 s  . c  om

    StringBuilder buf = new StringBuilder();
    for (int i = start; i <= stop; i++) {
        Token t = tokens.get(i);
        if (t.getType() == Token.EOF)
            break;
        buf.append(t.getText());
        if (i != stop) {
            buf.append(" ");
        }
    }
    return buf.toString();
}

---

From source file:com.satisfyingstructures.J2S.J2SRewriter.java

License:Open Source License

static String discoverLineBreakType(TokenStream tokens) {
    String s = "\n";
    for (int i = 0, sz = tokens.size(); i < sz; i++) {
        Token t = tokens.get(i);
        if (null == t || t.getType() != Java8Parser.LB)
            continue;
        s = t.getText();/*from  ww w. j  a v a  2  s.  c o  m*/
        if (-1 != (i = s.indexOf(s.charAt(0), 1))) // remove any later repetitions
            s = s.substring(0, i);
        break;
    }
    return s;
}

---

From source file:com.satisfyingstructures.J2S.J2SRewriter.java

License:Open Source License

static String discoverSingelIndentType(TokenStream tokens) {
    String singleIndent = "\t";
    int depth = 0;
    List<Token> recentIndentsByDepth = new ArrayList<>();
    Map<String, Integer> countOfIndentStyle = new HashMap<>();
    int countOfIndents = 0;
    for (int i = 0, sz = tokens.size(); i < sz; i++) {
        Token t = tokens.get(i);
        if (null == t)
            continue;
        switch (t.getType()) {
        case Java8Parser.RBRACE:
            break;
        case Java8Parser.LBRACE:
            depth++;//  www  .  j  av  a  2  s .c o m
            for (int j = recentIndentsByDepth.size(); j <= depth; j++)
                recentIndentsByDepth.add(j, null);
        default:
            continue;
        }
        if (i > 2) {
            Token tWS = tokens.get(i - 1);
            if (tWS.getType() == Java8Parser.WS && tokens.get(i - 2).getType() == Java8Parser.LB) {
                recentIndentsByDepth.set(depth, tWS);
                if (depth + 1 < recentIndentsByDepth.size()
                        && null != (t = recentIndentsByDepth.get(depth + 1))) {
                    String deep = tWS.getText();
                    String deeper = t.getText();
                    if (deeper.startsWith(deep)) {
                        singleIndent = deeper.substring(deep.length());
                        Integer count = countOfIndentStyle.get(singleIndent);
                        if (null == count)
                            countOfIndentStyle.put(singleIndent, 1);
                        else {
                            float share = count.floatValue() / (float) countOfIndents;
                            if (countOfIndents >= 4 && share == 1)
                                return singleIndent; // winner, consistent use
                            if (countOfIndents > 10 && share > .9)
                                return singleIndent; // winner, inconsistent use
                            if (countOfIndents > 20 && share > .7)
                                return singleIndent; // winner, variable use
                            countOfIndentStyle.put(singleIndent, count + 1);
                        }
                        countOfIndents++;
                    }
                    recentIndentsByDepth.set(depth + 1, null);
                }
            }
        }
        depth--;
    }
    // No early winner, so pick most frequent
    int best = 0;
    for (Map.Entry<String, Integer> indentStyle : countOfIndentStyle.entrySet()) {
        int count = indentStyle.getValue();
        countOfIndents -= count; // --> remaining
        if (best < count) {
            best = count;
            singleIndent = indentStyle.getKey();
            if (countOfIndents < count)
                return singleIndent; // cant be beaten now
        }
    }
    return singleIndent;
}

---

From source file:eu.mihosoft.vrl.licenseheaderutil.ChangeLicenseHeaderListener.java

License:Open Source License

@Override
public void enterCompilationUnit(JavaParser.CompilationUnitContext ctx) {
    TokenStream tokens = parser.getTokenStream();

    code = "";//from  ww w.j  ava 2s  .c o  m

    if (ctx.packageDeclaration() != null) {
        code = tokens.getText(ctx.packageDeclaration().start, tokens.get(tokens.size() - 1));
        hasPackage = true;
    } else if (!ctx.importDeclaration().isEmpty()) {
        code = tokens.getText(ctx.importDeclaration(0).start, tokens.get(tokens.size() - 1));
        hasPackage = false;
    } else if (!ctx.typeDeclaration().isEmpty()) {
        code = tokens.getText(ctx.typeDeclaration(0).start, tokens.get(tokens.size() - 1));
        hasPackage = false;
    }

}

---

From source file:io.mxnet.caffetranslator.CreateModelListener.java

License:Apache License

private String getPrototxt(TokenStream stream, int start, int end) {
    StringBuilder prototxt = new StringBuilder();
    for (int i = start; i <= end; i++) {
        Token token = stream.get(i);
        prototxt.append(token.getText());
    }//w w  w  .  j a v  a  2  s .  c o m
    String strPrototxt = prototxt.toString();
    return strPrototxt.replaceAll(" +num_examples:.*\\s", "");
}

---

From source file:net.certiv.json.parser.JsonErrorListener.java

License:Open Source License

@Override
public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine,
        String msg, RecognitionException e) {

    Parser parser = (Parser) recognizer;
    String name = parser.getSourceName();
    TokenStream tokens = parser.getInputStream();

    Token offSymbol = (Token) offendingSymbol;
    int thisError = offSymbol.getTokenIndex();
    if (offSymbol.getType() == -1 && thisError == tokens.size() - 1) {
        Log.debug(this, name + ": Incorrect error: " + msg);
        return;//from   ww  w.  j a  v  a2 s . c om
    }
    String offSymName = JsonLexer.VOCABULARY.getSymbolicName(offSymbol.getType());
    if (thisError > lastError + 10) {
        lastError = thisError - 10;
    }
    for (int idx = lastError + 1; idx <= thisError; idx++) {
        Token token = tokens.get(idx);
        if (token.getChannel() != Token.HIDDEN_CHANNEL)
            Log.error(this, name + ":" + token.toString());
    }
    lastError = thisError;

    List<String> stack = parser.getRuleInvocationStack();
    Collections.reverse(stack);

    Log.error(this, name + " rule stack: " + stack);
    Log.error(this, name + " line " + line + ":" + charPositionInLine + " at " + offSymName + ": " + msg);
}

---

From source file:org.ballerinalang.composer.service.workspace.langserver.util.completion.filters.PackageActionAndFunctionFilter.java

License:Open Source License

@Override
public List<SymbolInfo> filterItems(SuggestionsFilterDataModel dataModel, ArrayList<SymbolInfo> symbols,
        HashMap<String, Object> properties) {

    TokenStream tokenStream = dataModel.getTokenStream();
    int delimiterIndex = this.getPackageDelimeterTokenIndex(dataModel);
    String delimiter = tokenStream.get(delimiterIndex).getText();
    ArrayList<SymbolInfo> returnSymbolsInfoList = new ArrayList<>();

    if (".".equals(delimiter)) {
        // If the delimiter is "." then we are filtering the bound functions for the structs
        returnSymbolsInfoList.addAll(this.getBoundActionAndFunctions(dataModel, symbols, delimiterIndex));
    } else if (":".equals(delimiter)) {
        // We are filtering the package functions
        returnSymbolsInfoList.addAll(this.getActionsAndFunctions(dataModel, symbols, delimiterIndex));
    }/* www  .  j  av a 2 s .  c  o m*/

    return returnSymbolsInfoList;
}

---

From source file:org.ballerinalang.composer.service.workspace.langserver.util.completion.filters.PackageActionAndFunctionFilter.java

License:Open Source License

private ArrayList<SymbolInfo> getActionsAndFunctions(SuggestionsFilterDataModel dataModel,
        ArrayList<SymbolInfo> symbols, int delimiterIndex) {

    ArrayList<SymbolInfo> actionFunctionList = new ArrayList<>();
    TokenStream tokenStream = dataModel.getTokenStream();
    String packageName = tokenStream.get(delimiterIndex - 1).getText();

    SymbolInfo packageSymbolInfo = symbols.stream().filter(item -> {
        Scope.ScopeEntry scopeEntry = item.getScopeEntry();
        return item.getSymbolName().equals(packageName) && scopeEntry.symbol instanceof BPackageSymbol;
    }).findFirst().orElse(null);/*from  w w  w  .j a  v a2 s. com*/

    if (packageSymbolInfo != null) {
        Scope.ScopeEntry packageEntry = packageSymbolInfo.getScopeEntry();
        SymbolInfo symbolInfo = new SymbolInfo(packageSymbolInfo.getSymbolName(), packageEntry);

        symbolInfo.getScopeEntry().symbol.scope.entries.forEach((name, value) -> {
            if (value.symbol instanceof BInvokableSymbol
                    && ((BInvokableSymbol) value.symbol).receiverSymbol == null) {
                SymbolInfo actionFunctionSymbol = new SymbolInfo(name.toString(), value);
                actionFunctionList.add(actionFunctionSymbol);
            }
        });
    }

    return actionFunctionList;
}

---

From source file:org.ballerinalang.composer.service.workspace.langserver.util.completion.filters.PackageActionAndFunctionFilter.java

License:Open Source License

private ArrayList<SymbolInfo> getBoundActionAndFunctions(SuggestionsFilterDataModel dataModel,
        ArrayList<SymbolInfo> symbols, int delimiterIndex) {

    ArrayList<SymbolInfo> actionFunctionList = new ArrayList<>();
    TokenStream tokenStream = dataModel.getTokenStream();
    String variableName = tokenStream.get(delimiterIndex - 1).getText();
    SymbolInfo variable = this.getVariableByName(variableName, symbols);
    Map<Name, Scope.ScopeEntry> entries = null;

    if (variable == null) {
        return actionFunctionList;
    }/*from   ww  w  .  j  a  v a2 s  .c o m*/

    String packageID = variable.getScopeEntry().symbol.getType().tsymbol.pkgID.toString();
    String builtinPkgName = dataModel.getSymbolTable().builtInPackageSymbol.name.getValue();

    SymbolInfo packageSymbolInfo = symbols.stream().filter(item -> {
        Scope.ScopeEntry scopeEntry = item.getScopeEntry();
        return (scopeEntry.symbol instanceof BPackageSymbol)
                && scopeEntry.symbol.pkgID.toString().equals(packageID);
    }).findFirst().orElse(null);

    if (packageSymbolInfo == null && packageID.equals(builtinPkgName)) {
        entries = dataModel.getSymbolTable().builtInPackageSymbol.scope.entries;
    } else if (packageSymbolInfo != null) {
        entries = packageSymbolInfo.getScopeEntry().symbol.scope.entries;
    }

    if (entries != null) {
        entries.forEach((name, scopeEntry) -> {
            if (scopeEntry.symbol instanceof BInvokableSymbol
                    && ((BInvokableSymbol) scopeEntry.symbol).receiverSymbol != null) {
                String symbolBoundedName = ((BInvokableSymbol) scopeEntry.symbol).receiverSymbol
                        .getType().tsymbol.name.getValue();
                String checkValue = variable.getScopeEntry().symbol.getType().tsymbol.name.getValue();
                if (symbolBoundedName.equals(checkValue)) {
                    SymbolInfo actionFunctionSymbol = new SymbolInfo(name.toString(), scopeEntry);
                    actionFunctionList.add(actionFunctionSymbol);
                }
            }
        });
    }

    return actionFunctionList;
}

---

From source file:org.ballerinalang.composer.service.workspace.langserver.util.completion.filters.PackageActionAndFunctionFilter.java

License:Open Source License

/**
 * Get the index of a certain token//from ww  w .j  a  va2  s .com
 * @param tokenString - token string
 * @param from - start searching from
 * @param dataModel - suggestions filter data model
 * @return {@link Integer}
 */
public int getIndexOfTokenString(String tokenString, int from, SuggestionsFilterDataModel dataModel) {
    TokenStream tokenStream = dataModel.getTokenStream();
    int resultTokenIndex = -1;
    int searchIndex = from;

    while (true) {
        if (searchIndex < 0 || tokenStream.size() - 1 < searchIndex) {
            break;
        }
        Token token = tokenStream.get(searchIndex);
        if (token.getChannel() != Token.DEFAULT_CHANNEL || !token.getText().equals(tokenString)) {
            searchIndex++;
        } else {
            resultTokenIndex = searchIndex;
            break;
        }
    }

    return resultTokenIndex;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »