Example usage for org.antlr.v4.runtime TokenStream size

List of usage examples for org.antlr.v4.runtime TokenStream size

  1. HOME
  2. Java
  3. org.antlr
  4. org.antlr.v4.runtime.*
  5. TokenStream
  6. size

Introduction

In this page you can find the example usage for org.antlr.v4.runtime TokenStream size.

Prototype

int size();

Source Link

Document

Returns the total number of symbols in the stream, including a single EOF symbol.

Usage

From source file:com.huawei.streaming.cql.semanticanalyzer.parser.CQLErrorStrategy.java

License:Apache License

@NotNull
private String getText(TokenStream tokens, Interval interval) {
    int start = interval.a;
    int stop = interval.b;
    if (start < 0 || stop < 0)
        return "";

    if (stop >= tokens.size())
        stop = tokens.size() - 1;/* w w w .j  av  a 2s  .  c o m*/

    StringBuilder buf = new StringBuilder();
    for (int i = start; i <= stop; i++) {
        Token t = tokens.get(i);
        if (t.getType() == Token.EOF)
            break;
        buf.append(t.getText());
        if (i != stop) {
            buf.append(" ");
        }
    }
    return buf.toString();
}

From source file:com.satisfyingstructures.J2S.J2SRewriter.java

License:Open Source License

static String discoverLineBreakType(TokenStream tokens) {
    String s = "\n";
    for (int i = 0, sz = tokens.size(); i < sz; i++) {
        Token t = tokens.get(i);// ww w. ja  v  a 2s .c  o m
        if (null == t || t.getType() != Java8Parser.LB)
            continue;
        s = t.getText();
        if (-1 != (i = s.indexOf(s.charAt(0), 1))) // remove any later repetitions
            s = s.substring(0, i);
        break;
    }
    return s;
}

From source file:com.satisfyingstructures.J2S.J2SRewriter.java

License:Open Source License

static String discoverSingelIndentType(TokenStream tokens) {
    String singleIndent = "\t";
    int depth = 0;
    List<Token> recentIndentsByDepth = new ArrayList<>();
    Map<String, Integer> countOfIndentStyle = new HashMap<>();
    int countOfIndents = 0;
    for (int i = 0, sz = tokens.size(); i < sz; i++) {
        Token t = tokens.get(i);//ww  w .  j a va2s  . com
        if (null == t)
            continue;
        switch (t.getType()) {
        case Java8Parser.RBRACE:
            break;
        case Java8Parser.LBRACE:
            depth++;
            for (int j = recentIndentsByDepth.size(); j <= depth; j++)
                recentIndentsByDepth.add(j, null);
        default:
            continue;
        }
        if (i > 2) {
            Token tWS = tokens.get(i - 1);
            if (tWS.getType() == Java8Parser.WS && tokens.get(i - 2).getType() == Java8Parser.LB) {
                recentIndentsByDepth.set(depth, tWS);
                if (depth + 1 < recentIndentsByDepth.size()
                        && null != (t = recentIndentsByDepth.get(depth + 1))) {
                    String deep = tWS.getText();
                    String deeper = t.getText();
                    if (deeper.startsWith(deep)) {
                        singleIndent = deeper.substring(deep.length());
                        Integer count = countOfIndentStyle.get(singleIndent);
                        if (null == count)
                            countOfIndentStyle.put(singleIndent, 1);
                        else {
                            float share = count.floatValue() / (float) countOfIndents;
                            if (countOfIndents >= 4 && share == 1)
                                return singleIndent; // winner, consistent use
                            if (countOfIndents > 10 && share > .9)
                                return singleIndent; // winner, inconsistent use
                            if (countOfIndents > 20 && share > .7)
                                return singleIndent; // winner, variable use
                            countOfIndentStyle.put(singleIndent, count + 1);
                        }
                        countOfIndents++;
                    }
                    recentIndentsByDepth.set(depth + 1, null);
                }
            }
        }
        depth--;
    }
    // No early winner, so pick most frequent
    int best = 0;
    for (Map.Entry<String, Integer> indentStyle : countOfIndentStyle.entrySet()) {
        int count = indentStyle.getValue();
        countOfIndents -= count; // --> remaining
        if (best < count) {
            best = count;
            singleIndent = indentStyle.getKey();
            if (countOfIndents < count)
                return singleIndent; // cant be beaten now
        }
    }
    return singleIndent;
}

From source file:com.twosigma.beaker.groovy.autocomplete.GrammarPredicates.java

License:Apache License

public static boolean isClassName(TokenStream _input) {
    try {//www  .  ja  v a  2 s .c  om
        int i = 1;
        Token token = _input.LT(i);
        while (token != null && i < _input.size() && _input.LT(i + 1).getType() == GroovyParser.DOT) {
            i = i + 2;
            token = _input.LT(i);
        }
        if (token == null)
            return false;
        // TODO here
        return Character.isUpperCase(Character.codePointAt(token.getText(), 0));
    } catch (Exception e) {
        e.printStackTrace();
    }

    return false;
}

From source file:eu.mihosoft.vrl.licenseheaderutil.ChangeLicenseHeaderListener.java

License:Open Source License

@Override
public void enterCompilationUnit(JavaParser.CompilationUnitContext ctx) {
    TokenStream tokens = parser.getTokenStream();

    code = "";/*from   ww w  .j a  v a  2 s  . com*/

    if (ctx.packageDeclaration() != null) {
        code = tokens.getText(ctx.packageDeclaration().start, tokens.get(tokens.size() - 1));
        hasPackage = true;
    } else if (!ctx.importDeclaration().isEmpty()) {
        code = tokens.getText(ctx.importDeclaration(0).start, tokens.get(tokens.size() - 1));
        hasPackage = false;
    } else if (!ctx.typeDeclaration().isEmpty()) {
        code = tokens.getText(ctx.typeDeclaration(0).start, tokens.get(tokens.size() - 1));
        hasPackage = false;
    }

}

From source file:net.certiv.json.parser.JsonErrorListener.java

License:Open Source License

@Override
public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine,
        String msg, RecognitionException e) {

    Parser parser = (Parser) recognizer;
    String name = parser.getSourceName();
    TokenStream tokens = parser.getInputStream();

    Token offSymbol = (Token) offendingSymbol;
    int thisError = offSymbol.getTokenIndex();
    if (offSymbol.getType() == -1 && thisError == tokens.size() - 1) {
        Log.debug(this, name + ": Incorrect error: " + msg);
        return;/* w  w  w.jav a2s.  c  o m*/
    }
    String offSymName = JsonLexer.VOCABULARY.getSymbolicName(offSymbol.getType());
    if (thisError > lastError + 10) {
        lastError = thisError - 10;
    }
    for (int idx = lastError + 1; idx <= thisError; idx++) {
        Token token = tokens.get(idx);
        if (token.getChannel() != Token.HIDDEN_CHANNEL)
            Log.error(this, name + ":" + token.toString());
    }
    lastError = thisError;

    List<String> stack = parser.getRuleInvocationStack();
    Collections.reverse(stack);

    Log.error(this, name + " rule stack: " + stack);
    Log.error(this, name + " line " + line + ":" + charPositionInLine + " at " + offSymName + ": " + msg);
}

From source file:org.ballerinalang.composer.service.workspace.langserver.util.completion.filters.PackageActionAndFunctionFilter.java

License:Open Source License

/**
 * Get the index of a certain token/*from  w ww .  j a  v a  2 s . co m*/
 * @param tokenString - token string
 * @param from - start searching from
 * @param dataModel - suggestions filter data model
 * @return {@link Integer}
 */
public int getIndexOfTokenString(String tokenString, int from, SuggestionsFilterDataModel dataModel) {
    TokenStream tokenStream = dataModel.getTokenStream();
    int resultTokenIndex = -1;
    int searchIndex = from;

    while (true) {
        if (searchIndex < 0 || tokenStream.size() - 1 < searchIndex) {
            break;
        }
        Token token = tokenStream.get(searchIndex);
        if (token.getChannel() != Token.DEFAULT_CHANNEL || !token.getText().equals(tokenString)) {
            searchIndex++;
        } else {
            resultTokenIndex = searchIndex;
            break;
        }
    }

    return resultTokenIndex;
}

From source file:org.ballerinalang.composer.service.workspace.langserver.util.completion.filters.PackageActionAndFunctionFilter.java

License:Open Source License

private int getPackageDelimeterTokenIndex(SuggestionsFilterDataModel dataModel) {
    ArrayList<String> terminalTokens = new ArrayList<>(Arrays.asList(new String[] { ";", "}", "{" }));
    int currentTokenIndex = dataModel.getTokenIndex();
    int searchTokenIndex = currentTokenIndex;
    TokenStream tokenStream = dataModel.getTokenStream();
    int delimiterIndex = -1;
    String currentTokenStr = tokenStream.get(searchTokenIndex).getText();

    if (terminalTokens.contains(currentTokenStr)) {
        searchTokenIndex -= 1;//from  ww w.  ja  v  a  2  s .c  o  m
        while (true) {
            if (tokenStream.get(searchTokenIndex).getChannel() == Token.DEFAULT_CHANNEL) {
                break;
            } else {
                searchTokenIndex -= 1;
            }
        }
    }

    while (true) {
        if (searchTokenIndex >= tokenStream.size()) {
            break;
        }
        String tokenString = tokenStream.get(searchTokenIndex).getText();
        if (".".equals(tokenString) || ":".equals(tokenString)) {
            delimiterIndex = searchTokenIndex;
            break;
        } else if (terminalTokens.contains(tokenString)) {
            break;
        } else {
            searchTokenIndex++;
        }
    }

    return delimiterIndex;
}

From source file:org.ballerinalang.composer.service.workspace.langserver.util.completion.resolvers.AbstractItemResolver.java

License:Open Source License

/**
 * Check whether the token stream corresponds to a action invocation or a function invocation
 * @param dataModel - Suggestions filter data model
 * @return {@link Boolean}//  w ww .ja va2  s . c  o m
 */
protected boolean isActionOrFunctionInvocationStatement(SuggestionsFilterDataModel dataModel) {
    ArrayList<String> terminalTokens = new ArrayList<>(Arrays.asList(new String[] { ";", "}", "{" }));
    TokenStream tokenStream = dataModel.getTokenStream();
    int searchTokenIndex = dataModel.getTokenIndex();
    String currentTokenStr = tokenStream.get(searchTokenIndex).getText();

    if (terminalTokens.contains(currentTokenStr)) {
        searchTokenIndex -= 1;
        while (true) {
            if (tokenStream.get(searchTokenIndex).getChannel() == Token.DEFAULT_CHANNEL) {
                break;
            } else {
                searchTokenIndex -= 1;
            }
        }
    }

    while (true) {
        if (searchTokenIndex >= tokenStream.size()) {
            return false;
        }
        String tokenString = tokenStream.get(searchTokenIndex).getText();
        if (terminalTokens.contains(tokenString)) {
            return false;
        } else if (tokenString.equals(".") || tokenString.equals(":")) {
            return true;
        } else {
            searchTokenIndex++;
        }
    }
}

From source file:org.ballerinalang.composer.service.workspace.langserver.util.filters.PackageActionAndFunctionFilter.java

License:Open Source License

/**
 * Get the index of a certain token/*from   www . j  a  v a2 s . com*/
 * @param tokenString - token string
 * @param from - start searching from
 * @param dataModel - suggestions filter data model
 * @return {@link Integer}
 */
private int getIndexOfTokenString(String tokenString, int from, SuggestionsFilterDataModel dataModel) {
    TokenStream tokenStream = dataModel.getTokenStream();
    int resultTokenIndex = -1;
    int searchIndex = from;

    while (true) {
        if (searchIndex < 0 || tokenStream.size() - 1 < searchIndex) {
            break;
        }
        Token token = tokenStream.get(searchIndex);
        if (token.getChannel() != Token.DEFAULT_CHANNEL || !token.getText().equals(tokenString)) {
            searchIndex++;
        } else {
            resultTokenIndex = searchIndex;
            break;
        }
    }

    return resultTokenIndex;
}