Example usage for org.antlr.v4.runtime Token getTokenIndex

Introduction

In this page you can find the example usage for org.antlr.v4.runtime Token getTokenIndex.

Prototype

int getTokenIndex();

Source Link

Document

An index from 0..n-1 of the token object in the input stream.

Usage

From source file:org.ballerinalang.langserver.completions.CompletionSubRuleParser.java

License:Open Source License

private static boolean fillAnnotationAttachmentMetaInfo(LSContext ctx) {
    if (ctx.get(CompletionKeys.TOKEN_STREAM_KEY) == null) {
        return false;
    }//from w  w w.j ava  2s .  c  o  m
    TokenStream tokenStream = ctx.get(CompletionKeys.TOKEN_STREAM_KEY);
    Position cursorPosition = ctx.get(DocumentServiceKeys.POSITION_KEY).getPosition();
    Queue<String> fieldsQueue = new LinkedList<>();
    ArrayList<String> terminalTokens = new ArrayList<>(
            Arrays.asList(UtilSymbolKeys.FUNCTION_KEYWORD_KEY, UtilSymbolKeys.SERVICE_KEYWORD_KEY,
                    UtilSymbolKeys.ENDPOINT_KEYWORD_KEY, UtilSymbolKeys.OPEN_PARENTHESES_KEY));
    int currentTokenIndex = CommonUtil.getCurrentTokenFromTokenStream(ctx);
    int startIndex = currentTokenIndex;
    int line = cursorPosition.getLine();
    int col = cursorPosition.getCharacter();
    String annotationName;
    String pkgAlias = "";

    while (true) {
        if (startIndex > tokenStream.size()
                || CommonUtil.getPreviousDefaultToken(tokenStream, startIndex) == null) {
            return false;
        }
        Token token = CommonUtil.getPreviousDefaultToken(tokenStream, startIndex);
        if (token == null || terminalTokens.contains(token.getText())) {
            return false;
        }
        if (token.getText().equals(UtilSymbolKeys.ANNOTATION_START_SYMBOL_KEY)) {
            // Breaks when we meet the first annotation start before the given start index
            break;
        } else if (token.getText().equals(UtilSymbolKeys.OPEN_BRACE_KEY)) {
            startIndex = token.getTokenIndex();
            // For each annotation found, capture the package alias
            if (UtilSymbolKeys.PKG_DELIMITER_KEYWORD
                    .equals(CommonUtil.getNthDefaultTokensToLeft(tokenStream, startIndex, 2).getText())) {
                pkgAlias = CommonUtil.getNthDefaultTokensToLeft(tokenStream, startIndex, 3).getText();
            }
        } else {
            startIndex = token.getTokenIndex();
        }
    }

    String tempTokenString = "";
    while (true) {
        if (startIndex > tokenStream.size()) {
            return false;
        }
        Token token = CommonUtil.getNextDefaultToken(tokenStream, startIndex);
        int tokenLine = token.getLine() - 1;
        int tokenCol = token.getCharPositionInLine();
        if (terminalTokens.contains(token.getText()) || line < tokenLine
                || (line == tokenLine - 1 && col - 1 < tokenCol)) {
            break;
        } else if (UtilSymbolKeys.OPEN_BRACE_KEY.equals(token.getText())) {
            fieldsQueue.add(tempTokenString);
        } else if (UtilSymbolKeys.CLOSE_BRACE_KEY.equals(token.getText())) {
            fieldsQueue.remove();
        } else if (!UtilSymbolKeys.PKG_DELIMITER_KEYWORD.equals(token.getText())) {
            tempTokenString = token.getText();
        }
        startIndex = token.getTokenIndex();
    }

    annotationName = fieldsQueue.poll();
    ctx.put(CompletionKeys.ANNOTATION_ATTACHMENT_META_KEY, new AnnotationAttachmentMetaInfo(annotationName,
            fieldsQueue, pkgAlias, getNextNodeTypeFlag(tokenStream, currentTokenIndex, ctx)));
    ctx.put(CompletionKeys.SYMBOL_ENV_NODE_KEY, new BLangAnnotationAttachment());

    return true;
}

---

From source file:org.ballerinalang.langserver.completions.CompletionSubRuleParser.java

License:Open Source License

private static AnnotationNodeKind getNextNodeTypeFlag(TokenStream tokenStream, int index, LSContext context) {
    AnnotationNodeKind nodeType = context.get(CompletionKeys.NEXT_NODE_KEY);
    Map<String, AnnotationNodeKind> flagsMap = new HashMap<>();
    flagsMap.put(UtilSymbolKeys.SERVICE_KEYWORD_KEY, AnnotationNodeKind.SERVICE);
    flagsMap.put(UtilSymbolKeys.RESOURCE_KEYWORD_KEY, AnnotationNodeKind.RESOURCE);

    while (true) {
        if (tokenStream == null || index >= tokenStream.size()) {
            break;
        }/*from  w  w w .j a va2  s.c  o m*/
        Token token = CommonUtil.getNextDefaultToken(tokenStream, index);
        if (token.getText().equals(UtilSymbolKeys.SEMI_COLON_SYMBOL_KEY)) {
            break;
        } else if (flagsMap.containsKey(token.getText())) {
            nodeType = flagsMap.get(token.getText());
            break;
        }
        index = token.getTokenIndex();
    }

    return nodeType;
}

---

From source file:org.ballerinalang.langserver.completions.CompletionSubRuleParser.java

License:Open Source License

private static void reCalculatePoppedTokensForTopLevel(LSContext ctx) {
    if (ctx.get(CompletionKeys.TOKEN_STREAM_KEY) == null) {
        return;//ww  w  .  j av  a  2s. c o  m
    }
    int currentTokenIndex = CommonUtil.getCurrentTokenFromTokenStream(ctx);
    TokenStream tokenStream = ctx.get(CompletionKeys.TOKEN_STREAM_KEY);
    Stack<Token> tokenStack = new Stack<>();
    while (true) {
        if (currentTokenIndex < 0) {
            break;
        }
        Token token = CommonUtil.getPreviousDefaultToken(tokenStream, currentTokenIndex);
        if (token == null) {
            return;
        }
        String tokenString = token.getText();
        tokenStack.push(token);
        if (tokenString.equals(ItemResolverConstants.SERVICE)
                || tokenString.equals(ItemResolverConstants.FUNCTION)
                || tokenString.equals(ItemResolverConstants.TYPE)
                || tokenString.equals(ItemResolverConstants.ENDPOINT)) {
            break;
        }
        currentTokenIndex = token.getTokenIndex();
    }
    Collections.reverse(tokenStack);
    ctx.put(CompletionKeys.FORCE_CONSUMED_TOKENS_KEY, tokenStack);
}

---

From source file:org.ballerinalang.langserver.completions.providers.subproviders.parsercontext.ParserRuleServiceDefinitionCompletionProvider.java

License:Open Source License

@Override
public List<CompletionItem> resolveItems(LSContext ctx) {
    List<CompletionItem> completionItems = new ArrayList<>();
    TokenStream tokenStream = ctx.get(CompletionKeys.TOKEN_STREAM_KEY);
    Stack<Token> poppedTokens = ctx.get(CompletionKeys.FORCE_CONSUMED_TOKENS_KEY);
    int startIndex = poppedTokens.peek().getTokenIndex() + 1;
    String stopToken = "";

    // Backtrack the tokens from the head of the popped tokens in order determine the cursor position
    tokenScanner: while (true) {
        Token token = CommonUtil.getPreviousDefaultToken(tokenStream, startIndex);
        String tokenString = token.getText();
        switch (tokenString) {
        case ItemResolverConstants.SERVICE:
        case ItemResolverConstants.ON:
        case ItemResolverConstants.NEW:
        case UtilSymbolKeys.PKG_DELIMITER_KEYWORD:
            stopToken = tokenString;/* w  w  w . j  ava2 s. c  o m*/
            break tokenScanner;
        default:
            break;
        }

        startIndex = token.getTokenIndex();
    }

    boolean isSnippet = ctx.get(CompletionKeys.CLIENT_CAPABILITIES_KEY).getCompletionItem().getSnippetSupport();

    switch (stopToken) {
    case ItemResolverConstants.ON: {
        // suggest all the visible, defined listeners
        List<SymbolInfo> filtered = this.filterListenerVariables(ctx.get(CompletionKeys.VISIBLE_SYMBOLS_KEY));
        completionItems.addAll(this.getCompletionItemList(filtered, ctx));
        completionItems.add(Snippet.KW_NEW.get().build(ctx, isSnippet));
        break;
    }
    case ItemResolverConstants.NEW: {
        List<SymbolInfo> filteredSymbols = this
                .filterListenerTypes(ctx.get(CompletionKeys.VISIBLE_SYMBOLS_KEY));
        completionItems.addAll(this.getCompletionItemList(filteredSymbols, ctx));
        completionItems.addAll(this.getPackagesCompletionItems(ctx));
        break;
    }
    case UtilSymbolKeys.PKG_DELIMITER_KEYWORD: {
        Either<List<CompletionItem>, List<SymbolInfo>> eitherList = SymbolFilters
                .get(DelimiterBasedContentFilter.class).filterItems(ctx);
        completionItems.addAll(this.getCompletionItemList(eitherList, ctx));
        break;
    }
    default: {
        // Fill the on keyword completion item
        completionItems.add(Snippet.KW_ON.get().build(ctx, isSnippet));
        break;
    }
    }

    return completionItems;
}

---

From source file:org.ballerinalang.langserver.completions.resolvers.parsercontext.ParserRuleServiceDefinitionContextResolver.java

License:Open Source License

@Override
public List<CompletionItem> resolveItems(LSServiceOperationContext ctx) {
    List<CompletionItem> completionItems = new ArrayList<>();
    TokenStream tokenStream = ctx.get(CompletionKeys.TOKEN_STREAM_KEY);
    Stack<Token> poppedTokens = ctx.get(CompletionKeys.FORCE_CONSUMED_TOKENS_KEY);
    int startIndex = poppedTokens.peek().getTokenIndex() + 1;
    String stopToken = "";

    // Backtrack the tokens from the head of the popped tokens in order determine the cursor position
    tokenScanner: while (true) {
        Token token = CommonUtil.getPreviousDefaultToken(tokenStream, startIndex);
        String tokenString = token.getText();
        switch (tokenString) {
        case ItemResolverConstants.SERVICE:
        case ItemResolverConstants.ON:
        case ItemResolverConstants.NEW:
        case UtilSymbolKeys.PKG_DELIMITER_KEYWORD:
            stopToken = tokenString;/* w  w  w  .ja v a  2 s  .  co  m*/
            break tokenScanner;
        default:
            break;
        }

        startIndex = token.getTokenIndex();
    }

    boolean isSnippet = ctx.get(CompletionKeys.CLIENT_CAPABILITIES_KEY).getCompletionItem().getSnippetSupport();

    switch (stopToken) {
    case ItemResolverConstants.ON: {
        // suggest all the visible, defined listeners
        List<SymbolInfo> filtered = this.filterListenerVariables(ctx.get(CompletionKeys.VISIBLE_SYMBOLS_KEY));
        completionItems.addAll(this.getCompletionItemList(filtered, ctx));
        completionItems.add(Snippet.KW_NEW.get().build(ctx, isSnippet));
        break;
    }
    case ItemResolverConstants.NEW: {
        List<SymbolInfo> filteredSymbols = this
                .filterListenerTypes(ctx.get(CompletionKeys.VISIBLE_SYMBOLS_KEY));
        completionItems.addAll(this.getCompletionItemList(filteredSymbols, ctx));
        completionItems.addAll(this.getPackagesCompletionItems(ctx));
        break;
    }
    case UtilSymbolKeys.PKG_DELIMITER_KEYWORD: {
        Either<List<CompletionItem>, List<SymbolInfo>> eitherList = SymbolFilters
                .get(DelimiterBasedContentFilter.class).filterItems(ctx);
        completionItems.addAll(this.getCompletionItemList(eitherList, ctx));
        break;
    }
    default: {
        // Fill the on keyword completion item
        completionItems.add(Snippet.KW_ON.get().build(ctx, isSnippet));
        break;
    }
    }

    return completionItems;
}

---

From source file:org.ballerinalang.langserver.completions.resolvers.parsercontext.ParserRuleTypeNameContextResolver.java

License:Open Source License

private static List<SymbolInfo> filterEndpointContextSymbolInfo(TextDocumentServiceContext context) {
    List<SymbolInfo> symbolInfos = context.get(CompletionKeys.VISIBLE_SYMBOLS_KEY);
    int currentTokenIndex = context.get(DocumentServiceKeys.TOKEN_INDEX_KEY) - 1;
    TokenStream tokenStream = context.get(DocumentServiceKeys.TOKEN_STREAM_KEY);
    Token packageAlias = CommonUtil.getPreviousDefaultToken(tokenStream, currentTokenIndex);
    Token constraintStart = CommonUtil.getPreviousDefaultToken(tokenStream, packageAlias.getTokenIndex() - 1);
    List<SymbolInfo> returnList = new ArrayList<>();

    if (!(constraintStart.getText().equals("<") || constraintStart.getText().equals("create"))) {
        PackageActionFunctionAndTypesFilter filter = new PackageActionFunctionAndTypesFilter();
        returnList.addAll(filter.filterItems(context));
    } else {//from  www  .ja v  a2 s . c  om
        SymbolInfo packageSymbolInfo = symbolInfos.stream().filter(item -> {
            Scope.ScopeEntry scopeEntry = item.getScopeEntry();
            return item.getSymbolName().equals(packageAlias.getText())
                    && scopeEntry.symbol instanceof BPackageSymbol;
        }).findFirst().orElse(null);

        if (packageSymbolInfo != null) {
            packageSymbolInfo.getScopeEntry().symbol.scope.entries.forEach((name, value) -> {
                if (value.symbol.kind != null && value.symbol.kind.toString().equals(CONNECTOR_KIND)) {
                    returnList.add(new SymbolInfo(name.toString(), value));
                }
            });
        }
    }

    return returnList;
}

---

From source file:org.ballerinalang.langserver.completions.util.filters.ConnectorInitExpressionItemFilter.java

License:Open Source License

public List<SymbolInfo> filterItems(TextDocumentServiceContext context) {
    TokenStream tokenStream = context.get(DocumentServiceKeys.TOKEN_STREAM_KEY);
    int capturedTokenIndex = context.get(DocumentServiceKeys.TOKEN_INDEX_KEY);
    // Since the lang server is zero based indexing, we need to increase the line number to align with antlr token
    int cursorLine = context.get(DocumentServiceKeys.POSITION_KEY).getPosition().getLine() + 1;
    int cursorChar = context.get(DocumentServiceKeys.POSITION_KEY).getPosition().getCharacter();

    Token capturedToken = tokenStream.get(capturedTokenIndex);
    Token evaluatorToken = null;/*from www.j a v a 2  s  .  co  m*/
    int loopIncrementer = 1;
    int tokenIterator = capturedTokenIndex;

    if (capturedToken.getLine() > cursorLine
            || (capturedToken.getLine() == cursorLine && capturedToken.getCharPositionInLine() >= cursorChar)) {
        loopIncrementer = -1;
    }

    tokenIterator += loopIncrementer;

    while (true) {
        if (tokenIterator < 0) {
            break;
        }
        Token tempToken = tokenStream.get(tokenIterator);
        if (tempToken.getLine() == cursorLine && tempToken.getCharPositionInLine() == cursorChar) {
            evaluatorToken = getFirstNonHiddenToken(tempToken.getTokenIndex(), -1, tokenStream);
            break;
        }
        tokenIterator += loopIncrementer;
    }

    if (evaluatorToken == null) {
        return new ArrayList<>();
    }

    return getItemsList(evaluatorToken, context);
}

---

From source file:org.ballerinalang.langserver.sourceprune.LHSTokenTraverser.java

License:Open Source License

private boolean terminateLHSTraverse(Token token, TokenStream tokenStream) {
    int type = token.getType();
    if (type == BallerinaParser.RIGHT_PARENTHESIS) {
        this.rightParenthesisCount++;
        this.alterTokenText(token);
        return false;
    }//w ww .j  a v a  2 s . c  o  m
    if (type == BallerinaParser.LEFT_PARENTHESIS) {
        Optional<Token> tokenToLeft = CommonUtil.getPreviousDefaultToken(tokenStream, token.getTokenIndex());
        if (this.rightParenthesisCount > 0) {
            this.rightParenthesisCount--;
            this.alterTokenText(token);
            return false;
        } else if (tokenToLeft.isPresent() && (BallerinaParser.IF == tokenToLeft.get().getType()
                || BallerinaParser.WHILE == tokenToLeft.get().getType())) {
            // Cursor is within the if/ else if/ while condition
            this.replaceCondition(tokenStream, token.getTokenIndex());
        }
        return true;
    }
    /*
    Right brace is used as a terminal token and should be used as a terminating point for block statements.
    in cases such as following example,
    xyz {f1:1, f2:{f3:4}} = 
    right brace cannot consider as a terminal token, therefore we track the previous token and consider that to
    identify blocks. Until the right brace count is zero all the following right braces are altered (refer example)
     */
    if (type == BallerinaParser.RIGHT_BRACE && (this.lastAlteredToken == BallerinaParser.ASSIGN
            || this.lastAlteredToken == BallerinaParser.EQUAL_GT || this.rightBraceCount > 0)) {
        this.alterTokenText(token);
        this.rightBraceCount++;
        return false;
    }
    if (type == BallerinaParser.LEFT_BRACE && this.rightBraceCount > 0) {
        this.alterTokenText(token);
        this.rightBraceCount--;
        return false;
    }
    /*
    Specially capture the LT token in order to avoid Right token removal during the following case
    public annotation <resource,r> ...
    RHS Token traverser will check for the less than symbol count when a GT token found
     */
    if (type == BallerinaParser.LT) {
        this.alterTokenText(token);
        this.ltSymbolCount++;
        return false;
    }
    if (type == BallerinaParser.RETURNS) {
        this.alterTokenText(token);
        return !this.capturedAssignToken;
    }
    // Handle the ON token replacing since this is used in both service and JSON streaming input
    boolean onServiceRule = CommonUtil.getNDefaultTokensToLeft(tokenStream, 2, token.getTokenIndex()).stream()
            .map(Token::getType).collect(Collectors.toList()).contains(BallerinaParser.SERVICE);
    if (token.getType() == BallerinaParser.COMMA || (!onServiceRule && token.getType() == BallerinaParser.ON)) {
        this.alterTokenText(token);
    }

    return rightParenthesisCount == 0 && this.rightBraceCount == 0;
}

---

From source file:org.eclipse.titan.common.parsers.cfg.CfgParseTreePrinter.java

License:Open Source License

/**
 * Builds token text including hidden tokens before the token
 * @param aToken token to print/*  www .j  a va2 s.  c  o  m*/
 * @param aTokens token list from the lexer (all, hidden and not hidden also)
 * @param aPrintHiddenBefore true to print hidden tokens before the token
 * @param aResolveMode mode of resolving
 * @param aFile the parse tree of this file to print
 *                        needed only if aResolveMode != NO_RESOLVING, in case of [ORDERED_INCLUDE]
 */
private void printToken(final Token aToken, final List<Token> aTokens, final boolean aPrintHiddenBefore,
        final ResolveMode aResolveMode, final Path aFile) {
    final int tokenIndex = aToken.getTokenIndex();
    if (tokenIndex > -1 && aPrintHiddenBefore) {
        // Token has no index if tokenIndex == -1.
        // If a token is added to the parse tree after parse time, token start index in unknown (-1),
        // because token has no index in the token list.
        printHiddenTokensBefore(aToken, aTokens);
    }

    // the only non-hidden token
    if (aResolveMode != ResolveMode.NO_RESOLVING) {
        resolveToken(aToken, aResolveMode, aFile);
    } else {
        final String tokenText = aToken.getText();
        mSb.append(tokenText != null ? tokenText : "");
    }
}

---

From source file:org.eclipse.titan.common.parsers.cfg.CfgParseTreePrinter.java

License:Open Source License

/**
 * Builds hidden tokens before the token
 * @param aToken the token, this will NOT be printed
 * @param aTokens token list from the lexer (all, hidden and not hidden also)
 */// w ww. j  a  va2 s.c o  m
private void printHiddenTokensBefore(final Token aToken, final List<Token> aTokens) {
    final int tokenIndex = aToken.getTokenIndex();
    if (tokenIndex == -1) {
        // Token has no index.
        // If a token is added to the parse tree after parse time, token start index in unknown (-1),
        // because token has no index in the token list.
        return;
    }
    int startHiddenIndex = tokenIndex;
    while (isHiddenToken(startHiddenIndex - 1, aTokens)) {
        startHiddenIndex--;
    }
    for (int i = startHiddenIndex; i < tokenIndex; i++) {
        final Token t = aTokens.get(i);
        final String tokenText = t.getText();
        mSb.append(tokenText != null ? tokenText : "");
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• »