List of usage examples for org.antlr.v4.runtime Token getTokenIndex
int getTokenIndex();
From source file:alfa.util.Util.java
public static String At(Token token) { String s = " @ line:"; s += token.getLine();//from w w w. ja v a2 s. c o m s += " char: "; s += token.getTokenIndex(); return s; }
From source file:codesniffer.java8.adapter.AdapterUtil.java
License:Open Source License
/** * If there are no statements within a block, we need a special method to grab any comments that * might exist between braces./*from w w w. j av a 2s . co m*/ * * @param node * @param parserRuleContext * @param adapterParameters */ public static void setInternalComments(Node node, ParserRuleContext parserRuleContext, AdapterParameters adapterParameters) { BufferedTokenStream tokens = adapterParameters.getTokens(); if (node == null || parserRuleContext == null || tokens == null) { throw new IllegalArgumentException("Parameters must not be null"); } Token startToken = parserRuleContext.getStart(); Token stopToken = parserRuleContext.getStop(); List<Token> commentTokens; ArrayList<Comment> internalCommentList = new ArrayList<>(); // Checking to the right of the start token will check inside the statement commentTokens = tokens.getHiddenTokensToRight(startToken.getTokenIndex(), Java8Lexer.COMMENTS); if (commentTokens != null) { internalCommentList.ensureCapacity(commentTokens.size()); for (Token commentToken : commentTokens) { // Skip already claimed comments (prevents comment repeats) if (adapterParameters.isCommentTokenClaimed(commentToken.getTokenIndex())) { continue; } else { // Claim it adapterParameters.claimCommentToken(commentToken.getTokenIndex()); } if (commentToken.getText().startsWith("/**")) { DocumentComment javadocComment = new DocumentComment(commentToken.getText()); internalCommentList.add(javadocComment); } else if (commentToken.getText().startsWith("/*")) { BlockComment blockComment = new BlockComment(commentToken.getText()); internalCommentList.add(blockComment); } else if (commentToken.getText().startsWith("//")) { LineComment lineComment = new LineComment(commentToken.getText()); internalCommentList.add(lineComment); } } } if (internalCommentList.size() > 0) { if (node.getOrphanComments() != null) { node.getOrphanComments().addAll(internalCommentList); } else { node.setOrphanComments(internalCommentList); } } // if (internalCommentList.size() > 0) { // if (node.getInternalComments() != null) { // node.getInternalComments().addAll(internalCommentList); // } else { // node.setInternalComments(internalCommentList); // } // } }
From source file:codesniffer.java8.adapter.AdapterUtil.java
License:Open Source License
public static void setComments(Node node, ParserRuleContext parserRuleContext, AdapterParameters adapterParameters) { BufferedTokenStream tokens = adapterParameters.getTokens(); if (node == null || parserRuleContext == null || tokens == null) { // Just return return;/* www. j a va2 s. c o m*/ } Token startToken = parserRuleContext.getStart(); List<Token> commentTokens = null; try { // commentTokens = tokens.getHiddenTokensToLeft(startToken.getTokenIndex(), Java8Lexer.COMMENTS); } catch (Exception ex) { ex.printStackTrace(); } if (commentTokens != null && commentTokens.size() > 0) { Iterator<Token> iter = commentTokens.iterator(); Token ct = iter.next(); int idx = ct.getTokenIndex(); if (adapterParameters.isCommentTokenClaimed(idx)) { adapterParameters.claimCommentToken(idx); node.setComment(extractComment(ct)); if (iter.hasNext()) { List<Comment> orf = new ArrayList<>(8); while (iter.hasNext()) { orf.add(extractComment(iter.next())); } node.setOrphanComments(orf); } } } /** * obsolete api. bowen 7/19 */ // List<Comment> beginCommentList = new LinkedList<Comment>(); // List<Comment> endCommentList = new LinkedList<Comment>(); // if (commentTokens != null) { // for (Token commentToken : commentTokens) { // // // Skip already claimed comments (prevents comment repeats) // if (adapterParameters.isCommentTokenClaimed(commentToken.getTokenIndex())) { // continue; // } else { // // Claim it // adapterParameters.claimCommentToken(commentToken.getTokenIndex()); // } // // if (commentToken.getText().trim().startsWith("/**")) { // DocumentComment javadocComment = new DocumentComment(commentToken.getText()); // beginCommentList.add(javadocComment); // } else if (commentToken.getText().trim().startsWith("/*")) { // BlockComment blockComment = new BlockComment(commentToken.getText()); // beginCommentList.add(blockComment); // } else if (commentToken.getText().trim().startsWith("//")) { // LineComment lineComment = new LineComment(commentToken.getText()); // beginCommentList.add(lineComment); // } // } // } // commentTokens = tokens.getHiddenTokensToRight(stopToken.getTokenIndex(), Java8Lexer.COMMENTS); // if (commentTokens != null) { // for (Token commentToken : commentTokens) { // // if (commentToken.getLine() == stopToken.getLine()) { // // // Skip already claimed comments (prevents comment repeats) // if (adapterParameters.isCommentTokenClaimed(commentToken.getTokenIndex())) { // continue; // } else { // // Claim it // adapterParameters.claimCommentToken(commentToken.getTokenIndex()); // } // // if (commentToken.getText().trim().startsWith("/**")) { // DocumentComment javadocComment = new DocumentComment(commentToken.getText()); // endCommentList.add(javadocComment); // } else if (commentToken.getText().trim().startsWith("/*")) { // BlockComment blockComment = new BlockComment(commentToken.getText()); // endCommentList.add(blockComment); // } else if (commentToken.getText().trim().startsWith("//")) { // LineComment lineComment = new LineComment(commentToken.getText()); // endCommentList.add(lineComment); // } // } // } // } // if (beginCommentList.size() > 0) { // if (node.getBeginComments() != null) { // node.getBeginComments().addAll(beginCommentList); // } else { // node.setBeginComments(beginCommentList); // } // } // // if (endCommentList.size() > 0) { // if (node.getEndComments() != null) { // node.getEndComments().addAll(endCommentList); // } else { // node.setEndComments(endCommentList); // } // } }
From source file:com.blazebit.persistence.impl.expression.JPQLSelectExpressionVisitorImpl.java
License:Apache License
private StringBuilder getTextWithSurroundingHiddenTokens(Token token) { StringBuilder sb = new StringBuilder(); List<Token> hiddenTokens = tokens.getHiddenTokensToLeft(token.getTokenIndex()); if (hiddenTokens != null) { for (Token t : hiddenTokens) { sb.append(t.getText());/*w w w. j ava2s. c o m*/ } } sb.append(token.getText()); hiddenTokens = tokens.getHiddenTokensToRight(token.getTokenIndex()); if (hiddenTokens != null) { for (Token t : hiddenTokens) { sb.append(t.getText()); } } return sb; }
From source file:com.blazebit.persistence.impl.expression.JPQLSelectExpressionVisitorImpl.java
License:Apache License
/** * * @param start (exclusive)//from w w w . ja v a 2s . com * @param end (exclusive) * @return */ private StringBuilder getText(Token start, Token end) { int startIndex = start.getTokenIndex() + 1; int endIndex = end.getTokenIndex(); if (startIndex < endIndex) { StringBuilder sb = new StringBuilder(); for (int i = startIndex; i < endIndex; i++) { sb.append(tokens.get(i).getText()); } return sb; } return null; }
From source file:com.facebook.presto.sql.parser.ErrorHandler.java
License:Apache License
@Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String message, RecognitionException e) { try {/*from www .jav a2 s . c om*/ Parser parser = (Parser) recognizer; ATN atn = parser.getATN(); ATNState currentState; Token currentToken; RuleContext context; if (e != null) { currentState = atn.states.get(e.getOffendingState()); currentToken = e.getOffendingToken(); context = e.getCtx(); if (e instanceof NoViableAltException) { currentToken = ((NoViableAltException) e).getStartToken(); } } else { currentState = atn.states.get(parser.getState()); currentToken = parser.getCurrentToken(); context = parser.getContext(); } Analyzer analyzer = new Analyzer(atn, parser.getVocabulary(), specialRules, specialTokens, ignoredRules, parser.getTokenStream()); Multimap<Integer, String> candidates = analyzer.process(currentState, currentToken.getTokenIndex(), context); // pick the candidate tokens associated largest token index processed (i.e., the path that consumed the most input) String expected = candidates.asMap().entrySet().stream().max(Comparator.comparing(Map.Entry::getKey)) .get().getValue().stream().sorted().collect(Collectors.joining(", ")); message = String.format("mismatched input '%s'. Expecting: %s", ((Token) offendingSymbol).getText(), expected); } catch (Exception exception) { LOG.error(exception, "Unexpected failure when handling parsing error. This is likely a bug in the implementation"); } throw new ParsingException(message, e, line, charPositionInLine); }
From source file:com.huawei.streaming.cql.semanticanalyzer.parser.CQLErrorStrategy.java
License:Apache License
@NotNull private String getText(TokenStream tokens, Token start, Token stop) { if (start != null && stop != null) { return getText(tokens, Interval.of(start.getTokenIndex(), stop.getTokenIndex())); }/*from ww w . ja v a 2 s. com*/ return ""; }
From source file:com.microsoft.thrifty.schema.parser.ThriftListener.java
License:Open Source License
private List<Token> getLeadingComments(Token token) { List<Token> hiddenTokens = tokenStream.getHiddenTokensToLeft(token.getTokenIndex(), Lexer.HIDDEN); if (hiddenTokens == null || hiddenTokens.isEmpty()) { return Collections.emptyList(); }/*from ww w .j a v a2 s . c om*/ List<Token> comments = new ArrayList<>(hiddenTokens.size()); for (Token hiddenToken : hiddenTokens) { if (isComment(hiddenToken) && !trailingDocTokenIndexes.get(hiddenToken.getTokenIndex())) { comments.add(hiddenToken); } } return comments; }
From source file:com.microsoft.thrifty.schema.parser.ThriftListener.java
License:Open Source License
/** * Read comments following the given token, until the first newline is encountered. * * INVARIANT://from w w w. ja v a 2 s . com * Assumes that the parse tree is being walked top-down, left to right! * * Trailing-doc tokens are marked as such, so that subsequent searches for "leading" * doc don't grab tokens already used as "trailing" doc. If the walk order is *not* * top-down, left-to-right, then the assumption underpinning the separation of leading * and trailing comments is broken. * * @param endToken the token from which to search for trailing comment tokens. * @return a list, possibly empty, of all trailing comment tokens. */ private List<Token> getTrailingComments(Token endToken) { List<Token> hiddenTokens = tokenStream.getHiddenTokensToRight(endToken.getTokenIndex(), Lexer.HIDDEN); if (hiddenTokens == null || hiddenTokens.isEmpty()) { return Collections.emptyList(); } Token maybeTrailingDoc = hiddenTokens.get(0); // only one trailing comment is possible if (isComment(maybeTrailingDoc)) { trailingDocTokenIndexes.set(maybeTrailingDoc.getTokenIndex()); return Collections.singletonList(maybeTrailingDoc); } return Collections.emptyList(); }
From source file:com.satisfyingstructures.J2S.J2SConvertBasicFor.java
License:Open Source License
private String convertBasicForStatementToForInLoopOrSayWhyNot(ParserRuleContext ctx) { int forRule = ctx.getRuleIndex(); if (forRule != Java8Parser.RULE_basicForStatement && forRule != Java8Parser.RULE_basicForStatementNoShortIf) return "statement kind is not as expected"; // not our expected parameter type // Get to know more about our for statement // 'for' '(' forInit? ';' expression? ';' forUpdate? ')' ( statement | statementNoShortIf ) Boolean noShortIf = forRule == Java8Parser.RULE_basicForStatementNoShortIf; Java8Parser.ForInitContext forInitCtx = ctx.getChild(Java8Parser.ForInitContext.class, 0); Java8Parser.ExpressionContext expressionCtx = ctx.getChild(Java8Parser.ExpressionContext.class, 0); Java8Parser.ForUpdateContext forUpdateCtx = ctx.getChild(Java8Parser.ForUpdateContext.class, 0); ParserRuleContext statementCtx = ctx.getChild( noShortIf ? Java8Parser.StatementNoShortIfContext.class : Java8Parser.StatementContext.class, 0); ParserRuleContext statementSubCtx = statementCtx.getChild(ParserRuleContext.class, 0); ParserRuleContext statementSubSubCtx = statementSubCtx.getChild(ParserRuleContext.class, 0); Boolean statementisEmpty = statementSubSubCtx.getRuleIndex() == Java8Parser.RULE_emptyStatement; /*// w w w . jav a 2 s . c o m 'for' '(' forInit? ';' expression? ';' forUpdate? ')' ( statement | statementNoShortIf ) Swift 3.0 has got rid of for(;;) statements for stong business cases such as... 'It is rarely used' 'not very Swift-like' 'The value of this construct is limited' ...and other total crap. We can convert simple equivalents of for ( i = startvalue ; i < endvalue ; i += step) to for i in start..<end or for i in start.stride(to: end by: step) To identify this we look for 1) have a forUpdate, which... a) operates on a single loop variable forUpdate().statementExpressionList().statementExpression().count()==1 b) incorporates increment or decrement by a constant step (++i,i++,i+=step,--i,i--,i-=step,) statementExpression rule is RULE_(assignment|preinc|postinc|predec|postdec) c) operates on the same variable tested in expression (compare - 2b) 2) have an expression, which... a) should be a simple comparison (<,<=,!=,>,>=, implicit non-zero) b) one side should be same as the loop var (compare - 1c) c) other side should not mutate within the loop - we can't tell this, too difficult 3) forInit a) must be i) empty(start with loop var existing value), or ii) simple init of a single loop var, or iii) simple declaration of a loop var */ // 1) Update statement. We need one... if (null == forUpdateCtx) return "it lacks an update statement"; // 1a) and it must operate on a single variable if (forUpdateCtx.statementExpressionList().getChildCount() != 1) return "there is more than one expression in the update statement"; // 1b) and it must be a simple increment or decrement Java8Parser.StatementExpressionContext updateStatementExpressionCtx = forUpdateCtx.statementExpressionList() .statementExpression(0); // statementExpression : assignment | preIncrementExpression | preDecrementExpression // | postIncrementExpression | postDecrementExpression // | methodInvocation | classInstanceCreationExpression ParserRuleContext updateExpressionCtx = updateStatementExpressionCtx.getChild(ParserRuleContext.class, 0); int updateExpressionRule = updateExpressionCtx.getRuleIndex(); boolean ascending_sequence; boolean open_interval; ParserRuleContext stepExpressionCtx = null; switch (updateExpressionRule) { // unaryExpression : preIncrementExpression | preDecrementExpression // | '+' unaryExpression | '-' unaryExpression // | unaryExpressionNotPlusMinus // preDecrementExpression : '--' unaryExpression // preIncrementExpression : '++' unaryExpression case Java8Parser.RULE_preDecrementExpression: ascending_sequence = false; break; case Java8Parser.RULE_preIncrementExpression: ascending_sequence = true; break; // postfixExpression : ( primary | expressionName ) ( '++' | '--')* // postIncrementExpression : postfixExpression '++' // postDecrementExpression : postfixExpression '--' case Java8Parser.RULE_postDecrementExpression: ascending_sequence = false; break; case Java8Parser.RULE_postIncrementExpression: ascending_sequence = true; break; // assignment : leftHandSide assignmentOperator expression // leftHandSide : expressionName | fieldAccess | arrayAccess case Java8Parser.RULE_assignment: if (null != updateStatementExpressionCtx.assignment().leftHandSide().arrayAccess()) return "cant convert a loop variable that is an array element"; TerminalNode node = updateStatementExpressionCtx.assignment().assignmentOperator() .getChild(TerminalNode.class, 0); switch (node.getSymbol().getType()) { case Java8Parser.ADD_ASSIGN: ascending_sequence = true; break; case Java8Parser.SUB_ASSIGN: ascending_sequence = false; break; case Java8Parser.ASSIGN: // possibilities too complex to warrant extracting simple a=a+1 cases default: return "potentially too complex to create a sequence from this update operation"; } stepExpressionCtx = updateStatementExpressionCtx.assignment().expression(); break; default: // methodInvocation | classInstanceCreationExpression return "the expression in the update statement is too complex"; } // In each of the cases that we have not rejected, the loop variable is in the first child rule context of the // update statement. Get the text of the variable, rather than analysing the graph any further, as the // possibilities are endless; all that we require is that the loop variable text matches that in the text // expression and the init expression. ParserRuleContext loopVariable_updated_Ctx = updateExpressionCtx.getChild(ParserRuleContext.class, 0); String loopVariableTxt = loopVariable_updated_Ctx.getText(); // we want original text // 2) Expression if (null == expressionCtx) return "it lacks a test expression"; // expression : lambdaExpression | assignmentExpression if (null != expressionCtx.lambdaExpression()) return "cannot convert a lambda expression"; // assignmentExpression : conditionalExpression | assignment if (null != expressionCtx.assignmentExpression().assignment()) return "cannot convert an assignment within the test expression"; // 2a) must be a simple relation: // Descend the chain of expression rule pass-through branches until we find the one that is significant, then // test to see if expression contains a terminal that is one of !=, <, <=, >, >=. ParserRuleContext testExpressionCtx = J2SGrammarUtils.descendToSignificantExpression(expressionCtx); int testExpressionRule = testExpressionCtx.getRuleIndex(); TerminalNode node = testExpressionCtx.getChild(TerminalNode.class, 0); int testOperatorType = null != node ? node.getSymbol().getType() : 0; switch (testOperatorType) { case Java8Parser.NOTEQUAL: open_interval = true; break; case Java8Parser.LE: open_interval = false; break; case Java8Parser.GE: open_interval = false; break; case Java8Parser.LT: // can occur in relational and shift expressions case Java8Parser.GT: // can occur in relational and shift expressions if (testExpressionRule == Java8Parser.RULE_relationalExpression) { open_interval = true; break; } default: return "can only convert test expressions that use !=, <, <=, > or >="; } // 2b) relation must be testing same var as changed in update expression // The loop variable could be on the left or the right of the comparison operator int i; ParserRuleContext loopVariable_tested_Ctx = null; for (i = 0; i < 2; i++) { loopVariable_tested_Ctx = testExpressionCtx.getChild(ParserRuleContext.class, i); if (null != loopVariable_tested_Ctx && loopVariableTxt.equals(loopVariable_tested_Ctx.getText())) break; // found matching loop variable loopVariable_tested_Ctx = null; } if (null == loopVariable_tested_Ctx || (i == 1 && testExpressionCtx.getChildCount() > 3)) return "the test expression must be testing the same variable as changed in update expression"; ParserRuleContext terminalValueCtx = testExpressionCtx.getChild(ParserRuleContext.class, i ^ 1); // 2c) the terminal value side should not mutate within the loop // - way too difficult for us to determine this // 3) Loop init expression. Must be either... ParserRuleContext initialValueCtx; if (null == forInitCtx) // a) empty { // Using the loop variable's existing value from outside the scope initialValueCtx = loopVariable_tested_Ctx; } else if (null != forInitCtx.statementExpressionList()) // b) a simple init of a single loop var { /* // Could not convert... // for (i = 0; i<10; i++) // ...to a for..in statement because can only work with an assignment expression for loop variable initialisation. i = 0; while i<10 {j += 1 i += 1; } */ if (forInitCtx.statementExpressionList().getChildCount() != 1) return "can only work with initialisation of a single loop variable"; Java8Parser.StatementExpressionContext initExpressionCtx = forInitCtx.statementExpressionList() .statementExpression(0); if (null == initExpressionCtx.assignment()) return "can only work with an assignment expression for loop variable initialisation"; if (!loopVariableTxt.equals(initExpressionCtx.assignment().leftHandSide().getText())) return "the initialised variable is different from the updated variable"; // different to the loop variable initialValueCtx = initExpressionCtx.assignment().expression(); } else if (null != forInitCtx.localVariableDeclaration()) // c) a simple decl of a single loop var { // localVariableDeclaration : variableModifier* unannType variableDeclaratorList Java8Parser.VariableDeclaratorListContext vdlc = forInitCtx.localVariableDeclaration() .variableDeclaratorList(); // variableDeclaratorList : variableDeclarator (',' variableDeclarator)* if (vdlc.getChildCount() != 1) return "can only work with declaration of a single loop variable"; Java8Parser.VariableDeclaratorContext vdc = vdlc.variableDeclarator(0); // variableDeclarator : variableDeclaratorId ('=' variableInitializer)? if (!loopVariableTxt.equals(vdc.getChild(0).getText())) return "the declared loop variable is be different from the updated variable"; initialValueCtx = vdc.variableInitializer(); if (null == initialValueCtx) return "there is no initialiser for the loop variable"; } else return "loop initialisation is in unexpected form"; // Now we have all the components we need String forInLoopText; // Use actual text with replacements String initialValueTxt = rewriter.getText(initialValueCtx); String terminalValueTxt = rewriter.getText(terminalValueCtx); // !!!: watch out... // if we use the actual text from the update expression, we can find that the pre/post-inc/dec has been // converted to the add/sub-assign form and because structure is lost when rewriting, the new form can // stick to the variable when we retrieve it. There's no easy solution for this (and any similar occurrences), // but we side step it by getting the text of loop variable from the test expression: loopVariableTxt = rewriter.getText(loopVariable_tested_Ctx); if (null != stepExpressionCtx || !ascending_sequence) { String stepExpressionText = stepExpressionCtx == null ? "-1" : ascending_sequence ? rewriter.getText(stepExpressionCtx) : "-(" + rewriter.getText(stepExpressionCtx) + ")"; forInLoopText = "for " + loopVariableTxt + " in " + loopVariableTxt + ".stride(from: " + initialValueTxt + (open_interval ? ", to: " : ", through: ") + terminalValueTxt + ", by: " + stepExpressionText + ")"; } else { forInLoopText = "for " + loopVariableTxt + " in " + initialValueTxt + (open_interval ? " ..< " : " ... ") + terminalValueTxt; } Token startToken = ctx.getToken(Java8Parser.FOR, 0).getSymbol(); Token endToken = ctx.getToken(Java8Parser.RPAREN, 0).getSymbol(); CharStream cs = startToken.getInputStream(); String originalExpressionTxt = cs.getText(Interval.of(startToken.getStartIndex(), endToken.getStopIndex())); rewriter.insertComment(originalExpressionTxt + " converted to", ctx, J2SRewriter.CommentWhere.beforeLineBreak); int startIndex = startToken.getTokenIndex(); int endIndex = endToken.getTokenIndex(); // Problem: (see notes in J2SRewriter.replaceAndAdjustWhitespace) Before converting to for-in, the loop will // also have had parentheses removed (and other transforms); rewriter may have coallesced some of the changes // so that the old end boundary no longer exists. (- a shortcoming of TokenStreamRewriter) // Workaround: test if endIndex is straddled by changed interval, and if so, extend our interval to the end of // the change. (Pretty horrendous to have to work around this here, but I don't yet see an easy way of fixing // the underlying problem or a generalised way of working around it.) Interval interval = rewriter.getChangedIntervalContaining(endIndex, endIndex); if (null != interval && interval.a <= endIndex && interval.b > endIndex) endIndex = interval.b; rewriter.replaceAndAdjustWhitespace(startIndex, endIndex, forInLoopText); return null; }