List of usage examples for org.antlr.v4.runtime Token getInputStream
CharStream getInputStream();
From source file:com.antsdb.saltedfish.sql.mysql.ExprGenerator.java
License:Open Source License
private static byte[] getBytes(Literal_value_binaryContext rule) { Token token = rule.STRING_LITERAL().getSymbol(); byte[] bytes = new byte[token.getStopIndex() - token.getStartIndex() - 1]; CharStream cs = token.getInputStream(); int pos = cs.index(); cs.seek(token.getStartIndex() + 1);/*from w w w .j a v a 2 s . c o m*/ int j = 0; for (int i = 0; i < bytes.length; i++) { int ch = cs.LA(i + 1); if (ch == '\\') { i++; ch = cs.LA(i + 1); if (ch == '0') { ch = 0; } else if (ch == 'n') { ch = '\n'; } else if (ch == 'r') { ch = '\r'; } else if (ch == 'Z') { ch = '\032'; } } bytes[j] = (byte) ch; j++; } cs.seek(pos); if (j != bytes.length) { // esacpe characters byte[] old = bytes; bytes = new byte[j]; System.arraycopy(old, 0, bytes, 0, j); } return bytes; }
From source file:com.satisfyingstructures.J2S.J2SConvertBasicFor.java
License:Open Source License
private String convertBasicForStatementToForInLoopOrSayWhyNot(ParserRuleContext ctx) { int forRule = ctx.getRuleIndex(); if (forRule != Java8Parser.RULE_basicForStatement && forRule != Java8Parser.RULE_basicForStatementNoShortIf) return "statement kind is not as expected"; // not our expected parameter type // Get to know more about our for statement // 'for' '(' forInit? ';' expression? ';' forUpdate? ')' ( statement | statementNoShortIf ) Boolean noShortIf = forRule == Java8Parser.RULE_basicForStatementNoShortIf; Java8Parser.ForInitContext forInitCtx = ctx.getChild(Java8Parser.ForInitContext.class, 0); Java8Parser.ExpressionContext expressionCtx = ctx.getChild(Java8Parser.ExpressionContext.class, 0); Java8Parser.ForUpdateContext forUpdateCtx = ctx.getChild(Java8Parser.ForUpdateContext.class, 0); ParserRuleContext statementCtx = ctx.getChild( noShortIf ? Java8Parser.StatementNoShortIfContext.class : Java8Parser.StatementContext.class, 0); ParserRuleContext statementSubCtx = statementCtx.getChild(ParserRuleContext.class, 0); ParserRuleContext statementSubSubCtx = statementSubCtx.getChild(ParserRuleContext.class, 0); Boolean statementisEmpty = statementSubSubCtx.getRuleIndex() == Java8Parser.RULE_emptyStatement; /*//from w ww. j ava 2s. c om 'for' '(' forInit? ';' expression? ';' forUpdate? ')' ( statement | statementNoShortIf ) Swift 3.0 has got rid of for(;;) statements for stong business cases such as... 'It is rarely used' 'not very Swift-like' 'The value of this construct is limited' ...and other total crap. We can convert simple equivalents of for ( i = startvalue ; i < endvalue ; i += step) to for i in start..<end or for i in start.stride(to: end by: step) To identify this we look for 1) have a forUpdate, which... a) operates on a single loop variable forUpdate().statementExpressionList().statementExpression().count()==1 b) incorporates increment or decrement by a constant step (++i,i++,i+=step,--i,i--,i-=step,) statementExpression rule is RULE_(assignment|preinc|postinc|predec|postdec) c) operates on the same variable tested in expression (compare - 2b) 2) have an expression, which... a) should be a simple comparison (<,<=,!=,>,>=, implicit non-zero) b) one side should be same as the loop var (compare - 1c) c) other side should not mutate within the loop - we can't tell this, too difficult 3) forInit a) must be i) empty(start with loop var existing value), or ii) simple init of a single loop var, or iii) simple declaration of a loop var */ // 1) Update statement. We need one... if (null == forUpdateCtx) return "it lacks an update statement"; // 1a) and it must operate on a single variable if (forUpdateCtx.statementExpressionList().getChildCount() != 1) return "there is more than one expression in the update statement"; // 1b) and it must be a simple increment or decrement Java8Parser.StatementExpressionContext updateStatementExpressionCtx = forUpdateCtx.statementExpressionList() .statementExpression(0); // statementExpression : assignment | preIncrementExpression | preDecrementExpression // | postIncrementExpression | postDecrementExpression // | methodInvocation | classInstanceCreationExpression ParserRuleContext updateExpressionCtx = updateStatementExpressionCtx.getChild(ParserRuleContext.class, 0); int updateExpressionRule = updateExpressionCtx.getRuleIndex(); boolean ascending_sequence; boolean open_interval; ParserRuleContext stepExpressionCtx = null; switch (updateExpressionRule) { // unaryExpression : preIncrementExpression | preDecrementExpression // | '+' unaryExpression | '-' unaryExpression // | unaryExpressionNotPlusMinus // preDecrementExpression : '--' unaryExpression // preIncrementExpression : '++' unaryExpression case Java8Parser.RULE_preDecrementExpression: ascending_sequence = false; break; case Java8Parser.RULE_preIncrementExpression: ascending_sequence = true; break; // postfixExpression : ( primary | expressionName ) ( '++' | '--')* // postIncrementExpression : postfixExpression '++' // postDecrementExpression : postfixExpression '--' case Java8Parser.RULE_postDecrementExpression: ascending_sequence = false; break; case Java8Parser.RULE_postIncrementExpression: ascending_sequence = true; break; // assignment : leftHandSide assignmentOperator expression // leftHandSide : expressionName | fieldAccess | arrayAccess case Java8Parser.RULE_assignment: if (null != updateStatementExpressionCtx.assignment().leftHandSide().arrayAccess()) return "cant convert a loop variable that is an array element"; TerminalNode node = updateStatementExpressionCtx.assignment().assignmentOperator() .getChild(TerminalNode.class, 0); switch (node.getSymbol().getType()) { case Java8Parser.ADD_ASSIGN: ascending_sequence = true; break; case Java8Parser.SUB_ASSIGN: ascending_sequence = false; break; case Java8Parser.ASSIGN: // possibilities too complex to warrant extracting simple a=a+1 cases default: return "potentially too complex to create a sequence from this update operation"; } stepExpressionCtx = updateStatementExpressionCtx.assignment().expression(); break; default: // methodInvocation | classInstanceCreationExpression return "the expression in the update statement is too complex"; } // In each of the cases that we have not rejected, the loop variable is in the first child rule context of the // update statement. Get the text of the variable, rather than analysing the graph any further, as the // possibilities are endless; all that we require is that the loop variable text matches that in the text // expression and the init expression. ParserRuleContext loopVariable_updated_Ctx = updateExpressionCtx.getChild(ParserRuleContext.class, 0); String loopVariableTxt = loopVariable_updated_Ctx.getText(); // we want original text // 2) Expression if (null == expressionCtx) return "it lacks a test expression"; // expression : lambdaExpression | assignmentExpression if (null != expressionCtx.lambdaExpression()) return "cannot convert a lambda expression"; // assignmentExpression : conditionalExpression | assignment if (null != expressionCtx.assignmentExpression().assignment()) return "cannot convert an assignment within the test expression"; // 2a) must be a simple relation: // Descend the chain of expression rule pass-through branches until we find the one that is significant, then // test to see if expression contains a terminal that is one of !=, <, <=, >, >=. ParserRuleContext testExpressionCtx = J2SGrammarUtils.descendToSignificantExpression(expressionCtx); int testExpressionRule = testExpressionCtx.getRuleIndex(); TerminalNode node = testExpressionCtx.getChild(TerminalNode.class, 0); int testOperatorType = null != node ? node.getSymbol().getType() : 0; switch (testOperatorType) { case Java8Parser.NOTEQUAL: open_interval = true; break; case Java8Parser.LE: open_interval = false; break; case Java8Parser.GE: open_interval = false; break; case Java8Parser.LT: // can occur in relational and shift expressions case Java8Parser.GT: // can occur in relational and shift expressions if (testExpressionRule == Java8Parser.RULE_relationalExpression) { open_interval = true; break; } default: return "can only convert test expressions that use !=, <, <=, > or >="; } // 2b) relation must be testing same var as changed in update expression // The loop variable could be on the left or the right of the comparison operator int i; ParserRuleContext loopVariable_tested_Ctx = null; for (i = 0; i < 2; i++) { loopVariable_tested_Ctx = testExpressionCtx.getChild(ParserRuleContext.class, i); if (null != loopVariable_tested_Ctx && loopVariableTxt.equals(loopVariable_tested_Ctx.getText())) break; // found matching loop variable loopVariable_tested_Ctx = null; } if (null == loopVariable_tested_Ctx || (i == 1 && testExpressionCtx.getChildCount() > 3)) return "the test expression must be testing the same variable as changed in update expression"; ParserRuleContext terminalValueCtx = testExpressionCtx.getChild(ParserRuleContext.class, i ^ 1); // 2c) the terminal value side should not mutate within the loop // - way too difficult for us to determine this // 3) Loop init expression. Must be either... ParserRuleContext initialValueCtx; if (null == forInitCtx) // a) empty { // Using the loop variable's existing value from outside the scope initialValueCtx = loopVariable_tested_Ctx; } else if (null != forInitCtx.statementExpressionList()) // b) a simple init of a single loop var { /* // Could not convert... // for (i = 0; i<10; i++) // ...to a for..in statement because can only work with an assignment expression for loop variable initialisation. i = 0; while i<10 {j += 1 i += 1; } */ if (forInitCtx.statementExpressionList().getChildCount() != 1) return "can only work with initialisation of a single loop variable"; Java8Parser.StatementExpressionContext initExpressionCtx = forInitCtx.statementExpressionList() .statementExpression(0); if (null == initExpressionCtx.assignment()) return "can only work with an assignment expression for loop variable initialisation"; if (!loopVariableTxt.equals(initExpressionCtx.assignment().leftHandSide().getText())) return "the initialised variable is different from the updated variable"; // different to the loop variable initialValueCtx = initExpressionCtx.assignment().expression(); } else if (null != forInitCtx.localVariableDeclaration()) // c) a simple decl of a single loop var { // localVariableDeclaration : variableModifier* unannType variableDeclaratorList Java8Parser.VariableDeclaratorListContext vdlc = forInitCtx.localVariableDeclaration() .variableDeclaratorList(); // variableDeclaratorList : variableDeclarator (',' variableDeclarator)* if (vdlc.getChildCount() != 1) return "can only work with declaration of a single loop variable"; Java8Parser.VariableDeclaratorContext vdc = vdlc.variableDeclarator(0); // variableDeclarator : variableDeclaratorId ('=' variableInitializer)? if (!loopVariableTxt.equals(vdc.getChild(0).getText())) return "the declared loop variable is be different from the updated variable"; initialValueCtx = vdc.variableInitializer(); if (null == initialValueCtx) return "there is no initialiser for the loop variable"; } else return "loop initialisation is in unexpected form"; // Now we have all the components we need String forInLoopText; // Use actual text with replacements String initialValueTxt = rewriter.getText(initialValueCtx); String terminalValueTxt = rewriter.getText(terminalValueCtx); // !!!: watch out... // if we use the actual text from the update expression, we can find that the pre/post-inc/dec has been // converted to the add/sub-assign form and because structure is lost when rewriting, the new form can // stick to the variable when we retrieve it. There's no easy solution for this (and any similar occurrences), // but we side step it by getting the text of loop variable from the test expression: loopVariableTxt = rewriter.getText(loopVariable_tested_Ctx); if (null != stepExpressionCtx || !ascending_sequence) { String stepExpressionText = stepExpressionCtx == null ? "-1" : ascending_sequence ? rewriter.getText(stepExpressionCtx) : "-(" + rewriter.getText(stepExpressionCtx) + ")"; forInLoopText = "for " + loopVariableTxt + " in " + loopVariableTxt + ".stride(from: " + initialValueTxt + (open_interval ? ", to: " : ", through: ") + terminalValueTxt + ", by: " + stepExpressionText + ")"; } else { forInLoopText = "for " + loopVariableTxt + " in " + initialValueTxt + (open_interval ? " ..< " : " ... ") + terminalValueTxt; } Token startToken = ctx.getToken(Java8Parser.FOR, 0).getSymbol(); Token endToken = ctx.getToken(Java8Parser.RPAREN, 0).getSymbol(); CharStream cs = startToken.getInputStream(); String originalExpressionTxt = cs.getText(Interval.of(startToken.getStartIndex(), endToken.getStopIndex())); rewriter.insertComment(originalExpressionTxt + " converted to", ctx, J2SRewriter.CommentWhere.beforeLineBreak); int startIndex = startToken.getTokenIndex(); int endIndex = endToken.getTokenIndex(); // Problem: (see notes in J2SRewriter.replaceAndAdjustWhitespace) Before converting to for-in, the loop will // also have had parentheses removed (and other transforms); rewriter may have coallesced some of the changes // so that the old end boundary no longer exists. (- a shortcoming of TokenStreamRewriter) // Workaround: test if endIndex is straddled by changed interval, and if so, extend our interval to the end of // the change. (Pretty horrendous to have to work around this here, but I don't yet see an easy way of fixing // the underlying problem or a generalised way of working around it.) Interval interval = rewriter.getChangedIntervalContaining(endIndex, endIndex); if (null != interval && interval.a <= endIndex && interval.b > endIndex) endIndex = interval.b; rewriter.replaceAndAdjustWhitespace(startIndex, endIndex, forInLoopText); return null; }
From source file:org.apache.hive.hplsql.Exec.java
License:Apache License
/** * Append the text preserving the formatting (space symbols) between tokens *//*from w ww .j ava 2s.c o m*/ void append(StringBuilder str, String appendStr, Token start, Token stop) { String spaces = start.getInputStream() .getText(new org.antlr.v4.runtime.misc.Interval(start.getStartIndex(), stop.getStopIndex())); spaces = spaces.substring(start.getText().length(), spaces.length() - stop.getText().length()); str.append(spaces); str.append(appendStr); }
From source file:org.ledyba.sora.parser.FortranTokenStream.java
License:Open Source License
/** * Create a subset list of the non-whitespace tokens in the current line. *//*from www . j a va 2 s .c om*/ private ArrayList<Token> createPackedList() { int i = 0; Token tk = null; ArrayList<Token> pList = new ArrayList<>(this.lineLength + 1); for (i = 0; i < currLine.size(); i++) { tk = getTokenFromCurrLine(i); try { if (tk.getChannel() != Token.HIDDEN_CHANNEL) { pList.add(tk); } } catch (Exception e) { e.printStackTrace(); System.exit(1); } } // need to make sure the line was terminated with a T_EOS. this may // not happen if we're working on a file that ended w/o a newline Token last = pList.get(pList.size() - 1); if (last.getType() != FortranLexer.T_EOS) { Pair<TokenSource, CharStream> src = new Pair<>(last.getTokenSource(), last.getInputStream()); FortranToken eos = new FortranToken(src, FortranLexer.T_EOS, Token.DEFAULT_CHANNEL, last.getTokenIndex(), last.getTokenIndex() + 1); eos.setText("\n"); packedList.add(eos); } return pList; }
From source file:org.ledyba.sora.parser.FortranTokenStream.java
License:Open Source License
public void addTokenTo(int index, int line, int col, int tokenType, String tokenText) { try {//from ww w . j a va 2 s . c o m // for example: // index = 1 // packedList == label T_CONTINUE T_EOS (size is 3) // newTokenList.size() == 22 // 22-3+1=20 // so, inserted between the label and T_CONTINUE Token last = this.tokens.get(index <= 0 ? 0 : index - 1); Pair<TokenSource, CharStream> src = new Pair<>(last.getTokenSource(), last.getInputStream()); FortranToken token = new FortranToken(src, tokenType, Token.DEFAULT_CHANNEL, last.getStopIndex(), last.getStopIndex()); token.setText(tokenText == null ? "" : tokenText); token.setLine(line); token.setCharPositionInLine(col); this.packedList.add(index, token); } catch (Exception e) { e.printStackTrace(); System.exit(1); } return; }
From source file:org.ledyba.sora.parser.FortranTokenStream.java
License:Open Source License
public void addToken(int type, String text, int line, int col) { Token last = this.tokens.get(this.tokens.size() - 1); Pair<TokenSource, CharStream> src = new Pair<>(last.getTokenSource(), last.getInputStream()); FortranToken token = new FortranToken(src, type, Token.DEFAULT_CHANNEL, 0, 0); token.setLine(line);/*from w w w . ja va 2s .c o m*/ token.setCharPositionInLine(col); this.addToken(token); }
From source file:org.ledyba.sora.parser.FortranTokenStream.java
License:Open Source License
public void addTokenTo(int index, int type, String text, int line, int col) { Token last = this.tokens.get(index); Pair<TokenSource, CharStream> src = new Pair<>(last.getTokenSource(), last.getInputStream()); FortranToken token = new FortranToken(src, type, Token.DEFAULT_CHANNEL, 0, 0); token.setLine(line);//w w w . j a va 2 s . co m token.setCharPositionInLine(col); this.addTokenTo(index, token); }
From source file:org.lockss.tdb.AntlrUtil.java
License:Open Source License
/** * <p>/*ww w . j a va 2 s.c o m*/ * Throws a {@link SyntaxError} based on the offending {@link Token}. * </p> * * @param token * An offending token. * @param format * A format string. * @param args * Arguments for the format string. * @throws SyntaxError * <b>always</b> thrown by this method. * @since 1.67 * @see #errorMessage(String, int, int, String, Object...) */ public static void syntaxError(Token token, String format, Object... args) throws SyntaxError { String errMsg = errorMessage(token.getInputStream().getSourceName(), token.getLine(), token.getCharPositionInLine(), format, args); throw new SyntaxError(errMsg); }