List of usage examples for org.antlr.v4.runtime Token getStartIndex
int getStartIndex();
From source file:org.eclipse.titan.common.parsers.cfg.CfgLocation.java
License:Open Source License
/** * Constructor for ANTLR v4 tokens//from w ww. j av a 2s. c o m * @param aFile the parsed file * @param aStartToken the 1st token, its line and start position will be used for the location * NOTE: start position is the column index of the tokens 1st character. * Column index starts with 0. * @param aEndToken the last token, its end position will be used for the location. * NOTE: end position is the column index after the token's last character. */ public CfgLocation(final IFile aFile, final Token aStartToken, final Token aEndToken) { setLocation(aFile, aStartToken.getLine(), aStartToken.getStartIndex(), aEndToken.getStopIndex() + 1); }
From source file:org.eclipse.titan.designer.AST.ASN1.Block.java
License:Open Source License
public Block(final Token token) { if (token instanceof TokenWithIndexAndSubTokens) { tokenList = ((TokenWithIndexAndSubTokens) token).getSubTokens(); final IFile sourceFile = ((TokenWithIndexAndSubTokens) token).getSourceFile(); setLocation(new Location(sourceFile, token.getLine(), token.getStartIndex(), token.getStopIndex())); } else {// w ww.j ava2s .c o m setLocation(NULL_Location.INSTANCE); tokenList = ((TokenWithIndexAndSubTokens) token).getSubTokens(); } }
From source file:org.eclipse.titan.designer.AST.LargeLocation.java
License:Open Source License
/** * Constructor for ANTLR v4 tokens//from ww w .j a v a 2s . co m * @param aFile the parsed file * @param aStartToken the 1st token, its line and start position will be used for the location * NOTE: start position is the column index of the tokens 1st character. * Column index starts with 0. * @param aEndToken the last token, its end position will be used for the location. * NOTE: end position is the column index after the token's last character. */ public LargeLocation(IFile aFile, Token aStartToken, Token aEndToken) { super(aFile, aStartToken.getLine(), aStartToken.getStartIndex(), -1); endLine = -1; if (aEndToken != null) { setEndOffset(aEndToken.getStopIndex() + 1); endLine = aEndToken.getLine(); } }
From source file:org.eclipse.titan.designer.AST.Location.java
License:Open Source License
/** * Constructor for ANTLR v4 tokens//from w ww . j a v a 2s. c om * @param aFile the parsed file * @param aStartToken the 1st token, its line and start position will be used for the location * NOTE: start position is the column index of the tokens 1st character. * Column index starts with 0. * @param aEndToken the last token, its end position will be used for the location. * NOTE: end position is the column index after the token's last character. */ public Location(final IResource aFile, final Token aStartToken, final Token aEndToken) { setLocation(aFile, aStartToken.getLine(), aStartToken.getStartIndex(), aEndToken.getStopIndex() + 1); }
From source file:org.eclipse.titan.designer.AST.Location.java
License:Open Source License
/** * Sets the offset with an ANTLR v4 end token * @param aEndToken the new end token/* www . j av a 2s . c o m*/ */ public final void setOffset(final Token aToken) { this.setOffset(aToken.getStartIndex()); }
From source file:org.eclipse.titan.designer.parsers.asn1parser.TokenWithIndexAndSubTokens.java
License:Open Source License
public TokenWithIndexAndSubTokens(Token tok) { super(tok);/* w w w . j a va 2s . c om*/ tokenList = new ArrayList<Token>(); super.setStartIndex(tok.getStartIndex()); super.setStopIndex(tok.getStopIndex()); }
From source file:org.eclipse.titan.designer.parsers.ttcn3parser.ConditionalTransition.java
License:Open Source License
@Override public int fetch(int n) { if (fetchedEOF) { return 0; }//from w w w .ja va 2 s . c o m int i = 0; do { Token t; if (tokenStreamStack.isEmpty()) { t = getTokenSource().nextToken(); } else { t = tokenStreamStack.peek().getTokenSource().nextToken(); } if (t == null) { return 0; } int tokenType = t.getType(); if (tokenType == Ttcn3Lexer.PREPROCESSOR_DIRECTIVE) { lastPPDirectiveLocation = new Location(actualFile, t.getLine(), t.getStartIndex(), t.getStopIndex() + 1); // 1. the first # shall be discarded // 2. "\\\n" strings are removed, so multiline tokens, which are split by backslash are extracted to one line final String text = t.getText().substring(1).replace("\\\n", ""); Reader reader = new StringReader(text); CharStream charStream = new UnbufferedCharStream(reader); PreprocessorDirectiveLexer lexer = new PreprocessorDirectiveLexer(charStream); lexer.setTokenFactory(new PPDirectiveTokenFactory(true, t)); lexerListener = new PPListener(); lexer.removeErrorListeners(); lexer.addErrorListener(lexerListener); lexer.setLine(t.getLine()); lexer.setCharPositionInLine(t.getCharPositionInLine()); // 1. Previously it was UnbufferedTokenStream(lexer), but it was changed to BufferedTokenStream, because UnbufferedTokenStream seems to be unusable. It is an ANTLR 4 bug. // Read this: https://groups.google.com/forum/#!topic/antlr-discussion/gsAu-6d3pKU // pr_PatternChunk[StringBuilder builder, boolean[] uni]: // $builder.append($v.text); <-- exception is thrown here: java.lang.UnsupportedOperationException: interval 85..85 not in token buffer window: 86..341 // 2. Changed from BufferedTokenStream to CommonTokenStream, otherwise tokens with "-> channel(HIDDEN)" are not filtered out in lexer. final CommonTokenStream tokenStream = new CommonTokenStream(lexer); PreprocessorDirectiveParser localParser = new PreprocessorDirectiveParser(tokenStream); localParser.setBuildParseTree(false); parserListener = new PPListener(localParser); localParser.removeErrorListeners(); localParser.addErrorListener(parserListener); localParser.setIsActiveCode(condStateStack.isPassing()); localParser.setMacros(macros); localParser.setLine(t.getLine()); PreprocessorDirective ppDirective = null; ppDirective = localParser.pr_Directive().ppDirective; errorsStored.addAll(localParser.getErrorStorage()); warnings.addAll(localParser.getWarnings()); unsupportedConstructs.addAll(localParser.getUnsupportedConstructs()); if (ppDirective != null) { ppDirective.line = t.getLine(); if (ppDirective.isConditional()) { boolean preIsPassing = condStateStack.isPassing(); condStateStack.processDirective(ppDirective); boolean postIsPassing = condStateStack.isPassing(); if (preIsPassing != postIsPassing && tokenStreamStack.isEmpty() && getTokenSource() instanceof Ttcn3Lexer) { // included files are ignored because of ambiguity Location ppLocation = lastPPDirectiveLocation; if (ppLocation != null) { if (preIsPassing) { // switched to inactive: begin a new inactive location Location loc = new Location(actualFile, ppLocation.getLine(), ppLocation.getEndOffset(), ppLocation.getEndOffset()); inactiveCodeLocations.add(loc); } else { // switched to active: end the current inactive location int iclSize = inactiveCodeLocations.size(); if (iclSize > 0) { Location lastLocation = inactiveCodeLocations.get(iclSize - 1); lastLocation.setEndOffset(ppLocation.getOffset()); } } } } } else { // other directive types if (condStateStack.isPassing()) { // do something with the // directive switch (ppDirective.type) { case INCLUDE: { if (tokenStreamStack.size() > RECURSION_LIMIT) { // dumb but safe defense against infinite recursion, default value from gcc TITANMarker marker = new TITANMarker("Maximum #include recursion depth reached", ppDirective.line, -1, -1, IMarker.SEVERITY_ERROR, IMarker.PRIORITY_NORMAL); unsupportedConstructs.add(marker); } else { //TODO: Makes the Eclipse slow down processIncludeDirective(ppDirective); } } break; case ERROR: { String errorMessage = ppDirective.str == null ? "" : ppDirective.str; TITANMarker marker = new TITANMarker(errorMessage, ppDirective.line, -1, -1, IMarker.SEVERITY_ERROR, IMarker.PRIORITY_NORMAL); unsupportedConstructs.add(marker); } break; case WARNING: { String warningMessage = ppDirective.str == null ? "" : ppDirective.str; TITANMarker marker = new TITANMarker(warningMessage, ppDirective.line, -1, -1, IMarker.SEVERITY_WARNING, IMarker.PRIORITY_NORMAL); warnings.add(marker); } break; case LINECONTROL: case LINEMARKER: case PRAGMA: case NULL: { String reportPreference = Platform.getPreferencesService().getString( ProductConstants.PRODUCT_ID_DESIGNER, PreferenceConstants.REPORT_IGNORED_PREPROCESSOR_DIRECTIVES, GeneralConstants.WARNING, null); if (!GeneralConstants.IGNORE.equals(reportPreference)) { boolean isError = GeneralConstants.ERROR.equals(reportPreference); TITANMarker marker = new TITANMarker( MessageFormat.format("Preprocessor directive {0} is ignored", ppDirective.type.getName()), ppDirective.line, -1, -1, isError ? IMarker.SEVERITY_ERROR : IMarker.SEVERITY_WARNING, IMarker.PRIORITY_NORMAL); if (isError) { unsupportedConstructs.add(marker); } else { warnings.add(marker); } } } break; default: // ignore } } } } } else if (tokenType == Token.EOF) { if (!tokenStreamStack.isEmpty()) { // the included file ended, drop lexer // from the stack and ignore EOF token TokenStreamData tsd = tokenStreamStack.pop(); if (parser != null) { if (tokenStreamStack.isEmpty()) { parser.setActualFile(actualFile); parser.setLexer(actualLexer); } else { parser.setActualFile(tokenStreamStack.peek().file); parser.setLexer(tokenStreamStack.peek().lexer); } } if (tsd.reader != null) { try { tsd.reader.close(); } catch (IOException e) { } } } else { fetchedEOF = true; condStateStack.eofCheck(); tokens.add(t); ((CommonToken) t).setTokenIndex(tokens.size() - 1); --n; ++i; if (n == 0) { return i; } } } else { if (condStateStack.isPassing()) { tokens.add(t); ((CommonToken) t).setTokenIndex(tokens.size() - 1); --n; ++i; if (n == 0) { return i; } } } } while (true); }
From source file:org.eclipse.titan.runtime.core.cfgparser.CfgLocation.java
License:Open Source License
/** * Constructor for ANTLR v4 tokens//from w w w . j a v a 2s. c o m * @param aFile the parsed file * @param aStartToken the 1st token, its line and start position will be used for the location * NOTE: start position is the column index of the tokens 1st character. * Column index starts with 0. * @param aEndToken the last token, its end position will be used for the location. * NOTE: end position is the column index after the token's last character. */ public CfgLocation(final File aFile, final Token aStartToken, final Token aEndToken) { setLocation(aFile, aStartToken.getLine(), aStartToken.getStartIndex(), aEndToken.getStopIndex() + 1); }
From source file:org.elasticsearch.painless.ParserErrorStrategy.java
License:Apache License
@Override public void recover(Parser recognizer, RecognitionException re) { Token token = re.getOffendingToken(); String message;/*w ww . j a va 2 s . c o m*/ if (token == null) { message = "Error: no parse token found."; } else if (re instanceof InputMismatchException) { message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" + " unexpected token [" + getTokenErrorDisplay(token) + "]" + " was expecting one of [" + re.getExpectedTokens().toString(recognizer.getVocabulary()) + "]."; } else if (re instanceof NoViableAltException) { if (token.getType() == PainlessParser.EOF) { message = "Error: unexpected end of script."; } else { message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" + "invalid sequence of tokens near [" + getTokenErrorDisplay(token) + "]."; } } else { message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" + " unexpected token near [" + getTokenErrorDisplay(token) + "]."; } ParseException parseException = new ParseException(message, token == null ? -1 : token.getStartIndex()); parseException.initCause(re); throw new RuntimeException(parseException); }
From source file:org.elasticsearch.painless.ParserErrorStrategy.java
License:Apache License
@Override public Token recoverInline(Parser recognizer) throws RecognitionException { Token token = recognizer.getCurrentToken(); String message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" + " unexpected token [" + getTokenErrorDisplay(token) + "]" + " was expecting one of [" + recognizer.getExpectedTokens().toString(recognizer.getVocabulary()) + "]."; ParseException parseException = new ParseException(message, token.getStartIndex()); throw new RuntimeException(parseException); }