List of usage examples for org.antlr.v4.runtime Token EOF
int EOF
To view the source code for org.antlr.v4.runtime Token EOF.
Click Source Link
From source file:ai.grakn.graql.internal.parser.QueryParserImpl.java
License:Open Source License
/** * @param reader a reader representing several queries * @return a list of queries//from w w w .j a v a2s .c o m */ @Override public <T extends Query<?>> Stream<T> parseList(Reader reader) { UnbufferedCharStream charStream = new UnbufferedCharStream(reader); GraqlErrorListener errorListener = GraqlErrorListener.withoutQueryString(); GraqlLexer lexer = createLexer(charStream, errorListener); /* We tell the lexer to copy the text into each generated token. Normally when calling `Token#getText`, it will look into the underlying `TokenStream` and call `TokenStream#size` to check it is in-bounds. However, `UnbufferedTokenStream#size` is not supported (because then it would have to read the entire input). To avoid this issue, we set this flag which will copy over the text into each `Token`, s.t. that `Token#getText` will just look up the copied text field. */ lexer.setTokenFactory(new CommonTokenFactory(true)); // Use an unbuffered token stream so we can handle extremely large input strings UnbufferedTokenStream tokenStream = new UnbufferedTokenStream(ChannelTokenSource.of(lexer)); GraqlParser parser = createParser(tokenStream, errorListener); /* The "bail" error strategy prevents us reading all the way to the end of the input, e.g. ``` match $x isa person; insert $x has name "Bob"; match $x isa movie; get; ^ ``` In this example, when ANTLR reaches the indicated `match`, it considers two possibilities: 1. this is the end of the query 2. the user has made a mistake. Maybe they accidentally pasted the `match` here. Because of case 2, ANTLR will parse beyond the `match` in order to produce a more helpful error message. This causes memory issues for very large queries, so we use the simpler "bail" strategy that will immediately stop when it hits `match`. */ parser.setErrorHandler(new BailErrorStrategy()); // This is a lazy iterator that will only consume a single query at a time, without parsing any further. // This means it can pass arbitrarily long streams of queries in constant memory! Iterable<T> queryIterator = () -> new AbstractIterator<T>() { @Nullable @Override protected T computeNext() { int latestToken = tokenStream.LA(1); if (latestToken == Token.EOF) { endOfData(); return null; } else { // This will parse and consume a single query, even if it doesn't reach an EOF // When we next run it, it will start where it left off in the stream return (T) QUERY.parse(parser, errorListener); } } }; return StreamSupport.stream(queryIterator.spliterator(), false); }
From source file:com.github.gfx.android.orma.migration.sqliteparser.SQLiteParserUtils.java
License:Apache License
static void appendTokenList(final SQLiteComponent component, ParseTree node) { node.accept(new AbstractParseTreeVisitor<Void>() { @Override/*from w w w . j a v a 2 s . com*/ public Void visitTerminal(TerminalNode node) { int type = node.getSymbol().getType(); if (type == Token.EOF) { return null; } if (node.getParent() instanceof SQLiteParser.Any_nameContext) { component.tokens.add(new SQLiteComponent.Name(node.getText())); } else if (isKeyword(type)) { component.tokens.add(new SQLiteComponent.Keyword(node.getText())); } else { component.tokens.add(node.getText()); } return null; } }); }
From source file:com.huawei.streaming.cql.semanticanalyzer.parser.CQLErrorStrategy.java
License:Apache License
/** * {@inheritDoc}/*from w ww . ja v a2 s.c o m*/ */ @Override public void reportNoViableAlternative(@NotNull Parser recognizer, @NotNull NoViableAltException e) { TokenStream tokens = recognizer.getInputStream(); String input; if (tokens instanceof TokenStream) { if (e.getStartToken().getType() == Token.EOF) input = "<EOF>"; else input = getText(tokens, e.getStartToken(), e.getOffendingToken()); } else { input = "<unknown input>"; } String msg = "no viable alternative at input " + escapeWSAndQuote(input); recognizer.notifyErrorListeners(e.getOffendingToken(), msg, e); }
From source file:com.huawei.streaming.cql.semanticanalyzer.parser.CQLErrorStrategy.java
License:Apache License
@NotNull private String getText(TokenStream tokens, Interval interval) { int start = interval.a; int stop = interval.b; if (start < 0 || stop < 0) return ""; if (stop >= tokens.size()) stop = tokens.size() - 1;//from w w w.j av a 2 s . co m StringBuilder buf = new StringBuilder(); for (int i = start; i <= stop; i++) { Token t = tokens.get(i); if (t.getType() == Token.EOF) break; buf.append(t.getText()); if (i != stop) { buf.append(" "); } } return buf.toString(); }
From source file:com.oracle.truffle.dsl.processor.expression.ExpressionParser.java
License:Open Source License
public final Comparison_factorContext comparison_factor() throws RecognitionException { Comparison_factorContext _localctx = new Comparison_factorContext(_ctx, getState()); enterRule(_localctx, 4, RULE_comparison_factor); int _la;// w w w. j av a 2 s. c o m try { enterOuterAlt(_localctx, 1); { setState(24); _localctx.f1 = negate_factor(); _localctx.result = _localctx.f1.result; setState(30); _errHandler.sync(this); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__1) | (1L << T__2) | (1L << T__3) | (1L << T__4) | (1L << T__5) | (1L << T__6))) != 0)) { { setState(26); _localctx.op = _input.LT(1); _la = _input.LA(1); if (!((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__1) | (1L << T__2) | (1L << T__3) | (1L << T__4) | (1L << T__5) | (1L << T__6))) != 0))) { _localctx.op = _errHandler.recoverInline(this); } else { if (_input.LA(1) == Token.EOF) matchedEOF = true; _errHandler.reportMatch(this); consume(); } setState(27); _localctx.f2 = negate_factor(); _localctx.result = new Binary((_localctx.op != null ? _localctx.op.getText() : null), _localctx.result, _localctx.f2.result); } } } } catch (RecognitionException re) { _localctx.exception = re; _errHandler.reportError(this, re); _errHandler.recover(this, re); } finally { exitRule(); } return _localctx; }
From source file:com.oracle.truffle.sl.parser.SimpleLanguageParser.java
License:Open Source License
public final Logic_factorContext logic_factor() throws RecognitionException { Logic_factorContext _localctx = new Logic_factorContext(_ctx, getState()); enterRule(_localctx, 18, RULE_logic_factor); int _la;/*from w w w . j a v a 2 s .co m*/ try { enterOuterAlt(_localctx, 1); { setState(147); _localctx.arithmetic = arithmetic(); _localctx.result = _localctx.arithmetic.result; setState(153); _errHandler.sync(this); switch (getInterpreter().adaptivePredict(_input, 9, _ctx)) { case 1: { setState(149); _localctx.op = _input.LT(1); _la = _input.LA(1); if (!((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__16) | (1L << T__17) | (1L << T__18) | (1L << T__19) | (1L << T__20) | (1L << T__21))) != 0))) { _localctx.op = _errHandler.recoverInline(this); } else { if (_input.LA(1) == Token.EOF) matchedEOF = true; _errHandler.reportMatch(this); consume(); } setState(150); _localctx.arithmetic = arithmetic(); _localctx.result = factory.createBinary(_localctx.op, _localctx.result, _localctx.arithmetic.result); } break; } } } catch (RecognitionException re) { _localctx.exception = re; _errHandler.reportError(this, re); _errHandler.recover(this, re); } finally { exitRule(); } return _localctx; }
From source file:com.oracle.truffle.sl.parser.SimpleLanguageParser.java
License:Open Source License
public final ArithmeticContext arithmetic() throws RecognitionException { ArithmeticContext _localctx = new ArithmeticContext(_ctx, getState()); enterRule(_localctx, 20, RULE_arithmetic); int _la;// www .ja va 2s .c o m try { int _alt; enterOuterAlt(_localctx, 1); { setState(155); _localctx.term = term(); _localctx.result = _localctx.term.result; setState(163); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input, 10, _ctx); while (_alt != 2 && _alt != org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER) { if (_alt == 1) { { { setState(157); _localctx.op = _input.LT(1); _la = _input.LA(1); if (!(_la == T__22 || _la == T__23)) { _localctx.op = _errHandler.recoverInline(this); } else { if (_input.LA(1) == Token.EOF) matchedEOF = true; _errHandler.reportMatch(this); consume(); } setState(158); _localctx.term = term(); _localctx.result = factory.createBinary(_localctx.op, _localctx.result, _localctx.term.result); } } } setState(165); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input, 10, _ctx); } } } catch (RecognitionException re) { _localctx.exception = re; _errHandler.reportError(this, re); _errHandler.recover(this, re); } finally { exitRule(); } return _localctx; }
From source file:com.oracle.truffle.sl.parser.SimpleLanguageParser.java
License:Open Source License
public final TermContext term() throws RecognitionException { TermContext _localctx = new TermContext(_ctx, getState()); enterRule(_localctx, 22, RULE_term); int _la;/* w w w .j a v a 2 s. c o m*/ try { int _alt; enterOuterAlt(_localctx, 1); { setState(166); _localctx.factor = factor(); _localctx.result = _localctx.factor.result; setState(174); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input, 11, _ctx); while (_alt != 2 && _alt != org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER) { if (_alt == 1) { { { setState(168); _localctx.op = _input.LT(1); _la = _input.LA(1); if (!(_la == T__24 || _la == T__25)) { _localctx.op = _errHandler.recoverInline(this); } else { if (_input.LA(1) == Token.EOF) matchedEOF = true; _errHandler.reportMatch(this); consume(); } setState(169); _localctx.factor = factor(); _localctx.result = factory.createBinary(_localctx.op, _localctx.result, _localctx.factor.result); } } } setState(176); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input, 11, _ctx); } } } catch (RecognitionException re) { _localctx.exception = re; _errHandler.reportError(this, re); _errHandler.recover(this, re); } finally { exitRule(); } return _localctx; }
From source file:com.sample.JavaErrorStrategy.java
License:BSD License
/** * The default implementation of {@link ANTLRErrorStrategy#sync} makes sure * that the current lookahead symbol is consistent with what were expecting * at this point in the ATN. You can call this anytime but ANTLR only * generates code to check before subrules/loops and each iteration. * * <p>/* w ww . ja va2s . co m*/ * Implements Jim Idle's magic sync mechanism in closures and optional * subrules. E.g., * </p> * * <pre> * a : sync ( stuff sync )* ; * sync : {consume to what can follow sync} ; * </pre> * * At the start of a sub rule upon error, {@link #sync} performs single * token deletion, if possible. If it can't do that, it bails on the current * rule and uses the default error recovery, which consumes until the * resynchronization set of the current rule. * * <p> * If the sub rule is optional ({@code (...)?}, {@code (...)*}, or block * with an empty alternative), then the expected set includes what follows * the subrule. * </p> * * <p> * During loop iteration, it consumes until it sees a token that can start a * sub rule or what follows loop. Yes, that is pretty aggressive. We opt to * stay in the loop as long as possible. * </p> * * <p> * <strong>ORIGINS</strong> * </p> * * <p> * Previous versions of ANTLR did a poor job of their recovery within loops. * A single mismatch token or missing token would force the parser to bail * out of the entire rules surrounding the loop. So, for rule * </p> * * <pre> * classDef : 'class' ID '{' member* '}' * </pre> * * input with an extra token between members would force the parser to * consume until it found the next class definition rather than the next * member definition of the current class. * * <p> * This functionality cost a little bit of effort because the parser has to * compare token set at the start of the loop and at each iteration. If for * some reason speed is suffering for you, you can turn off this * functionality by simply overriding this method as a blank { }. * </p> */ @Override public void sync(Parser recognizer) throws RecognitionException { ATNState s = recognizer.getInterpreter().atn.states.get(recognizer.getState()); // System.err.println("sync @ "+s.stateNumber+"="+s.getClass().getSimpleName()); // If already recovering, don't try to sync if (inErrorRecoveryMode(recognizer)) { return; } TokenStream tokens = recognizer.getInputStream(); int la = tokens.LA(1); // try cheaper subset first; might get lucky. seems to shave a wee bit // off if (recognizer.getATN().nextTokens(s).contains(la) || la == Token.EOF) return; // Return but don't end recovery. only do that upon valid token match if (recognizer.isExpectedToken(la)) { return; } switch (s.getStateType()) { case ATNState.BLOCK_START: case ATNState.STAR_BLOCK_START: case ATNState.PLUS_BLOCK_START: case ATNState.STAR_LOOP_ENTRY: // report error and recover if possible if (singleTokenDeletion(recognizer) != null) { return; } throw new InputMismatchException(recognizer); case ATNState.PLUS_LOOP_BACK: case ATNState.STAR_LOOP_BACK: // System.err.println("at loop back: "+s.getClass().getSimpleName()); reportUnwantedToken(recognizer); IntervalSet expecting = recognizer.getExpectedTokens(); IntervalSet whatFollowsLoopIterationOrRule = expecting.or(getErrorRecoverySet(recognizer)); consumeUntil(recognizer, whatFollowsLoopIterationOrRule); break; default: // do nothing if we can't identify the exact kind of ATN state break; } }
From source file:com.sample.JavaErrorStrategy.java
License:BSD License
/** * This is called by {@link #reportError} when the exception is a * {@link NoViableAltException}./*from w ww. j av a 2 s . com*/ * * @see #reportError * * @param recognizer * the parser instance * @param e * the recognition exception */ protected void reportNoViableAlternative(@NotNull Parser recognizer, @NotNull NoViableAltException e) { TokenStream tokens = recognizer.getInputStream(); String input; if (tokens != null) { if (e.getStartToken().getType() == Token.EOF) input = "<EOF>"; else input = tokens.getText(e.getStartToken(), e.getOffendingToken()); } else { input = "<unknown input>"; } String msg = "no viable alternative at input " + escapeWSAndQuote(input); recognizer.notifyErrorListeners(e.getOffendingToken(), msg, e); }