List of usage examples for org.antlr.v4.runtime IntStream EOF
int EOF
To view the source code for org.antlr.v4.runtime IntStream EOF.
Click Source Link
From source file:AntlrCaseInsensitiveInputStream.java
License:BSD License
@Override public int LA(int i) { if (i == 0) { return 0; // undefined }/*from w ww .j a v a2 s. co m*/ if (i < 0) { i++; // e.g., translate LA(-1) to use offset i=0; then data[p+0-1] if ((p + i - 1) < 0) { return IntStream.EOF; // invalid; no char before first char } } if ((p + i - 1) >= n) { return IntStream.EOF; } return lookaheadData[p + i - 1]; }
From source file:ai.grakn.graql.internal.parser.QueryParser.java
License:Open Source License
/** * Consume a single query from the given token stream. * * @param tokenStream the {@link TokenStream} to consume * @return a new {@link TokenSource} containing the tokens comprising the query *//*from ww w . j a v a 2s . c om*/ private TokenSource consumeOneQuery(TokenStream tokenStream) { List<Token> tokens = new ArrayList<>(); boolean startedQuery = false; while (true) { Token token = tokenStream.LT(1); boolean isNewQuery = NEW_QUERY_TOKENS.contains(token.getType()); boolean isEndOfTokenStream = token.getType() == IntStream.EOF; boolean isEndOfFirstQuery = startedQuery && isNewQuery; // Stop parsing tokens after reaching the end of the first query if (isEndOfTokenStream || isEndOfFirstQuery) break; if (isNewQuery) startedQuery = true; tokens.add(token); tokenStream.consume(); } return new ListTokenSource(tokens); }
From source file:com.antsdb.saltedfish.server.mysql.packet.MysqlCharStream.java
License:Open Source License
@Override public void consume() { if (p >= n) { assert LA(1) == IntStream.EOF; throw new IllegalStateException("cannot consume EOF"); }/*from w w w .ja v a2s.com*/ //System.out.println("prev p="+p+", c="+(char)data[p]); if (p < n) { p++; //System.out.println("p moves to "+p+" (c='"+(char)data[p]+"')"); } }
From source file:com.antsdb.saltedfish.server.mysql.packet.MysqlCharStream.java
License:Open Source License
@Override public int LA(int i) { if (i == 0) { return 0; // undefined }/*from w w w .j a v a2 s . c om*/ if (i < 0) { i++; // e.g., translate LA(-1) to use offset i=0; then data[p+0-1] if ((p + i - 1) < 0) { return IntStream.EOF; // invalid; no char before first char } } if ((p + i - 1) >= n) { //System.out.println("char LA("+i+")=EOF; p="+p); return IntStream.EOF; } //System.out.println("char LA("+i+")="+(char)data[p+i-1]+"; p="+p); //System.out.println("LA("+i+"); p="+p+" n="+n+" data.length="+data.length); int ch = this.buf.get(p + i - 1); return ch; }
From source file:com.cisco.yangide.core.parser.YangParserUtil.java
License:Open Source License
public static String formatYangSource(YangFormattingPreferences preferences, char[] content, int indentationLevel, String lineSeparator) { ANTLRInputStream input = new ANTLRInputStream(content, content.length); final YangLexer lexer = new YangLexer(input) { @Override/* ww w. j a va2s .co m*/ public void skip() { // disable skipping of comment tokens } }; LexerErrorListener errorListener = new LexerErrorListener(); lexer.addErrorListener(errorListener); final BufferedTokenStream tokens = new BufferedTokenStream(lexer); final ITokenFormatter formatter = new YangTokenFormatter(preferences, indentationLevel, lineSeparator); while (tokens.LT(1).getType() != IntStream.EOF) { formatter.process(tokens.LT(1)); tokens.consume(); } if (errorListener.isErrorDetected()) { // Source that contains parsing errors should never be formatted return String.valueOf(content); } return formatter.getFormattedContent(); }
From source file:com.facebook.presto.sql.parser.DelimiterLexer.java
License:Apache License
@Override public Token nextToken() { if (_input == null) { throw new IllegalStateException("nextToken requires a non-null input stream."); }//from w ww . ja va2s. c o m // Mark start location in char stream so unbuffered streams are // guaranteed at least have text of current token int tokenStartMarker = _input.mark(); try { outer: while (true) { if (_hitEOF) { emitEOF(); return _token; } _token = null; _channel = Token.DEFAULT_CHANNEL; _tokenStartCharIndex = _input.index(); _tokenStartCharPositionInLine = getInterpreter().getCharPositionInLine(); _tokenStartLine = getInterpreter().getLine(); _text = null; do { _type = Token.INVALID_TYPE; int ttype = -1; // This entire method is copied from org.antlr.v4.runtime.Lexer, with the following bit // added to match the delimiters before we attempt to match the token boolean found = false; for (String terminator : delimiters) { if (match(terminator)) { ttype = SqlBaseParser.DELIMITER; found = true; break; } } if (!found) { try { ttype = getInterpreter().match(_input, _mode); } catch (LexerNoViableAltException e) { notifyListeners(e); // report error recover(e); ttype = SKIP; } } if (_input.LA(1) == IntStream.EOF) { _hitEOF = true; } if (_type == Token.INVALID_TYPE) { _type = ttype; } if (_type == SKIP) { continue outer; } } while (_type == MORE); if (_token == null) { emit(); } return _token; } } finally { // make sure we release marker after match or // unbuffered char stream will keep buffering _input.release(tokenStartMarker); } }
From source file:io.prestosql.sql.parser.CaseInsensitiveStream.java
License:Apache License
@Override public int LA(int i) { int result = stream.LA(i); switch (result) { case 0:/*from w w w.ja v a 2s. c o m*/ case IntStream.EOF: return result; default: return Character.toUpperCase(result); } }
From source file:net.certiv.json.parser.LexerAdaptor.java
License:Open Source License
public boolean norLA(String... terminals) { ANTLRInputStream input = (ANTLRInputStream) getInputStream(); for (String str : terminals) { int index = 0; for (int idx = 0; idx < str.length(); idx++) { if (input.LA(index + 1) == IntStream.EOF) { break; }//from w ww.ja va2 s . co m char s = str.charAt(idx); char la = (char) input.LA(index + 1); if (s != la) { break; } index++; } if (index == str.length()) { return false; } } return true; }
From source file:net.certiv.json.parser.LexerAdaptor.java
License:Open Source License
public int skipToEol(ANTLRInputStream input, int index) { while (input.LA(index) != IntStream.EOF && input.LA(index) != '\n') { index++;//from w ww.j av a 2 s . com } return index; }
From source file:net.certiv.json.parser.LexerAdaptor.java
License:Open Source License
public int skipToEoc(ANTLRInputStream input, int index) { while (input.LA(index) != IntStream.EOF) { if (input.LA(index) == '/' && input.LA(index - 1) == '*') { return index; }//from ww w.ja v a2 s. c om index++; } return index; }