Example usage for org.antlr.v4.runtime Token DEFAULT

Introduction

In this page you can find the example usage for org.antlr.v4.runtime Token DEFAULT_CHANNEL.

Prototype

int DEFAULT_CHANNEL

To view the source code for org.antlr.v4.runtime Token DEFAULT_CHANNEL.

Click Source Link

Document

All tokens go to the parser (unless skip() is called in that rule) on a particular "channel".

Usage

From source file:ai.grakn.graql.internal.parser.ChannelTokenSource.java

License:Open Source License

public static ChannelTokenSource of(TokenSource source) {
    return of(source, Token.DEFAULT_CHANNEL);
}

From source file:com.facebook.presto.sql.parser.DelimiterLexer.java

License:Apache License

@Override
public Token nextToken() {
    if (_input == null) {
        throw new IllegalStateException("nextToken requires a non-null input stream.");
    }//from   ww w  .  j  a v a  2  s.  co m

    // Mark start location in char stream so unbuffered streams are
    // guaranteed at least have text of current token
    int tokenStartMarker = _input.mark();
    try {
        outer: while (true) {
            if (_hitEOF) {
                emitEOF();
                return _token;
            }

            _token = null;
            _channel = Token.DEFAULT_CHANNEL;
            _tokenStartCharIndex = _input.index();
            _tokenStartCharPositionInLine = getInterpreter().getCharPositionInLine();
            _tokenStartLine = getInterpreter().getLine();
            _text = null;
            do {
                _type = Token.INVALID_TYPE;
                int ttype = -1;

                // This entire method is copied from org.antlr.v4.runtime.Lexer, with the following bit
                // added to match the delimiters before we attempt to match the token
                boolean found = false;
                for (String terminator : delimiters) {
                    if (match(terminator)) {
                        ttype = SqlBaseParser.DELIMITER;
                        found = true;
                        break;
                    }
                }

                if (!found) {
                    try {
                        ttype = getInterpreter().match(_input, _mode);
                    } catch (LexerNoViableAltException e) {
                        notifyListeners(e); // report error
                        recover(e);
                        ttype = SKIP;
                    }
                }

                if (_input.LA(1) == IntStream.EOF) {
                    _hitEOF = true;
                }
                if (_type == Token.INVALID_TYPE) {
                    _type = ttype;
                }
                if (_type == SKIP) {
                    continue outer;
                }
            } while (_type == MORE);
            if (_token == null) {
                emit();
            }
            return _token;
        }
    } finally {
        // make sure we release marker after match or
        // unbuffered char stream will keep buffering
        _input.release(tokenStartMarker);
    }
}

From source file:com.sample.JavaErrorStrategy.java

License:BSD License

/**
 * Conjure up a missing token during error recovery.
 *
 * The recognizer attempts to recover from single missing symbols. But,
 * actions might refer to that missing symbol. For example, x=ID {f($x);}.
 * The action clearly assumes that there has been an identifier matched
 * previously and that $x points at that token. If that token is missing,
 * but the next token in the stream is what we want we assume that this
 * token is missing and we keep going. Because we have to return some token
 * to replace the missing token, we have to conjure one up. This method
 * gives the user control over the tokens returned for missing tokens.
 * Mostly, you will want to create something special for identifier tokens.
 * For literals such as '{' and ',', the default action in the parser or
 * tree parser works. It simply creates a CommonToken of the appropriate
 * type. The text will be the token. If you change what tokens must be
 * created by the lexer, override this method to create the appropriate
 * tokens./*from   w  w w .j  av a  2  s . c om*/
 */
@NotNull
protected Token getMissingSymbol(@NotNull Parser recognizer) {
    Token currentSymbol = recognizer.getCurrentToken();
    IntervalSet expecting = getExpectedTokens(recognizer);
    int expectedTokenType = expecting.getMinElement(); // get any element
    String tokenText;
    if (expectedTokenType == Token.EOF)
        tokenText = "<missing EOF>";
    else
        tokenText = "<missing " + recognizer.getTokenNames()[expectedTokenType] + ">";
    Token current = currentSymbol;
    Token lookback = recognizer.getInputStream().LT(-1);
    if (current.getType() == Token.EOF && lookback != null) {
        current = lookback;
    }
    return recognizer.getTokenFactory()
            .create(new Pair<TokenSource, CharStream>(current.getTokenSource(),
                    current.getTokenSource().getInputStream()), expectedTokenType, tokenText,
                    Token.DEFAULT_CHANNEL, -1, -1, current.getLine(), current.getCharPositionInLine());
}

From source file:org.apache.sysml.parser.pydml.PydmlLexer.java

License:Apache License

@Override
public Token nextToken() {
    if (_input.LA(1) == EOF && !this.indents.isEmpty()) {
        if (debugIndentRules)
            System.out.println("EOF reached and expecting some DEDENTS, so emitting them");

        tokens.poll();//  ww  w .j av  a  2  s.co m
        this.emit(commonToken(PydmlParser.NEWLINE, "\n"));

        // Now emit as much DEDENT tokens as needed.
        while (!indents.isEmpty()) {
            if (debugIndentRules)
                System.out.println("Emitting (inserted) DEDENTS");

            this.emit(createDedent());
            indents.pop();
        }
        // Put the EOF back on the token stream.
        this.emit(commonToken(PydmlParser.EOF, "<EOF>"));
    }
    Token next = super.nextToken();
    if (next.getChannel() == Token.DEFAULT_CHANNEL) {
        // Keep track of the last token on the default channel.
        this.lastToken = next;
    }
    Token retVal = tokens.isEmpty() ? next : tokens.poll();

    if (debugIndentRules)
        System.out.println("Returning nextToken: [" + retVal + "]<<" + tokens.isEmpty());

    return retVal;
}

From source file:org.ballerinalang.composer.service.workspace.langserver.util.completion.filters.PackageActionAndFunctionFilter.java

License:Open Source License

/**
 * Get the index of a certain token//from www.  j  a  v a  2s.com
 * @param tokenString - token string
 * @param from - start searching from
 * @param dataModel - suggestions filter data model
 * @return {@link Integer}
 */
public int getIndexOfTokenString(String tokenString, int from, SuggestionsFilterDataModel dataModel) {
    TokenStream tokenStream = dataModel.getTokenStream();
    int resultTokenIndex = -1;
    int searchIndex = from;

    while (true) {
        if (searchIndex < 0 || tokenStream.size() - 1 < searchIndex) {
            break;
        }
        Token token = tokenStream.get(searchIndex);
        if (token.getChannel() != Token.DEFAULT_CHANNEL || !token.getText().equals(tokenString)) {
            searchIndex++;
        } else {
            resultTokenIndex = searchIndex;
            break;
        }
    }

    return resultTokenIndex;
}

From source file:org.ballerinalang.composer.service.workspace.langserver.util.completion.filters.PackageActionAndFunctionFilter.java

License:Open Source License

private int getPackageDelimeterTokenIndex(SuggestionsFilterDataModel dataModel) {
    ArrayList<String> terminalTokens = new ArrayList<>(Arrays.asList(new String[] { ";", "}", "{" }));
    int currentTokenIndex = dataModel.getTokenIndex();
    int searchTokenIndex = currentTokenIndex;
    TokenStream tokenStream = dataModel.getTokenStream();
    int delimiterIndex = -1;
    String currentTokenStr = tokenStream.get(searchTokenIndex).getText();

    if (terminalTokens.contains(currentTokenStr)) {
        searchTokenIndex -= 1;//from   w w  w  .j av  a  2 s  . c  o  m
        while (true) {
            if (tokenStream.get(searchTokenIndex).getChannel() == Token.DEFAULT_CHANNEL) {
                break;
            } else {
                searchTokenIndex -= 1;
            }
        }
    }

    while (true) {
        if (searchTokenIndex >= tokenStream.size()) {
            break;
        }
        String tokenString = tokenStream.get(searchTokenIndex).getText();
        if (".".equals(tokenString) || ":".equals(tokenString)) {
            delimiterIndex = searchTokenIndex;
            break;
        } else if (terminalTokens.contains(tokenString)) {
            break;
        } else {
            searchTokenIndex++;
        }
    }

    return delimiterIndex;
}

From source file:org.ballerinalang.composer.service.workspace.langserver.util.completion.resolvers.AbstractItemResolver.java

License:Open Source License

/**
 * Check whether the token stream corresponds to a action invocation or a function invocation
 * @param dataModel - Suggestions filter data model
 * @return {@link Boolean}/*  ww  w  .j a  va 2  s  .  c om*/
 */
protected boolean isActionOrFunctionInvocationStatement(SuggestionsFilterDataModel dataModel) {
    ArrayList<String> terminalTokens = new ArrayList<>(Arrays.asList(new String[] { ";", "}", "{" }));
    TokenStream tokenStream = dataModel.getTokenStream();
    int searchTokenIndex = dataModel.getTokenIndex();
    String currentTokenStr = tokenStream.get(searchTokenIndex).getText();

    if (terminalTokens.contains(currentTokenStr)) {
        searchTokenIndex -= 1;
        while (true) {
            if (tokenStream.get(searchTokenIndex).getChannel() == Token.DEFAULT_CHANNEL) {
                break;
            } else {
                searchTokenIndex -= 1;
            }
        }
    }

    while (true) {
        if (searchTokenIndex >= tokenStream.size()) {
            return false;
        }
        String tokenString = tokenStream.get(searchTokenIndex).getText();
        if (terminalTokens.contains(tokenString)) {
            return false;
        } else if (tokenString.equals(".") || tokenString.equals(":")) {
            return true;
        } else {
            searchTokenIndex++;
        }
    }
}

From source file:org.ballerinalang.composer.service.workspace.langserver.util.filters.PackageActionAndFunctionFilter.java

License:Open Source License

/**
 * Get the index of a certain token//from  w w  w  . j av a  2  s . co  m
 * @param tokenString - token string
 * @param from - start searching from
 * @param dataModel - suggestions filter data model
 * @return {@link Integer}
 */
private int getIndexOfTokenString(String tokenString, int from, SuggestionsFilterDataModel dataModel) {
    TokenStream tokenStream = dataModel.getTokenStream();
    int resultTokenIndex = -1;
    int searchIndex = from;

    while (true) {
        if (searchIndex < 0 || tokenStream.size() - 1 < searchIndex) {
            break;
        }
        Token token = tokenStream.get(searchIndex);
        if (token.getChannel() != Token.DEFAULT_CHANNEL || !token.getText().equals(tokenString)) {
            searchIndex++;
        } else {
            resultTokenIndex = searchIndex;
            break;
        }
    }

    return resultTokenIndex;
}

From source file:org.ballerinalang.composer.service.workspace.langserver.util.resolvers.parsercontext.ParserRuleExpressionVariableDefStatementContextResolver.java

License:Open Source License

/**
 * Get the index of the given token/* ww w.j av  a2  s.co m*/
 * @param dataModel suggestions filter data model
 * @param tokenString token string to be found
 * @return {@link Integer}
 */
private int getTokenIndexOf(SuggestionsFilterDataModel dataModel, String tokenString) {
    TokenStream tokenStream = dataModel.getTokenStream();
    int currentTokenIndex = dataModel.getTokenIndex();
    int searchIndex = currentTokenIndex - 1;
    List<String> terminatingCharacters = Arrays.asList(";", "{", "}", "@");

    while (true) {
        if (tokenStream != null && searchIndex < tokenStream.size()) {
            Token token = tokenStream.get(searchIndex);
            String tokenStr = token.getText();

            if (token.getChannel() == Token.DEFAULT_CHANNEL && terminatingCharacters.contains(tokenStr)) {
                return -1;
            } else if (tokenStr.equals(tokenString)) {
                return searchIndex;
            } else {
                searchIndex--;
            }
        } else {
            return -1;
        }
    }
}

From source file:org.ballerinalang.langserver.common.utils.CommonUtil.java

License:Open Source License

private static Token getDefaultTokenToLeftOrRight(TokenStream tokenStream, int startIndex, int direction) {
    Token token = null;/* w w w  .  j av  a  2s.c o m*/
    while (true) {
        startIndex += direction;
        if (startIndex < 0 || startIndex == tokenStream.size()) {
            break;
        }
        token = tokenStream.get(startIndex);
        if (token.getChannel() == Token.DEFAULT_CHANNEL) {
            break;
        }
    }
    return token;
}

Example usage for org.antlr.v4.runtime Token DEFAULT_CHANNEL

Introduction

Prototype

Document

Usage