Example usage for org.antlr.v4.runtime Lexer HIDDEN

Introduction

In this page you can find the example usage for org.antlr.v4.runtime Lexer HIDDEN.

Prototype

int HIDDEN

To view the source code for org.antlr.v4.runtime Lexer HIDDEN.

Click Source Link

Usage

From source file:com.microsoft.thrifty.schema.parser.ThriftListener.java

License:Open Source License

private List<Token> getLeadingComments(Token token) {
    List<Token> hiddenTokens = tokenStream.getHiddenTokensToLeft(token.getTokenIndex(), Lexer.HIDDEN);

    if (hiddenTokens == null || hiddenTokens.isEmpty()) {
        return Collections.emptyList();
    }//  w  w w. j a  v a2s . c o m

    List<Token> comments = new ArrayList<>(hiddenTokens.size());
    for (Token hiddenToken : hiddenTokens) {
        if (isComment(hiddenToken) && !trailingDocTokenIndexes.get(hiddenToken.getTokenIndex())) {
            comments.add(hiddenToken);
        }
    }

    return comments;
}

---

From source file:com.microsoft.thrifty.schema.parser.ThriftListener.java

License:Open Source License

/**
 * Read comments following the given token, until the first newline is encountered.
 *
 * INVARIANT://from   w  ww.j a  v a  2s  . c o m
 * Assumes that the parse tree is being walked top-down, left to right!
 *
 * Trailing-doc tokens are marked as such, so that subsequent searches for "leading"
 * doc don't grab tokens already used as "trailing" doc.  If the walk order is *not*
 * top-down, left-to-right, then the assumption underpinning the separation of leading
 * and trailing comments is broken.
 *
 * @param endToken the token from which to search for trailing comment tokens.
 * @return a list, possibly empty, of all trailing comment tokens.
 */
private List<Token> getTrailingComments(Token endToken) {
    List<Token> hiddenTokens = tokenStream.getHiddenTokensToRight(endToken.getTokenIndex(), Lexer.HIDDEN);

    if (hiddenTokens == null || hiddenTokens.isEmpty()) {
        return Collections.emptyList();
    }

    Token maybeTrailingDoc = hiddenTokens.get(0); // only one trailing comment is possible

    if (isComment(maybeTrailingDoc)) {
        trailingDocTokenIndexes.set(maybeTrailingDoc.getTokenIndex());
        return Collections.singletonList(maybeTrailingDoc);
    }

    return Collections.emptyList();
}

---

From source file:net.sourceforge.pmd.cpd.SwiftTokenizer.java

License:BSD License

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();

    try {// ww  w .  j a v  a 2  s .  c  o m
        ANTLRInputStream ais = new ANTLRInputStream(buffer.toString());
        SwiftLexer lexer = new SwiftLexer(ais);

        lexer.removeErrorListeners();
        lexer.addErrorListener(new ErrorHandler());
        Token token = lexer.nextToken();

        while (token.getType() != Token.EOF) {
            if (token.getChannel() != Lexer.HIDDEN) {
                TokenEntry tokenEntry = new TokenEntry(token.getText(), sourceCode.getFileName(),
                        token.getLine());

                tokenEntries.add(tokenEntry);
            }
            token = lexer.nextToken();
        }
    } catch (ANTLRSyntaxError err) {
        // Wrap exceptions of the Swift tokenizer in a TokenMgrError, so
        // they are correctly handled
        // when CPD is executed with the '--skipLexicalErrors' command line
        // option
        throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line "
                + err.getLine() + ", column " + err.getColumn() + ".  Encountered: " + err.getMessage(),
                TokenMgrError.LEXICAL_ERROR);
    } finally {
        tokenEntries.add(TokenEntry.getEOF());
    }
}

---

From source file:net.sourceforge.pmd.cpd.token.AntlrToken.java

License:BSD License

public boolean isHidden() {
    return token.getChannel() == Lexer.HIDDEN;
}

---