Example usage for org.eclipse.jdt.internal.core.dom.rewrite TokenScanner isComment

Introduction

In this page you can find the example usage for org.eclipse.jdt.internal.core.dom.rewrite TokenScanner isComment.

Prototype

public static boolean isComment(int token) 

Source Link

Usage

From source file:comparison.SJavaTokenComparator.java

License:Open Source License

/**
 * Creates a token comparator for the given string.
 *
 * @param text the text to be tokenized//from   w  w w  .jav a 2  s .c  o m
 * @param textTokenComparatorFactory a factory to create text token comparators
 */
public SJavaTokenComparator(String text) {

    fText = text;

    int length = fText.length();
    fStarts = new int[length];
    fLengths = new int[length];
    fTokens = new String[length];
    fCount = 0;

    IScanner scanner = ToolFactory.createScanner(true, false, false, false); // returns comments & whitespace
    scanner.setSource(fText.toCharArray());
    int endPos = 0;
    try {
        int tokenType;
        while ((tokenType = scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
            int start = scanner.getCurrentTokenStartPosition();
            int end = scanner.getCurrentTokenEndPosition() + 1;
            // Comments are treated as a single token (see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=78063)
            if (!TokenScanner.isComment(tokenType)) {
                recordTokenRange(start, end - start);
            }
            endPos = end;
        }
    } catch (InvalidInputException ex) {
        // We couldn't parse part of the input. Fall through and make the rest a single token
    }
    // Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=13907
    if (endPos < length) {
        recordTokenRange(endPos, length - endPos);
    }
}

---