Example usage for org.eclipse.jdt.core.compiler IScanner getCurrentTokenEndPosition

List of usage examples for org.eclipse.jdt.core.compiler IScanner getCurrentTokenEndPosition

  1. HOME
  2. Java
  3. org.eclipse.jdt
  4. org.eclipse.jdt.core.compiler.*
  5. IScanner
  6. getCurrentTokenEndPosition

Introduction

In this page you can find the example usage for org.eclipse.jdt.core.compiler IScanner getCurrentTokenEndPosition.

Prototype

int getCurrentTokenEndPosition();

Source Link

Document

Answers the ending position of the current token inside the original source.

Usage

From source file:ca.uvic.cs.tagsea.extraction.TagExtractor.java

License:Open Source License

/**
 * Gets the comment regions associated with this document, restricted to the given offset and length, if no comment regions 
 * are found an empty array is returned/*  w  w w  . j  a  v  a 2s .c o  m*/
 * @param document
 * @param offset
 * @param length
 * @param returnSingleLine Wither to collect single line commments
 * @return Array of tag regions
 */
public static IRegion[] getCommentRegions(IDocument document, int offset, int length) {
    try {
        StringBuffer content = new StringBuffer(document.get(offset, length));
        IRegion range = new Region(offset, length);
        List<IRegion> commentRegions = new ArrayList<IRegion>();
        IScanner scanner = ToolFactory.createScanner(true, false, false, false);
        scanner.setSource(content.toString().toCharArray());

        int shift = range.getOffset();

        while (true) {
            int terminal = scanner.getNextToken();

            if (terminal == ITerminalSymbols.TokenNameCOMMENT_JAVADOC
                    || terminal == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
                int commentOffset = shift + scanner.getCurrentTokenStartPosition();
                int commentEnd = shift + scanner.getCurrentTokenEndPosition() + 1;
                commentRegions.add(new Region(commentOffset, commentEnd - commentOffset));
            } else if (terminal == ITerminalSymbols.TokenNameCOMMENT_LINE) {
                int commentOffset = shift + scanner.getCurrentTokenStartPosition();
                int commentEnd = shift + scanner.getCurrentTokenEndPosition() + 1;
                commentRegions.add(new Region(commentOffset, commentEnd - commentOffset));
            } else if (terminal == ITerminalSymbols.TokenNameEOF)
                break;
        }

        IRegion[] result = new IRegion[commentRegions.size()];
        commentRegions.toArray(result);
        return result;

    } catch (BadLocationException e) {
        TagSEAPlugin.log("", e);
    } catch (InvalidInputException e) {
        TagSEAPlugin.log("", e);
    }

    return new IRegion[0];
}

From source file:com.android.ide.eclipse.adt.internal.refactorings.extractstring.ExtractStringRefactoring.java

License:Open Source License

/**
 * Try to find the selected Java element in the compilation unit.
 *
 * If selection matches a string literal, capture it, otherwise add a fatal error
 * to the status./*www  .j  a v  a  2  s  .co m*/
 *
 * On success, advance the monitor by 3.
 * Returns status.isOK().
 */
private boolean findSelectionInJavaUnit(ICompilationUnit unit, RefactoringStatus status,
        IProgressMonitor monitor) {
    try {
        IBuffer buffer = unit.getBuffer();

        IScanner scanner = ToolFactory.createScanner(false, //tokenizeComments
                false, //tokenizeWhiteSpace
                false, //assertMode
                false //recordLineSeparator
        );
        scanner.setSource(buffer.getCharacters());
        monitor.worked(1);

        for (int token = scanner.getNextToken(); token != ITerminalSymbols.TokenNameEOF; token = scanner
                .getNextToken()) {
            if (scanner.getCurrentTokenStartPosition() <= mSelectionStart
                    && scanner.getCurrentTokenEndPosition() >= mSelectionEnd) {
                // found the token, but only keep if the right type
                if (token == ITerminalSymbols.TokenNameStringLiteral) {
                    mTokenString = new String(scanner.getCurrentTokenSource());
                }
                break;
            } else if (scanner.getCurrentTokenStartPosition() > mSelectionEnd) {
                // scanner is past the selection, abort.
                break;
            }
        }
    } catch (JavaModelException e1) {
        // Error in unit.getBuffer. Ignore.
    } catch (InvalidInputException e2) {
        // Error in scanner.getNextToken. Ignore.
    } finally {
        monitor.worked(1);
    }

    if (mTokenString != null) {
        // As a literal string, the token should have surrounding quotes. Remove them.
        // Note: unquoteAttrValue technically removes either " or ' paired quotes, whereas
        // the Java token should only have " quotes. Since we know the type to be a string
        // literal, there should be no confusion here.
        mTokenString = unquoteAttrValue(mTokenString);

        // We need a non-empty string literal
        if (mTokenString.length() == 0) {
            mTokenString = null;
        }
    }

    if (mTokenString == null) {
        status.addFatalError("Please select a Java string literal.");
    }

    monitor.worked(1);
    return status.isOK();
}

From source file:com.android.ide.eclipse.adt.refactorings.extractstring.ExtractStringRefactoring.java

License:Open Source License

/**
 * Try to find the selected Java element in the compilation unit.
 * //from   w w w.  ja  v  a 2  s.  co  m
 * If selection matches a string literal, capture it, otherwise add a fatal error
 * to the status.
 * 
 * On success, advance the monitor by 3.
 */
private boolean findSelectionInJavaUnit(ICompilationUnit unit, RefactoringStatus status,
        IProgressMonitor monitor) {
    try {
        IBuffer buffer = unit.getBuffer();

        IScanner scanner = ToolFactory.createScanner(false, //tokenizeComments
                false, //tokenizeWhiteSpace
                false, //assertMode
                false //recordLineSeparator
        );
        scanner.setSource(buffer.getCharacters());
        monitor.worked(1);

        for (int token = scanner.getNextToken(); token != ITerminalSymbols.TokenNameEOF; token = scanner
                .getNextToken()) {
            if (scanner.getCurrentTokenStartPosition() <= mSelectionStart
                    && scanner.getCurrentTokenEndPosition() >= mSelectionEnd) {
                // found the token, but only keep of the right type
                if (token == ITerminalSymbols.TokenNameStringLiteral) {
                    mTokenString = new String(scanner.getCurrentTokenSource());
                }
                break;
            } else if (scanner.getCurrentTokenStartPosition() > mSelectionEnd) {
                // scanner is past the selection, abort.
                break;
            }
        }
    } catch (JavaModelException e1) {
        // Error in unit.getBuffer. Ignore.
    } catch (InvalidInputException e2) {
        // Error in scanner.getNextToken. Ignore.
    } finally {
        monitor.worked(1);
    }

    if (mTokenString != null) {
        // As a literal string, the token should have surrounding quotes. Remove them.
        int len = mTokenString.length();
        if (len > 0 && mTokenString.charAt(0) == '"' && mTokenString.charAt(len - 1) == '"') {
            mTokenString = mTokenString.substring(1, len - 1);
        }
        // We need a non-empty string literal
        if (mTokenString.length() == 0) {
            mTokenString = null;
        }
    }

    if (mTokenString == null) {
        status.addFatalError("Please select a Java string literal.");
    }

    monitor.worked(1);
    return status.isOK();
}

From source file:com.cb.eclipse.folding.java.calculation.JavaProjectionCalculator.java

License:Open Source License

/**
 * The core logic of this class./*  w w  w.j  a  va 2  s .c o m*/
 * 
 * Computes using the strategy objects what regions are available as
 * projections for this particular element.
 * 
 * @param elem
 * @return @throws
 *         JavaModelException
 * @throws InvalidInputException
 */
private Set computeProjections(IJavaElement elem) throws JavaModelException, InvalidInputException {
    if (!(elem instanceof ISourceReference))
        return null;

    RegionCalculationStrategy strategy = StrategyFactory.instance(this, elem);
    strategy.initialize();
    Set regionSet;
    if (!strategy.shouldScan(elem)) {
        regionSet = strategy.result(); // call immediately...
    } else {
        ISourceReference reference = (ISourceReference) elem;
        ISourceRange range = reference.getSourceRange();

        // All other element types require some parsing...
        String contents = reference.getSource();

        if (contents == null)
            return null;

        IScanner scanner = ToolFactory.createScanner(true, false, false, false);
        scanner.setSource(contents.toCharArray());

        int shift = range.getOffset();
        int start = shift;

        while (true) {
            int token = scanner.getNextToken();

            start = shift + scanner.getCurrentTokenStartPosition();
            int end = shift + scanner.getCurrentTokenEndPosition() + 1;

            if (!strategy.keepProcessing(token) || token == ITerminalSymbols.TokenNameEOF) {
                break; // end case.
            }

            strategy.handle(token, start, end, elem);

        }

        strategy.postScan(start, elem);

        regionSet = strategy.result();

    }

    if (regionSet == null) {
        regionSet = Collections.EMPTY_SET;
    }

    strategy.dispose();

    return regionSet;

}

From source file:com.codenvy.ide.ext.java.server.internal.core.Member.java

License:Open Source License

public ISourceRange getJavadocRange() throws JavaModelException {
    ISourceRange range = getSourceRange();
    if (range == null)
        return null;
    IBuffer buf = null;/*from  ww  w  .ja  v a 2 s. c o  m*/
    if (isBinary()) {
        buf = getClassFile().getBuffer();
    } else {
        ICompilationUnit compilationUnit = getCompilationUnit();
        if (!compilationUnit.isConsistent()) {
            return null;
        }
        buf = compilationUnit.getBuffer();
    }
    final int start = range.getOffset();
    final int length = range.getLength();
    if (length > 0 && buf.getChar(start) == '/') {
        IScanner scanner = ToolFactory.createScanner(true, false, false, false);
        try {
            scanner.setSource(buf.getText(start, length).toCharArray());
            int docOffset = -1;
            int docEnd = -1;

            int terminal = scanner.getNextToken();
            loop: while (true) {
                switch (terminal) {
                case ITerminalSymbols.TokenNameCOMMENT_JAVADOC:
                    docOffset = scanner.getCurrentTokenStartPosition();
                    docEnd = scanner.getCurrentTokenEndPosition() + 1;
                    terminal = scanner.getNextToken();
                    break;
                case ITerminalSymbols.TokenNameCOMMENT_LINE:
                case ITerminalSymbols.TokenNameCOMMENT_BLOCK:
                    terminal = scanner.getNextToken();
                    continue loop;
                default:
                    break loop;
                }
            }
            if (docOffset != -1) {
                return new SourceRange(docOffset + start, docEnd - docOffset);
            }
        } catch (InvalidInputException ex) {
            // try if there is inherited Javadoc
        } catch (IndexOutOfBoundsException e) {
            // https://bugs.eclipse.org/bugs/show_bug.cgi?id=305001
        }
    }
    return null;
}

From source file:com.github.elucash.lambda4jdt.FoldingStructureProvider.java

License:Open Source License

/**
 * Computes the projection ranges for a given <code>ISourceReference</code>. More than one range
 * or none at all may be returned. If there are no foldable regions, an empty array is returned.
 * <p>/*  www. j av a2  s .co m*/
 * The last region in the returned array (if not empty) describes the region for the java element
 * that implements the source reference. Any preceding regions describe javadoc comments of that
 * java element.
 * </p>
 * @param reference a java element that is a source reference
 * @param ctx the folding context
 * @return the regions to be folded
 */
protected final IRegion[] computeProjectionRanges(ISourceReference reference,
        FoldingStructureComputationContext ctx) {
    try {
        ISourceRange range = reference.getSourceRange();
        if (!SourceRange.isAvailable(range))
            return new IRegion[0];

        String contents = reference.getSource();
        if (contents == null)
            return new IRegion[0];

        List<IRegion> regions = new ArrayList<IRegion>();
        if (!ctx.hasFirstType() && reference instanceof IType) {
            ctx.setFirstType((IType) reference);
            IRegion headerComment = computeHeaderComment(ctx);
            if (headerComment != null) {
                regions.add(headerComment);
                ctx.setHasHeaderComment();
            }
        }

        final int shift = range.getOffset();
        IScanner scanner = ctx.getScanner();
        scanner.resetTo(shift, shift + range.getLength());

        int start = shift;
        while (true) {

            int token = scanner.getNextToken();
            start = scanner.getCurrentTokenStartPosition();

            switch (token) {
            case ITerminalSymbols.TokenNameCOMMENT_JAVADOC:
            case ITerminalSymbols.TokenNameCOMMENT_BLOCK: {
                int end = scanner.getCurrentTokenEndPosition() + 1;
                regions.add(new Region(start, end - start));
                continue;
            }
            case ITerminalSymbols.TokenNameCOMMENT_LINE:
                continue;
            }

            break;
        }

        regions.add(new Region(start, shift + range.getLength() - start));

        return regions.toArray(new IRegion[regions.size()]);
    } catch (JavaModelException e) {
    } catch (InvalidInputException e) {
    }

    return new IRegion[0];
}

From source file:com.github.elucash.lambda4jdt.FoldingStructureProvider.java

License:Open Source License

private IRegion computeHeaderComment(FoldingStructureComputationContext ctx) throws JavaModelException {
    // search at most up to the first type
    ISourceRange range = ctx.getFirstType().getSourceRange();
    if (range == null)
        return null;
    int start = 0;
    int end = range.getOffset();

    /* code adapted from CommentFormattingStrategy:
     * scan the header content up to the first type. Once a comment is
     * found, accumulate any additional comments up to the stop condition.
     * The stop condition is reaching a package declaration, import container,
     * or the end of the input.//www . ja  v a  2 s. c  o m
     */
    IScanner scanner = ctx.getScanner();
    scanner.resetTo(start, end);

    int headerStart = -1;
    int headerEnd = -1;
    try {
        boolean foundComment = false;
        int terminal = scanner.getNextToken();
        while (terminal != ITerminalSymbols.TokenNameEOF && !(terminal == ITerminalSymbols.TokenNameclass
                || terminal == ITerminalSymbols.TokenNameinterface || terminal == ITerminalSymbols.TokenNameenum
                || (foundComment && (terminal == ITerminalSymbols.TokenNameimport
                        || terminal == ITerminalSymbols.TokenNamepackage)))) {

            if (terminal == ITerminalSymbols.TokenNameCOMMENT_JAVADOC
                    || terminal == ITerminalSymbols.TokenNameCOMMENT_BLOCK
                    || terminal == ITerminalSymbols.TokenNameCOMMENT_LINE) {
                if (!foundComment)
                    headerStart = scanner.getCurrentTokenStartPosition();
                headerEnd = scanner.getCurrentTokenEndPosition();
                foundComment = true;
            }
            terminal = scanner.getNextToken();
        }

    } catch (InvalidInputException ex) {
        return null;
    }

    if (headerEnd != -1) {
        return new Region(headerStart, headerEnd - headerStart);
    }
    return null;
}

From source file:com.google.googlejavaformat.java.JavaInput.java

License:Apache License

/**
 * Lex the input and build the list of toks.
 *
 * @param text the text to be lexed.//from   w ww .ja  va 2 s  .c  o  m
 * @param stopIds a set of Eclipse token names which should cause lexing to stop. If one of these
 *     is found, the returned list will include tokens up to but not including that token.
 */
ImmutableList<Tok> buildToks(String text, ImmutableSet<Integer> stopIds)
        throws InvalidInputException, FormatterException {
    stopIds = ImmutableSet.<Integer>builder().addAll(stopIds).add(ITerminalSymbols.TokenNameEOF).build();
    kN = 0;
    IScanner scanner = ToolFactory.createScanner(true, true, true, "1.8");
    scanner.setSource(text.toCharArray());
    int textLength = text.length();
    List<Tok> toks = new ArrayList<>();
    int charI = 0;
    int columnI = 0;
    while (scanner.getCurrentTokenEndPosition() < textLength - 1 && !stopIds.contains(scanner.getNextToken())) {
        int charI0 = scanner.getCurrentTokenStartPosition();
        // Get string, possibly with Unicode escapes.
        String originalTokText = text.substring(charI0, scanner.getCurrentTokenEndPosition() + 1);
        String tokText = new String(scanner.getCurrentTokenSource()); // Unicode escapes removed.
        char tokText0 = tokText.charAt(0); // The token's first character.
        final boolean isToken; // Is this tok a token?
        final boolean isNumbered; // Is this tok numbered? (tokens and comments)
        boolean extraNewline = false; // Extra newline at end?
        List<String> strings = new ArrayList<>();
        if (Character.isWhitespace(tokText0)) {
            isToken = false;
            isNumbered = false;
            boolean first = true;
            for (String spaces : NEWLINE_SPLITTER.split(originalTokText)) {
                if (!first) {
                    strings.add("\n");
                }
                if (!spaces.isEmpty()) {
                    strings.add(spaces);
                }
                first = false;
            }
        } else if (tokText.startsWith("'") || tokText.startsWith("\"")) {
            isToken = true;
            isNumbered = true;
            strings.add(originalTokText);
        } else if (tokText.startsWith("//") || tokText.startsWith("/*")) {
            // For compatibility with an earlier lexer, the newline after a // comment is its own tok.
            if (tokText.startsWith("//") && originalTokText.endsWith("\n")) {
                originalTokText = originalTokText.substring(0, originalTokText.length() - 1);
                tokText = tokText.substring(0, tokText.length() - 1);
                extraNewline = true;
            }
            isToken = false;
            isNumbered = true;
            strings.add(originalTokText);
        } else if (Character.isJavaIdentifierStart(tokText0) || Character.isDigit(tokText0)
                || tokText0 == '.' && tokText.length() > 1 && Character.isDigit(tokText.charAt(1))) {
            // Identifier, keyword, or numeric literal (a dot may begin a number, as in .2D).
            isToken = true;
            isNumbered = true;
            strings.add(tokText);
        } else {
            // Other tokens ("+" or "++" or ">>" are broken into one-character toks, because ">>"
            // cannot be lexed without syntactic knowledge. This implementation fails if the token
            // contains Unicode escapes.
            isToken = true;
            isNumbered = true;
            for (char c : tokText.toCharArray()) {
                strings.add(String.valueOf(c));
            }
        }
        if (strings.size() == 1) {
            toks.add(new Tok(isNumbered ? kN++ : -1, originalTokText, tokText, charI, columnI, isToken));
            for (char c : originalTokText.toCharArray()) {
                if (c == '\n') {
                    columnI = 0;
                } else {
                    ++columnI;
                }
                ++charI;
            }
        } else {
            if (strings.size() != 1 && !tokText.equals(originalTokText)) {
                throw new FormatterException(
                        "Unicode escapes not allowed in whitespace or multi-character operators");
            }
            for (String str : strings) {
                toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken));
                for (char c : str.toCharArray()) {
                    if (c == '\n') {
                        columnI = 0;
                    } else {
                        ++columnI;
                    }
                    ++charI;
                }
            }
        }
        if (extraNewline) {
            toks.add(new Tok(-1, "\n", "\n", charI, columnI, false));
            columnI = 0;
            ++charI;
        }
    }
    toks.add(new Tok(kN++, "", "", charI, columnI, true)); // EOF tok.
    --kN; // Don't count EOF tok.
    computeRanges(toks);
    return ImmutableList.copyOf(toks);
}

From source file:com.ibm.research.tagging.java.extractor.WaypointDefinitionExtractor.java

License:Open Source License

/**
 * Gets the comment regions associated with this document, restricted to the given offset and length, if no comment regions 
 * are found an empty array is returned//from  w  w w .jav  a  2s .  c o  m
 * @param document
 * @param offset
 * @param length
 * @param returnSingleLine Wither to collect single line commments
 * @return Array of tag regions
 */
public static IRegion[] getCommentRegions(IDocument document, int offset, int length) {
    try {
        StringBuffer content = new StringBuffer(document.get(offset, length));
        IRegion range = new Region(offset, length);
        List<IRegion> commentRegions = new ArrayList<IRegion>();
        IScanner scanner = ToolFactory.createScanner(true, false, false, false);
        scanner.setSource(content.toString().toCharArray());

        int shift = range.getOffset();

        while (true) {
            int terminal = scanner.getNextToken();

            if (terminal == ITerminalSymbols.TokenNameCOMMENT_JAVADOC
                    || terminal == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
                int commentOffset = shift + scanner.getCurrentTokenStartPosition();
                int commentEnd = shift + scanner.getCurrentTokenEndPosition() + 1;
                commentRegions.add(new Region(commentOffset, commentEnd - commentOffset));
            } else if (terminal == ITerminalSymbols.TokenNameCOMMENT_LINE) {
                int commentOffset = shift + scanner.getCurrentTokenStartPosition();
                int commentEnd = shift + scanner.getCurrentTokenEndPosition() + 1;
                commentRegions.add(new Region(commentOffset, commentEnd - commentOffset));
            } else if (terminal == ITerminalSymbols.TokenNameEOF)
                break;
        }

        IRegion[] result = new IRegion[commentRegions.size()];
        commentRegions.toArray(result);
        return result;

    } catch (BadLocationException e) {
        e.printStackTrace();
    } catch (InvalidInputException e) {
        e.printStackTrace();
    }

    return new IRegion[0];
}

From source file:comparison.SJavaTokenComparator.java

License:Open Source License

/**
 * Creates a token comparator for the given string.
 *
 * @param text the text to be tokenized//from  w ww .ja v  a 2s.  c o  m
 * @param textTokenComparatorFactory a factory to create text token comparators
 */
public SJavaTokenComparator(String text) {

    fText = text;

    int length = fText.length();
    fStarts = new int[length];
    fLengths = new int[length];
    fTokens = new String[length];
    fCount = 0;

    IScanner scanner = ToolFactory.createScanner(true, false, false, false); // returns comments & whitespace
    scanner.setSource(fText.toCharArray());
    int endPos = 0;
    try {
        int tokenType;
        while ((tokenType = scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
            int start = scanner.getCurrentTokenStartPosition();
            int end = scanner.getCurrentTokenEndPosition() + 1;
            // Comments are treated as a single token (see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=78063)
            if (!TokenScanner.isComment(tokenType)) {
                recordTokenRange(start, end - start);
            }
            endPos = end;
        }
    } catch (InvalidInputException ex) {
        // We couldn't parse part of the input. Fall through and make the rest a single token
    }
    // Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=13907
    if (endPos < length) {
        recordTokenRange(endPos, length - endPos);
    }
}