List of usage examples for org.eclipse.jdt.core ToolFactory createScanner
@SuppressWarnings("javadoc") public static IScanner createScanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean recordLineSeparator, String sourceLevel)
From source file:ca.uvic.chisel.diver.sequencediagrams.sc.java.model.NodeFinder.java
License:Open Source License
/** * A visitor that maps a selection to a given ASTNode. The result node is * determined as follows:// w w w . j av a2 s . c o m * <ul> * <li>first the visitor tries to find a node that is covered by <code>start</code> and * <code>length</code> where either <code>start</code> and <code>length</code> exactly * matches the node or where the text covered before and after the node only consists * of white spaces or comments.</li> * <li>if no such node exists than the node that encloses the range defined by * start and end is returned.</li> * <li>if the length is zero than also nodes are considered where the node's * start or end position matches <code>start</code>.</li> * <li>otherwise <code>null</code> is returned.</li> * </ul> * * @param root the root node from which the search starts * @param start the start offset * @param length the length * @param source the source of the compilation unit * * @return the result node * @throws JavaModelException if an error occurs in the Java model * * @since 3.0 */ public static ASTNode perform(ASTNode root, int start, int length, ITypeRoot source) throws JavaModelException { NodeFinder finder = new NodeFinder(start, length); root.accept(finder); ASTNode result = finder.getCoveredNode(); if (result == null) return null; Selection selection = Selection.createFromStartLength(start, length); if (selection.covers(result)) { IBuffer buffer = source.getBuffer(); if (buffer != null) { IScanner scanner = ToolFactory.createScanner(false, false, false, false); scanner.setSource(buffer.getText(start, length).toCharArray()); try { int token = scanner.getNextToken(); if (token != ITerminalSymbols.TokenNameEOF) { int tStart = scanner.getCurrentTokenStartPosition(); if (tStart == result.getStartPosition() - start) { scanner.resetTo(tStart + result.getLength(), length - 1); token = scanner.getNextToken(); if (token == ITerminalSymbols.TokenNameEOF) return result; } } } catch (InvalidInputException e) { } } } return finder.getCoveringNode(); }
From source file:ca.uvic.chisel.javasketch.internal.ast.NodeFinder.java
License:Open Source License
/** * Maps a selection to a given ASTNode, where the selection is given by a start and a length. * The result node is determined as follows: * <ul>/*from ww w. ja va 2s. c om*/ * <li>first the visitor tries to find a node that is covered by <code>start</code> and * <code>length</code> where either <code>start</code> and <code>length</code> exactly * matches the node or where the text covered before and after the node only consists * of white spaces or comments.</li> * <li>if no such node exists then the node that encloses the range defined by * <code>start</code> and <code>length</code> is returned.</li> * <li>if the length is zero then also nodes are considered where the node's * start or end position matches <code>start</code>.</li> * <li>otherwise <code>null</code> is returned.</li> * </ul> * * @param root the root node from which the search starts * @param start the given start * @param length the given length * @param source the source of the compilation unit * * @return the result node * @throws JavaModelException if an error occurs in the Java model */ public static ASTNode perform(ASTNode root, int start, int length, ITypeRoot source) throws JavaModelException { NodeFinder finder = new NodeFinder(root, start, length); ASTNode result = finder.getCoveredNode(); if (result == null) return null; int nodeStart = result.getStartPosition(); if (start <= nodeStart && ((nodeStart + result.getLength()) <= (start + length))) { IBuffer buffer = source.getBuffer(); if (buffer != null) { IScanner scanner = ToolFactory.createScanner(false, false, false, false); try { scanner.setSource(buffer.getText(start, length).toCharArray()); int token = scanner.getNextToken(); if (token != ITerminalSymbols.TokenNameEOF) { int tStart = scanner.getCurrentTokenStartPosition(); if (tStart == result.getStartPosition() - start) { scanner.resetTo(tStart + result.getLength(), length - 1); token = scanner.getNextToken(); if (token == ITerminalSymbols.TokenNameEOF) return result; } } } catch (InvalidInputException e) { // ignore } catch (IndexOutOfBoundsException e) { // https://bugs.eclipse.org/bugs/show_bug.cgi?id=305001 return null; } } } return finder.getCoveringNode(); }
From source file:ca.uvic.cs.tagsea.extraction.TagExtractor.java
License:Open Source License
/** * Gets the comment regions associated with this document, restricted to the given offset and length, if no comment regions * are found an empty array is returned//w w w .ja v a 2 s . c o m * @param document * @param offset * @param length * @param returnSingleLine Wither to collect single line commments * @return Array of tag regions */ public static IRegion[] getCommentRegions(IDocument document, int offset, int length) { try { StringBuffer content = new StringBuffer(document.get(offset, length)); IRegion range = new Region(offset, length); List<IRegion> commentRegions = new ArrayList<IRegion>(); IScanner scanner = ToolFactory.createScanner(true, false, false, false); scanner.setSource(content.toString().toCharArray()); int shift = range.getOffset(); while (true) { int terminal = scanner.getNextToken(); if (terminal == ITerminalSymbols.TokenNameCOMMENT_JAVADOC || terminal == ITerminalSymbols.TokenNameCOMMENT_BLOCK) { int commentOffset = shift + scanner.getCurrentTokenStartPosition(); int commentEnd = shift + scanner.getCurrentTokenEndPosition() + 1; commentRegions.add(new Region(commentOffset, commentEnd - commentOffset)); } else if (terminal == ITerminalSymbols.TokenNameCOMMENT_LINE) { int commentOffset = shift + scanner.getCurrentTokenStartPosition(); int commentEnd = shift + scanner.getCurrentTokenEndPosition() + 1; commentRegions.add(new Region(commentOffset, commentEnd - commentOffset)); } else if (terminal == ITerminalSymbols.TokenNameEOF) break; } IRegion[] result = new IRegion[commentRegions.size()]; commentRegions.toArray(result); return result; } catch (BadLocationException e) { TagSEAPlugin.log("", e); } catch (InvalidInputException e) { TagSEAPlugin.log("", e); } return new IRegion[0]; }
From source file:com.android.ide.eclipse.adt.internal.refactorings.extractstring.ExtractStringRefactoring.java
License:Open Source License
/** * Try to find the selected Java element in the compilation unit. * * If selection matches a string literal, capture it, otherwise add a fatal error * to the status.//from w w w . j a v a 2 s .co m * * On success, advance the monitor by 3. * Returns status.isOK(). */ private boolean findSelectionInJavaUnit(ICompilationUnit unit, RefactoringStatus status, IProgressMonitor monitor) { try { IBuffer buffer = unit.getBuffer(); IScanner scanner = ToolFactory.createScanner(false, //tokenizeComments false, //tokenizeWhiteSpace false, //assertMode false //recordLineSeparator ); scanner.setSource(buffer.getCharacters()); monitor.worked(1); for (int token = scanner.getNextToken(); token != ITerminalSymbols.TokenNameEOF; token = scanner .getNextToken()) { if (scanner.getCurrentTokenStartPosition() <= mSelectionStart && scanner.getCurrentTokenEndPosition() >= mSelectionEnd) { // found the token, but only keep if the right type if (token == ITerminalSymbols.TokenNameStringLiteral) { mTokenString = new String(scanner.getCurrentTokenSource()); } break; } else if (scanner.getCurrentTokenStartPosition() > mSelectionEnd) { // scanner is past the selection, abort. break; } } } catch (JavaModelException e1) { // Error in unit.getBuffer. Ignore. } catch (InvalidInputException e2) { // Error in scanner.getNextToken. Ignore. } finally { monitor.worked(1); } if (mTokenString != null) { // As a literal string, the token should have surrounding quotes. Remove them. // Note: unquoteAttrValue technically removes either " or ' paired quotes, whereas // the Java token should only have " quotes. Since we know the type to be a string // literal, there should be no confusion here. mTokenString = unquoteAttrValue(mTokenString); // We need a non-empty string literal if (mTokenString.length() == 0) { mTokenString = null; } } if (mTokenString == null) { status.addFatalError("Please select a Java string literal."); } monitor.worked(1); return status.isOK(); }
From source file:com.android.ide.eclipse.adt.refactorings.extractstring.ExtractStringRefactoring.java
License:Open Source License
/** * Try to find the selected Java element in the compilation unit. * /*from w w w .j a va2 s . c o m*/ * If selection matches a string literal, capture it, otherwise add a fatal error * to the status. * * On success, advance the monitor by 3. */ private boolean findSelectionInJavaUnit(ICompilationUnit unit, RefactoringStatus status, IProgressMonitor monitor) { try { IBuffer buffer = unit.getBuffer(); IScanner scanner = ToolFactory.createScanner(false, //tokenizeComments false, //tokenizeWhiteSpace false, //assertMode false //recordLineSeparator ); scanner.setSource(buffer.getCharacters()); monitor.worked(1); for (int token = scanner.getNextToken(); token != ITerminalSymbols.TokenNameEOF; token = scanner .getNextToken()) { if (scanner.getCurrentTokenStartPosition() <= mSelectionStart && scanner.getCurrentTokenEndPosition() >= mSelectionEnd) { // found the token, but only keep of the right type if (token == ITerminalSymbols.TokenNameStringLiteral) { mTokenString = new String(scanner.getCurrentTokenSource()); } break; } else if (scanner.getCurrentTokenStartPosition() > mSelectionEnd) { // scanner is past the selection, abort. break; } } } catch (JavaModelException e1) { // Error in unit.getBuffer. Ignore. } catch (InvalidInputException e2) { // Error in scanner.getNextToken. Ignore. } finally { monitor.worked(1); } if (mTokenString != null) { // As a literal string, the token should have surrounding quotes. Remove them. int len = mTokenString.length(); if (len > 0 && mTokenString.charAt(0) == '"' && mTokenString.charAt(len - 1) == '"') { mTokenString = mTokenString.substring(1, len - 1); } // We need a non-empty string literal if (mTokenString.length() == 0) { mTokenString = null; } } if (mTokenString == null) { status.addFatalError("Please select a Java string literal."); } monitor.worked(1); return status.isOK(); }
From source file:com.cb.eclipse.folding.java.calculation.JavaProjectionCalculator.java
License:Open Source License
/** * The core logic of this class.//from w w w . j a va 2 s . c om * * Computes using the strategy objects what regions are available as * projections for this particular element. * * @param elem * @return @throws * JavaModelException * @throws InvalidInputException */ private Set computeProjections(IJavaElement elem) throws JavaModelException, InvalidInputException { if (!(elem instanceof ISourceReference)) return null; RegionCalculationStrategy strategy = StrategyFactory.instance(this, elem); strategy.initialize(); Set regionSet; if (!strategy.shouldScan(elem)) { regionSet = strategy.result(); // call immediately... } else { ISourceReference reference = (ISourceReference) elem; ISourceRange range = reference.getSourceRange(); // All other element types require some parsing... String contents = reference.getSource(); if (contents == null) return null; IScanner scanner = ToolFactory.createScanner(true, false, false, false); scanner.setSource(contents.toCharArray()); int shift = range.getOffset(); int start = shift; while (true) { int token = scanner.getNextToken(); start = shift + scanner.getCurrentTokenStartPosition(); int end = shift + scanner.getCurrentTokenEndPosition() + 1; if (!strategy.keepProcessing(token) || token == ITerminalSymbols.TokenNameEOF) { break; // end case. } strategy.handle(token, start, end, elem); } strategy.postScan(start, elem); regionSet = strategy.result(); } if (regionSet == null) { regionSet = Collections.EMPTY_SET; } strategy.dispose(); return regionSet; }
From source file:com.codenvy.ide.ext.java.server.internal.core.Member.java
License:Open Source License
public ISourceRange getJavadocRange() throws JavaModelException { ISourceRange range = getSourceRange(); if (range == null) return null; IBuffer buf = null;/*from w w w . ja v a 2 s . co m*/ if (isBinary()) { buf = getClassFile().getBuffer(); } else { ICompilationUnit compilationUnit = getCompilationUnit(); if (!compilationUnit.isConsistent()) { return null; } buf = compilationUnit.getBuffer(); } final int start = range.getOffset(); final int length = range.getLength(); if (length > 0 && buf.getChar(start) == '/') { IScanner scanner = ToolFactory.createScanner(true, false, false, false); try { scanner.setSource(buf.getText(start, length).toCharArray()); int docOffset = -1; int docEnd = -1; int terminal = scanner.getNextToken(); loop: while (true) { switch (terminal) { case ITerminalSymbols.TokenNameCOMMENT_JAVADOC: docOffset = scanner.getCurrentTokenStartPosition(); docEnd = scanner.getCurrentTokenEndPosition() + 1; terminal = scanner.getNextToken(); break; case ITerminalSymbols.TokenNameCOMMENT_LINE: case ITerminalSymbols.TokenNameCOMMENT_BLOCK: terminal = scanner.getNextToken(); continue loop; default: break loop; } } if (docOffset != -1) { return new SourceRange(docOffset + start, docEnd - docOffset); } } catch (InvalidInputException ex) { // try if there is inherited Javadoc } catch (IndexOutOfBoundsException e) { // https://bugs.eclipse.org/bugs/show_bug.cgi?id=305001 } } return null; }
From source file:com.google.gdt.eclipse.core.TypeCreator.java
License:Open Source License
private boolean isValidComment(String template) { IScanner scanner = ToolFactory.createScanner(true, false, false, false); scanner.setSource(template.toCharArray()); try {//from w w w . jav a2 s. c o m int next = scanner.getNextToken(); while (TokenScanner.isComment(next)) { next = scanner.getNextToken(); } return next == ITerminalSymbols.TokenNameEOF; } catch (InvalidInputException e) { // If there are lexical errors, the comment is invalid } return false; }
From source file:com.google.googlejavaformat.java.JavaInput.java
License:Apache License
/** * Lex the input and build the list of toks. * * @param text the text to be lexed.// w ww .ja va 2s .co m * @param stopIds a set of Eclipse token names which should cause lexing to stop. If one of these * is found, the returned list will include tokens up to but not including that token. */ ImmutableList<Tok> buildToks(String text, ImmutableSet<Integer> stopIds) throws InvalidInputException, FormatterException { stopIds = ImmutableSet.<Integer>builder().addAll(stopIds).add(ITerminalSymbols.TokenNameEOF).build(); kN = 0; IScanner scanner = ToolFactory.createScanner(true, true, true, "1.8"); scanner.setSource(text.toCharArray()); int textLength = text.length(); List<Tok> toks = new ArrayList<>(); int charI = 0; int columnI = 0; while (scanner.getCurrentTokenEndPosition() < textLength - 1 && !stopIds.contains(scanner.getNextToken())) { int charI0 = scanner.getCurrentTokenStartPosition(); // Get string, possibly with Unicode escapes. String originalTokText = text.substring(charI0, scanner.getCurrentTokenEndPosition() + 1); String tokText = new String(scanner.getCurrentTokenSource()); // Unicode escapes removed. char tokText0 = tokText.charAt(0); // The token's first character. final boolean isToken; // Is this tok a token? final boolean isNumbered; // Is this tok numbered? (tokens and comments) boolean extraNewline = false; // Extra newline at end? List<String> strings = new ArrayList<>(); if (Character.isWhitespace(tokText0)) { isToken = false; isNumbered = false; boolean first = true; for (String spaces : NEWLINE_SPLITTER.split(originalTokText)) { if (!first) { strings.add("\n"); } if (!spaces.isEmpty()) { strings.add(spaces); } first = false; } } else if (tokText.startsWith("'") || tokText.startsWith("\"")) { isToken = true; isNumbered = true; strings.add(originalTokText); } else if (tokText.startsWith("//") || tokText.startsWith("/*")) { // For compatibility with an earlier lexer, the newline after a // comment is its own tok. if (tokText.startsWith("//") && originalTokText.endsWith("\n")) { originalTokText = originalTokText.substring(0, originalTokText.length() - 1); tokText = tokText.substring(0, tokText.length() - 1); extraNewline = true; } isToken = false; isNumbered = true; strings.add(originalTokText); } else if (Character.isJavaIdentifierStart(tokText0) || Character.isDigit(tokText0) || tokText0 == '.' && tokText.length() > 1 && Character.isDigit(tokText.charAt(1))) { // Identifier, keyword, or numeric literal (a dot may begin a number, as in .2D). isToken = true; isNumbered = true; strings.add(tokText); } else { // Other tokens ("+" or "++" or ">>" are broken into one-character toks, because ">>" // cannot be lexed without syntactic knowledge. This implementation fails if the token // contains Unicode escapes. isToken = true; isNumbered = true; for (char c : tokText.toCharArray()) { strings.add(String.valueOf(c)); } } if (strings.size() == 1) { toks.add(new Tok(isNumbered ? kN++ : -1, originalTokText, tokText, charI, columnI, isToken)); for (char c : originalTokText.toCharArray()) { if (c == '\n') { columnI = 0; } else { ++columnI; } ++charI; } } else { if (strings.size() != 1 && !tokText.equals(originalTokText)) { throw new FormatterException( "Unicode escapes not allowed in whitespace or multi-character operators"); } for (String str : strings) { toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken)); for (char c : str.toCharArray()) { if (c == '\n') { columnI = 0; } else { ++columnI; } ++charI; } } } if (extraNewline) { toks.add(new Tok(-1, "\n", "\n", charI, columnI, false)); columnI = 0; ++charI; } } toks.add(new Tok(kN++, "", "", charI, columnI, true)); // EOF tok. --kN; // Don't count EOF tok. computeRanges(toks); return ImmutableList.copyOf(toks); }
From source file:com.ibm.research.tagging.java.extractor.WaypointDefinitionExtractor.java
License:Open Source License
/** * Gets the comment regions associated with this document, restricted to the given offset and length, if no comment regions * are found an empty array is returned//w w w. j a v a 2s . c o m * @param document * @param offset * @param length * @param returnSingleLine Wither to collect single line commments * @return Array of tag regions */ public static IRegion[] getCommentRegions(IDocument document, int offset, int length) { try { StringBuffer content = new StringBuffer(document.get(offset, length)); IRegion range = new Region(offset, length); List<IRegion> commentRegions = new ArrayList<IRegion>(); IScanner scanner = ToolFactory.createScanner(true, false, false, false); scanner.setSource(content.toString().toCharArray()); int shift = range.getOffset(); while (true) { int terminal = scanner.getNextToken(); if (terminal == ITerminalSymbols.TokenNameCOMMENT_JAVADOC || terminal == ITerminalSymbols.TokenNameCOMMENT_BLOCK) { int commentOffset = shift + scanner.getCurrentTokenStartPosition(); int commentEnd = shift + scanner.getCurrentTokenEndPosition() + 1; commentRegions.add(new Region(commentOffset, commentEnd - commentOffset)); } else if (terminal == ITerminalSymbols.TokenNameCOMMENT_LINE) { int commentOffset = shift + scanner.getCurrentTokenStartPosition(); int commentEnd = shift + scanner.getCurrentTokenEndPosition() + 1; commentRegions.add(new Region(commentOffset, commentEnd - commentOffset)); } else if (terminal == ITerminalSymbols.TokenNameEOF) break; } IRegion[] result = new IRegion[commentRegions.size()]; commentRegions.toArray(result); return result; } catch (BadLocationException e) { e.printStackTrace(); } catch (InvalidInputException e) { e.printStackTrace(); } return new IRegion[0]; }