com.google.errorprone.util.ErrorProneTokens.java Source code

Introduction

Here is the source code for com.google.errorprone.util.ErrorProneTokens.java

Source

/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.errorprone.util;

import static com.google.common.base.Preconditions.checkArgument;

import com.google.common.collect.ImmutableList;
import com.sun.tools.javac.parser.JavaTokenizer;
import com.sun.tools.javac.parser.Scanner;
import com.sun.tools.javac.parser.ScannerFactory;
import com.sun.tools.javac.parser.Tokens.Comment;
import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
import com.sun.tools.javac.parser.Tokens.TokenKind;
import com.sun.tools.javac.parser.UnicodeReader;
import com.sun.tools.javac.util.Context;

/** A utility for tokenizing and preserving comments. */
public class ErrorProneTokens {

    /** Returns the tokens for the given source text, including comments. */
    public static ImmutableList<ErrorProneToken> getTokens(String source, Context context) {
        if (source == null) {
            return ImmutableList.of();
        }
        ScannerFactory fac = ScannerFactory.instance(context);
        char[] buffer = source.toCharArray();
        Scanner scanner = new AccessibleScanner(fac, new CommentSavingTokenizer(fac, buffer, buffer.length));
        ImmutableList.Builder<ErrorProneToken> tokens = ImmutableList.builder();
        do {
            scanner.nextToken();
            tokens.add(new ErrorProneToken(scanner.token()));
        } while (scanner.token().kind != TokenKind.EOF);
        return tokens.build();
    }

    /** A {@link JavaTokenizer} that saves comments. */
    static class CommentSavingTokenizer extends JavaTokenizer {
        CommentSavingTokenizer(ScannerFactory fac, char[] buffer, int length) {
            super(fac, buffer, length);
        }

        @Override
        protected Comment processComment(int pos, int endPos, CommentStyle style) {
            char[] buf = reader.getRawCharacters(pos, endPos);
            return new CommentWithTextAndPosition(pos, endPos, new AccessibleReader(fac, buf, buf.length), style);
        }
    }

    /** A {@link Comment} that saves its text and start position. */
    static class CommentWithTextAndPosition implements Comment {

        private final int pos;
        private final int endPos;
        private final AccessibleReader reader;
        private final CommentStyle style;

        private String text = null;

        public CommentWithTextAndPosition(int pos, int endPos, AccessibleReader reader, CommentStyle style) {
            this.pos = pos;
            this.endPos = endPos;
            this.reader = reader;
            this.style = style;
        }

        /**
         * Returns the source position of the character at index {@code index} in the comment text.
         *
         * <p>The handling of javadoc comments in javac has more logic to skip over leading
         * whitespace and '*' characters when indexing into doc comments, but we don't need any
         * of that.
         */
        @Override
        public int getSourcePos(int index) {
            checkArgument(0 <= index && index < (endPos - pos), "Expected %s in the range [0, %s)", index,
                    endPos - pos);
            return pos + index;
        }

        @Override
        public CommentStyle getStyle() {
            return style;
        }

        @Override
        public String getText() {
            String text = this.text;
            if (text == null) {
                this.text = text = new String(reader.getRawCharacters());
            }
            return text;
        }

        /**
         * We don't care about {@code @deprecated} javadoc tags (see the DepAnn check).
         *
         * @return false
         */
        @Override
        public boolean isDeprecated() {
            return false;
        }

        @Override
        public String toString() {
            return String.format("Comment: '%s'", getText());
        }
    }

    // Scanner(ScannerFactory, JavaTokenizer) is package-private
    static class AccessibleScanner extends Scanner {
        protected AccessibleScanner(ScannerFactory fac, JavaTokenizer tokenizer) {
            super(fac, tokenizer);
        }
    }

    // UnicodeReader(ScannerFactory, char[], int) is package-private
    static class AccessibleReader extends UnicodeReader {
        protected AccessibleReader(ScannerFactory fac, char[] buffer, int length) {
            super(fac, buffer, length);
        }
    }
}