TokenListList.java :  » IDE-Netbeans » lexer » org » netbeans » lib » lexer » Java Open Source

Java Open Source » IDE Netbeans » lexer 
lexer » org » netbeans » lib » lexer » TokenListList.java
/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */

package org.netbeans.lib.lexer;

import java.util.List;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.lib.editor.util.ArrayUtilities;
import org.netbeans.lib.editor.util.GapList;
import static org.netbeans.lib.lexer.LexerUtilsConstants.INVALID_STATE;

/**
 * List of token lists that collects all token lists for a given language path.
 * <br/>
 * There can be both lists with/without joining of the embedded sections.
 * <br/>
 * Initial implementation attempted to initialize the list of token lists lazily
 * upon asking for it by client. However there was a problem with fixing
 * of token list explorers' state when the list is partially initialized
 * and there is an update of the token hierarchy. Sometimes there were inconsistencies
 * that a particular token list appeared twice in the token list list.
 * <br/>
 * Current impl is non-lazy so once the list becomes created it gets fully initialized
 * by traversing the parent token lists's tokens for embeddings of the particular language.
 * <br/>
 * Advantages:
 * <ul>
 *   <li> Easier updating - no issues with incomplete exploration.
 *   <li> More errorsafe approach with joinSections - if any of the scanned lists is joinSections
 *        then the whole token list list becomes joinSections from the begining.
 *   <li> It's disputable how much time the lazy impl has been saving.
 *   <li> More deterministic behavior - helps to diagnose errors.
 * </ul>
 * 
 * <p>
 * GapList is used for faster updates and there can be either single top-level
 * non-EmbeddedTokenList token list or zero or more nested EmbeddedTokenList(s).
 * </p>
 * 
 * <p>
 * joinSections approach:
 * <br/>
 * Non-joining embedded token lists' contents will be lexed without token list list assistance.
 * <br/>
 * Joining embedded TLs will need TLL assistance so TLL instance gets created for them.
 * <br/>
 * If there are mixed joining/non-joining language embedding instances for the same
 * language path then the non-joining ones can possibly become initialized (lexed)
 * without TLL if they are asked individually. Later when first joining embedding is found
 * the token list list will be created and contain both joining and non-joining
 * embeddings but the joinSections will be respected for individual lexing.
 * Non-joining sections will be lexed individually and the join sections will be lexed as joined.
 * </p>
 *
 * @author Miloslav Metelka
 */

public final class TokenListList extends GapList<EmbeddedTokenList<?>> {

    private final TokenHierarchyOperation operation;

    private final LanguagePath languagePath;

    /**
     * Whether this token list is holding joint sections embeddings.
     * <br/>
     * If so it is mandatorily maintained.
     */
    private boolean joinSections;
    
    /**
     * Total count of children. It's maintained to quickly resolve
     * whether the list may be released.
     */
    private int childrenCount;


    public TokenListList(TokenHierarchyOperation<?,?> operation, LanguagePath languagePath) {
        super(4);
        this.operation = operation;
        this.languagePath = languagePath;

        // languagePath has size >= 2
        assert (languagePath.size() >= 2);
        Language<?> language = languagePath.innerLanguage();
        if (languagePath.size() > 2) {
            Object relexState = null;
            TokenListList parentTokenList = operation.tokenListList(languagePath.parent());
            for (int parentIndex = 0; parentIndex < parentTokenList.size(); parentIndex++) {
                TokenList<?> tokenList = parentTokenList.get(parentIndex);
                relexState = scanTokenList(tokenList, language, relexState);
            }
        } else { // Parent is root token list
            scanTokenList(operation.rootTokenList(), language, null);
        }
    }
    
    private Object scanTokenList(TokenList<?> tokenList, Language<?> language, Object relexState) {
        int tokenCount = tokenList.tokenCount();
        for (int i = 0; i < tokenCount; i++) {
            EmbeddedTokenList<?> etl = EmbeddingContainer.embeddedTokenList(
                tokenList, i, language);
            if (etl != null) {
                add(etl);
                if (etl.embedding().joinSections()) {
                    joinSections = true;
                    if (!etl.isInited()) {
                        etl.init(relexState);
                        relexState = LexerUtilsConstants.endState(etl, relexState);
                    }
                } else { // Not joining sections -> next section starts with null state
                    relexState = null;
                }
            }
        }
        return relexState;
    }
    
    public LanguagePath languagePath() {
        return languagePath;
    }
    
    /**
     * Return true if this list is mandatorily updated because there is
     * one or more embeddings that join sections.
     */
    public boolean joinSections() {
        return joinSections;
    }
    
    public void setJoinSections(boolean joinSections) {
        this.joinSections = joinSections;
    }
    
    public void increaseChildrenCount() {
        childrenCount++;
    }
    
    public void decreaseChildrenCount() {
        childrenCount--;
    }
    
    public boolean hasChildren() {
        return (childrenCount > 0);
    }
    
    public Object relexState(int index) {
        // Find the previous non-empty section or non-joining section
        Object relexState = INVALID_STATE;
        for (int i = index - 1; i >= 0 && relexState == INVALID_STATE; i--) {
            relexState = LexerUtilsConstants.endState(get(i));
        }
        if (relexState == INVALID_STATE) // Start from real begining
            relexState = null;
        return relexState;
    }

    /**
     * Return a valid token list or null if the index is too high.
     */
    public EmbeddedTokenList<?> getOrNull(int index) {
        return (index < size()) ? get(index) : null;
    }
    
    private static final EmbeddedTokenList<?>[] EMPTY_TOKEN_LIST_ARRAY = new EmbeddedTokenList<?>[0];

    public EmbeddedTokenList<?>[] replace(int index, int removeCount, List<? extends TokenList<?>> addTokenLists) {
        EmbeddedTokenList<?>[] removed;
        if (removeCount > 0) {
            removed = new EmbeddedTokenList<?>[removeCount];
            copyElements(index, index + removeCount, removed, 0);
            remove(index, removeCount);
        } else {
            removed = EMPTY_TOKEN_LIST_ARRAY;
        }
        @SuppressWarnings("unchecked")
        List<EmbeddedTokenList<?>> etlLists = (List<EmbeddedTokenList<?>>)addTokenLists;
        addAll(index, etlLists);
        return removed;
    }

    void childAdded() {
        throw new UnsupportedOperationException("Not yet implemented");
    }

    TokenHierarchyOperation operation() {
        return operation;
    }

    int modCount() {
        return operation.modCount();
    }

    public int findIndex(int offset) {
        int high = size() - 1;
        int low = 0;
        while (low <= high) {
            int mid = (low + high) >> 1;
            EmbeddedTokenList<?> etl = get(mid);
            // Ensure that the startOffset() will be updated
            etl.embeddingContainer().updateStatus();
            int cmp = etl.startOffset() - offset;
            if (cmp < 0)
                low = mid + 1;
            else if (cmp > 0)
                high = mid - 1;
            else { // cmp == 0 -> take the previous one
                low = mid;
                break;
            }
        }
        return low;
    }
    
    /**
     * Find an index during update of the token list.
     * <br>
     * If there was a removal performed and some of the contained token lists
     * were removed then these TLs then the token lists beyond the modification point
     * will be forced to update itself which may 
     */
    public int findIndexDuringUpdate(EmbeddedTokenList targetEtl, int modOffset, int removedLength) {
        int high = size() - 1;
        int low = 0;
        int targetStartOffset = targetEtl.startOffset();
        if (targetStartOffset > modOffset && targetEtl.embeddingContainer().isRemoved()) {
            targetStartOffset = Math.max(targetStartOffset - removedLength, modOffset);
        }
        while (low <= high) {
            int mid = (low + high) >> 1;
            EmbeddedTokenList<?> etl = get(mid);
            // Ensure that the startOffset() will be updated
            etl.embeddingContainer().updateStatusImpl();
            int startOffset = etl.startOffset();
            if (startOffset > modOffset && etl.embeddingContainer().isRemoved()) {
                startOffset = Math.max(startOffset - removedLength, modOffset);
            }
            int cmp = startOffset - targetStartOffset;
            if (cmp < 0)
                low = mid + 1;
            else if (cmp > 0)
                high = mid - 1;
            else {
                low = mid;
                // Now it's also possible that there was a larger remove when multiple token lists
                // inside the removed area were removed and they all have startOffset being modOffset.
                // In such case these need to be searched by linear search in both directions
                // from the found one.
                if (etl != targetEtl) {
                    low--;
                    while (low >= 0) {
                        etl = get(low);
                        if (etl == targetEtl) { // Quick check for match
                            return low;
                        }
                        // Check whether this was appropriate attempt for match
                        etl.embeddingContainer().updateStatusImpl();
                        startOffset = etl.startOffset();
                        if (startOffset > modOffset && etl.embeddingContainer().isRemoved()) {
                            startOffset = Math.max(startOffset - removedLength, modOffset);
                        }
                        if (startOffset != modOffset)
                            break;
                        low--;
                    }
                    
                    // Go up from mid
                    low = mid + 1;
                    while (low < size()) {
                        etl = get(low);
                        if (etl == targetEtl) { // Quick check for match
                            return low;
                        }
                        // Check whether this was appropriate attempt for match
                        etl.embeddingContainer().updateStatusImpl();
                        startOffset = etl.startOffset();
                        if (startOffset > modOffset && etl.embeddingContainer().isRemoved()) {
                            startOffset = Math.max(startOffset - removedLength, modOffset);
                        }
                        if (startOffset != modOffset)
                            break;
                        low++;
                    }
                }
                break;
            }
        }
        return low;
    }
    
    public String checkConsistency() {
        // Check whether the token lists are in a right order
        int lastEndOffset = 0;
        for (int i = 0; i < size(); i++) {
            EmbeddedTokenList<?> etl = get(i);
            etl.embeddingContainer().updateStatusImpl();
            if (etl.startOffset() < lastEndOffset) {
                return "TOKEN-LIST-LIST Invalid start offset at index=" + i +
                        ": etl[" + i + "].startOffset()=" + etl.startOffset() +
                        " < lastEndOffset=" + lastEndOffset +
                        "\n" + this;
            }
            if (etl.startOffset() > etl.endOffset()) {
                return "TOKEN-LIST-LIST Invalid end offset at index=" + i +
                        ": etl[" + i + "].startOffset()=" + etl.startOffset() +
                        " > etl[" + i + "].endOffset()="+ etl.endOffset() +
                        "\n" + this;
            }
            if (etl.embeddingContainer() == null) {
                return "TOKEN-LIST-LIST Null ec at index=" + i + "\n" + this;
            }
            if (etl.embeddingContainer().isRemoved()) {
                return "TOKEN-LIST-LIST Removed ec fat index=" + i + "\n" + this;
            }
            lastEndOffset = etl.endOffset();
        }
        return null;
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder(2048);
        sb.append("TokenListList for ");
        sb.append(languagePath().mimePath());
        if (joinSections()) {
            sb.append(", joinSections");
        }
        if (hasChildren()) {
            sb.append(", hasChildren");
        }
        sb.append('\n');
        int digitCount = ArrayUtilities.digitCount(size());
        for (int i = 0; i < size(); i++) {
            EmbeddedTokenList<?> etl = get(i);
            ArrayUtilities.appendBracketedIndex(sb, i, digitCount);
            etl.embeddingContainer().updateStatus();
            sb.append(etl.toStringHeader());
            EmbeddingContainer ec = etl.embeddingContainer();
            if (ec != null && ec.isRemoved()) {
                sb.append(", <--REMOVED-->");
            }
            sb.append('\n');
            LexerUtilsConstants.appendTokenListIndented(sb, etl, 4);
        }
        return sb.toString();
    }

}
java2s.com  | Contact Us | Privacy Policy
Copyright 2009 - 12 Demo Source and Support. All rights reserved.
All other trademarks are property of their respective owners.