Java tutorial
// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.util; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.Lists; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import java.util.ArrayList; /** * Provides memory-efficient bidirectional mapping String <-> unique integer. * Uses byte-wise compressed prefix trie internally. * <p> * Class allows index retrieval for the given string, addition of the new * index and string retrieval for the given index. It also allows efficient * serialization of the internal data structures. * <p> * Internally class stores list of nodes with each node containing byte[] * representation of compressed trie node: * <pre> * varint32 parentIndex; // index of the parent node * varint32 keylen; // length of the node key * byte[keylen] key; // node key data * repeated jumpEntry { // Zero or more jump entries, referencing child nodes * byte key // jump key (first byte of the child node key) * varint32 nodeIndex // child index * } * <p> * Note that jumpEntry key byte is actually duplicated in the child node * instance. This is done to improve performance of the index->string * lookup (so we can avoid jump table parsing during this lookup). * <p> * Root node of the trie must have parent id pointing to itself. * <p> * TODO(bazel-team): (2010) Consider more fine-tuned locking mechanism - e.g. * distinguishing between read and write locks. */ @ThreadSafe public class CompactStringIndexer extends AbstractIndexer { private static final int NOT_FOUND = -1; private ArrayList<byte[]> nodes; // Compressed prefix trie nodes. private int rootId; // Root node id. /* * Creates indexer instance. */ public CompactStringIndexer(int expectedCapacity) { Preconditions.checkArgument(expectedCapacity > 0); nodes = Lists.newArrayListWithExpectedSize(expectedCapacity); rootId = NOT_FOUND; } /** * Allocates new node index. Must be called only from * synchronized methods. */ private int allocateIndex() { nodes.add(null); return nodes.size() - 1; } /** * Replaces given node record with the new one. Must be called only from * synchronized methods. * <p> * Subclasses can override this method to be notified when an update actually * takes place. */ @ThreadCompatible protected void updateNode(int index, byte[] content) { nodes.set(index, content); } /** * Returns parent id for the given node content. * * @return parent node id */ private int getParentId(byte[] content) { int[] intHolder = new int[1]; VarInt.getVarInt(content, 0, intHolder); return intHolder[0]; } /** * Creates new node using specified key suffix. Must be called from * synchronized methods. * * @param parentNode parent node id * @param key original key that is being added to the indexer * @param offset node key offset in the original key. * * @return new node id corresponding to the given key */ private int createNode(int parentNode, byte[] key, int offset) { int index = allocateIndex(); int len = key.length - offset; Preconditions.checkState(len >= 0); // Content consists of parent id, key length and key. There are no jump records. byte[] content = new byte[VarInt.varIntSize(parentNode) + VarInt.varIntSize(len) + len]; // Add parent id. int contentOffset = VarInt.putVarInt(parentNode, content, 0); // Add node key length. contentOffset = VarInt.putVarInt(len, content, contentOffset); // Add node key content. System.arraycopy(key, offset, content, contentOffset, len); updateNode(index, content); return index; } /** * Updates jump entry index in the given node. * * @param node node id to update * @param oldIndex old jump entry index * @param newIndex updated jump entry index */ private void updateJumpEntry(int node, int oldIndex, int newIndex) { byte[] content = nodes.get(node); int[] intHolder = new int[1]; int offset = VarInt.getVarInt(content, 0, intHolder); // parent id offset = VarInt.getVarInt(content, offset, intHolder); // key length offset += intHolder[0]; // Offset now points to the first jump entry. while (offset < content.length) { int next = VarInt.getVarInt(content, offset + 1, intHolder); // jump index if (intHolder[0] == oldIndex) { // Substitute oldIndex value with newIndex. byte[] newContent = new byte[content.length + VarInt.varIntSize(newIndex) - VarInt.varIntSize(oldIndex)]; System.arraycopy(content, 0, newContent, 0, offset + 1); offset = VarInt.putVarInt(newIndex, newContent, offset + 1); System.arraycopy(content, next, newContent, offset, content.length - next); updateNode(node, newContent); return; } else { offset = next; } } StringBuilder builder = new StringBuilder().append("Index ").append(oldIndex) .append(" is not present in the node ").append(node).append(", "); dumpNodeContent(builder, content); throw new IllegalArgumentException(builder.toString()); } /** * Creates new branch node content at the predefined location, splitting * prefix from the given node and optionally adding another child node * jump entry. * * @param originalNode node that will be split * @param newBranchNode new branch node id * @param splitOffset offset at which to split original node key * @param indexKey optional additional jump key * @param childIndex optional additional jump index. Optional jump entry will * be skipped if this index is set to NOT_FOUND. */ private void createNewBranchNode(int originalNode, int newBranchNode, int splitOffset, byte indexKey, int childIndex) { byte[] content = nodes.get(originalNode); int[] intHolder = new int[1]; int keyOffset = VarInt.getVarInt(content, 0, intHolder); // parent id // If original node is a root node, new branch node will become new root. So set parent id // appropriately (for root node it is set to the node's own id). int parentIndex = (originalNode == intHolder[0] ? newBranchNode : intHolder[0]); keyOffset = VarInt.getVarInt(content, keyOffset, intHolder); // key length Preconditions.checkState(intHolder[0] >= splitOffset); // Calculate new content size. int newSize = VarInt.varIntSize(parentIndex) + VarInt.varIntSize(splitOffset) + splitOffset + 1 + VarInt.varIntSize(originalNode) + (childIndex != NOT_FOUND ? 1 + VarInt.varIntSize(childIndex) : 0); // New content consists of parent id, new key length, truncated key and two jump records. byte[] newContent = new byte[newSize]; // Add parent id. int contentOffset = VarInt.putVarInt(parentIndex, newContent, 0); // Add adjusted key length. contentOffset = VarInt.putVarInt(splitOffset, newContent, contentOffset); // Add truncated key content and first jump key. System.arraycopy(content, keyOffset, newContent, contentOffset, splitOffset + 1); // Add index for the first jump key. contentOffset = VarInt.putVarInt(originalNode, newContent, contentOffset + splitOffset + 1); // If requested, add additional jump entry. if (childIndex != NOT_FOUND) { // Add second jump key. newContent[contentOffset] = indexKey; // Add index for the second jump key. VarInt.putVarInt(childIndex, newContent, contentOffset + 1); } updateNode(newBranchNode, newContent); } /** * Inject newly created branch node into the trie data structure. Method * will update parent node jump entry to point to the new branch node (or * will update root id if branch node becomes new root) and will truncate * key prefix from the original node that was split (that prefix now * resides in the branch node). * * @param originalNode node that will be split * @param newBranchNode new branch node id * @param commonPrefixLength how many bytes should be split into the new branch node. */ private void injectNewBranchNode(int originalNode, int newBranchNode, int commonPrefixLength) { byte[] content = nodes.get(originalNode); int parentId = getParentId(content); if (originalNode == parentId) { rootId = newBranchNode; // update root index } else { updateJumpEntry(parentId, originalNode, newBranchNode); } // Truncate prefix from the original node and set its parent to the our new branch node. int[] intHolder = new int[1]; int suffixOffset = VarInt.getVarInt(content, 0, intHolder); // parent id suffixOffset = VarInt.getVarInt(content, suffixOffset, intHolder); // key length int len = intHolder[0] - commonPrefixLength; Preconditions.checkState(len >= 0); suffixOffset += commonPrefixLength; // New content consists of parent id, new key length and duplicated key suffix. byte[] newContent = new byte[VarInt.varIntSize(newBranchNode) + VarInt.varIntSize(len) + (content.length - suffixOffset)]; // Add parent id. int contentOffset = VarInt.putVarInt(newBranchNode, newContent, 0); // Add new key length. contentOffset = VarInt.putVarInt(len, newContent, contentOffset); // Add key and jump table. System.arraycopy(content, suffixOffset, newContent, contentOffset, content.length - suffixOffset); updateNode(originalNode, newContent); } /** * Adds new child node (that uses specified key suffix) to the given * current node. * Example: * <pre> * Had "ab". Adding "abcd". * * 1:"ab",'c'->2 * 1:"ab" -> \ * 2:"cd" * </pre> */ private int addChildNode(int parentNode, byte[] key, int keyOffset) { int child = createNode(parentNode, key, keyOffset); byte[] content = nodes.get(parentNode); // Add jump table entry to the parent node. int entryOffset = content.length; // New content consists of original content and additional jump record. byte[] newContent = new byte[entryOffset + 1 + VarInt.varIntSize(child)]; // Copy original content. System.arraycopy(content, 0, newContent, 0, entryOffset); // Add jump key. newContent[entryOffset] = key[keyOffset]; // Add jump index. VarInt.putVarInt(child, newContent, entryOffset + 1); updateNode(parentNode, newContent); return child; } /** * Splits node into two at the specified offset. * Example: * <pre> * Had "abcd". Adding "ab". * * 2:"ab",'c'->1 * 1:"abcd" -> \ * 1:"cd" * </pre> */ private int splitNodeSuffix(int nodeToSplit, int commonPrefixLength) { int newBranchNode = allocateIndex(); // Create new node with truncated key. createNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength, (byte) 0, NOT_FOUND); injectNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength); return newBranchNode; } /** * Splits node into two at the specified offset and adds another leaf. * Example: * <pre> * Had "abcd". Adding "abef". * * 3:"ab",'c'->1,'e'->2 * 1:"abcd" -> / \ * 1:"cd" 2:"ef" * </pre> */ private int addBranch(int nodeToSplit, byte[] key, int offset, int commonPrefixLength) { int newBranchNode = allocateIndex(); int child = createNode(newBranchNode, key, offset + commonPrefixLength); // Create new node with the truncated key and reference to the new child node. createNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength, key[offset + commonPrefixLength], child); injectNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength); return child; } private int findOrCreateIndexInternal(int node, byte[] key, int offset, boolean createIfNotFound) { byte[] content = nodes.get(node); int[] intHolder = new int[1]; int contentOffset = VarInt.getVarInt(content, 0, intHolder); // parent id contentOffset = VarInt.getVarInt(content, contentOffset, intHolder); // key length int skyKeyLen = intHolder[0]; int remainingKeyLen = key.length - offset; int minKeyLen = Math.min(skyKeyLen, remainingKeyLen); // Compare given key/offset content with the node key. Skip first key byte for recursive // calls - this byte is equal to the byte in the jump entry and was already compared. for (int i = (offset > 0 ? 1 : 0); i < minKeyLen; i++) { // compare key if (key[offset + i] != content[contentOffset + i]) { // Mismatch found somewhere in the middle of the node key. If requested, node // should be split and another leaf added for the new key. return createIfNotFound ? addBranch(node, key, offset, i) : NOT_FOUND; } } if (remainingKeyLen > minKeyLen) { // Node key matched portion of the key - find appropriate jump entry. If found - recursion. // If not - mismatch (we will add new child node if requested). contentOffset += skyKeyLen; while (contentOffset < content.length) { if (key[offset + skyKeyLen] == content[contentOffset]) { // compare index value VarInt.getVarInt(content, contentOffset + 1, intHolder); // Found matching jump entry - recursively compare the child. return findOrCreateIndexInternal(intHolder[0], key, offset + skyKeyLen, createIfNotFound); } else { // Jump entry key does not match. Skip rest of the entry data. contentOffset = VarInt.getVarInt(content, contentOffset + 1, intHolder); } } // There are no matching jump entries - report mismatch or create a new leaf if necessary. return createIfNotFound ? addChildNode(node, key, offset + skyKeyLen) : NOT_FOUND; } else if (skyKeyLen > minKeyLen) { // Key suffix is a subset of the node key. Report mismatch or split the node if requested). return createIfNotFound ? splitNodeSuffix(node, minKeyLen) : NOT_FOUND; } else { // Node key exactly matches key suffix - return associated index value. return node; } } private synchronized int findOrCreateIndex(byte[] key, boolean createIfNotFound) { if (rootId == NOT_FOUND) { // Root node does not seem to exist - create it if needed. if (createIfNotFound) { rootId = createNode(0, key, 0); Preconditions.checkState(rootId == 0); return 0; } else { return NOT_FOUND; } } return findOrCreateIndexInternal(rootId, key, 0, createIfNotFound); } private byte[] reconstructKeyInternal(int node, int suffixSize) { byte[] content = nodes.get(node); Preconditions.checkNotNull(content); int[] intHolder = new int[1]; int contentOffset = VarInt.getVarInt(content, 0, intHolder); // parent id int parentNode = intHolder[0]; contentOffset = VarInt.getVarInt(content, contentOffset, intHolder); // key length int len = intHolder[0]; byte[] key; if (node != parentNode) { // We haven't reached root node yet. Make a recursive call, adjusting suffix length. key = reconstructKeyInternal(parentNode, suffixSize + len); } else { // We are in a root node. Finally allocate array for the key. It will be filled up // on our way back from recursive call tree. key = new byte[suffixSize + len]; } // Fill appropriate portion of the full key with the node key content. System.arraycopy(content, contentOffset, key, key.length - suffixSize - len, len); return key; } private byte[] reconstructKey(int node) { return reconstructKeyInternal(node, 0); } /* (non-Javadoc) * @see com.google.devtools.build.lib.util.StringIndexer#clear() */ @Override public synchronized void clear() { nodes.clear(); } /* (non-Javadoc) * @see com.google.devtools.build.lib.util.StringIndexer#size() */ @Override public synchronized int size() { return nodes.size(); } protected int getOrCreateIndexForBytes(byte[] bytes) { return findOrCreateIndex(bytes, true); } protected synchronized boolean addBytes(byte[] bytes) { int count = nodes.size(); int index = getOrCreateIndexForBytes(bytes); return index >= count; } protected int getIndexForBytes(byte[] bytes) { return findOrCreateIndex(bytes, false); } /* (non-Javadoc) * @see com.google.devtools.build.lib.util.StringIndexer#getOrCreateIndex(java.lang.String) */ @Override public int getOrCreateIndex(String s) { return getOrCreateIndexForBytes(string2bytes(s)); } /* (non-Javadoc) * @see com.google.devtools.build.lib.util.StringIndexer#getIndex(java.lang.String) */ @Override public int getIndex(String s) { return getIndexForBytes(string2bytes(s)); } /* (non-Javadoc) * @see com.google.devtools.build.lib.util.StringIndexer#addString(java.lang.String) */ @Override public boolean addString(String s) { return addBytes(string2bytes(s)); } protected synchronized byte[] getBytesForIndex(int i) { Preconditions.checkArgument(i >= 0); if (i >= nodes.size()) { return null; } return reconstructKey(i); } /* (non-Javadoc) * @see com.google.devtools.build.lib.util.StringIndexer#getStringForIndex(int) */ @Override public String getStringForIndex(int i) { byte[] bytes = getBytesForIndex(i); return bytes != null ? bytes2string(bytes) : null; } private void dumpNodeContent(StringBuilder builder, byte[] content) { int[] intHolder = new int[1]; int offset = VarInt.getVarInt(content, 0, intHolder); builder.append("parent: ").append(intHolder[0]); offset = VarInt.getVarInt(content, offset, intHolder); int len = intHolder[0]; builder.append(", len: ").append(len).append(", key: \"").append(new String(content, offset, len, UTF_8)) .append('"'); offset += len; while (offset < content.length) { builder.append(", '").append(new String(content, offset, 1, UTF_8)).append("': "); offset = VarInt.getVarInt(content, offset + 1, intHolder); builder.append(intHolder[0]); } builder.append(", size: ").append(content.length); } private int dumpContent(StringBuilder builder, int node, int indent, boolean[] seen) { for (int i = 0; i < indent; i++) { builder.append(" "); } builder.append(node).append(": "); if (node >= nodes.size()) { builder.append("OUT_OF_BOUNDS\n"); return 0; } else if (seen[node]) { builder.append("ALREADY_SEEN\n"); return 0; } seen[node] = true; byte[] content = nodes.get(node); if (content == null) { builder.append("NULL\n"); return 0; } dumpNodeContent(builder, content); builder.append("\n"); int contentSize = content.length; int[] intHolder = new int[1]; int contentOffset = VarInt.getVarInt(content, 0, intHolder); // parent id contentOffset = VarInt.getVarInt(content, contentOffset, intHolder); // key length contentOffset += intHolder[0]; while (contentOffset < content.length) { contentOffset = VarInt.getVarInt(content, contentOffset + 1, intHolder); contentSize += dumpContent(builder, intHolder[0], indent + 1, seen); } return contentSize; } @Override public synchronized String toString() { StringBuilder builder = new StringBuilder(); builder.append("size = ").append(nodes.size()).append("\n"); if (!nodes.isEmpty()) { int contentSize = dumpContent(builder, rootId, 0, new boolean[nodes.size()]); builder.append("contentSize = ").append(contentSize).append("\n"); } return builder.toString(); } }