Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs.protocol; import java.util.Arrays; import java.util.EnumSet; import java.util.LinkedList; import java.util.List; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.BlockStoragePolicySpi; import org.apache.hadoop.fs.StorageType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * A block storage policy describes how to select the storage types * for the replicas of a block. */ @InterfaceAudience.Private public class BlockStoragePolicy implements BlockStoragePolicySpi { public static final Logger LOG = LoggerFactory.getLogger(BlockStoragePolicy.class); /** A 4-bit policy ID */ private final byte id; /** Policy name */ private final String name; /** The storage types to store the replicas of a new block. */ private final StorageType[] storageTypes; /** The fallback storage type for block creation. */ private final StorageType[] creationFallbacks; /** The fallback storage type for replication. */ private final StorageType[] replicationFallbacks; /** * Whether the policy is inherited during file creation. * If set then the policy cannot be changed after file creation. */ private boolean copyOnCreateFile; @VisibleForTesting public BlockStoragePolicy(byte id, String name, StorageType[] storageTypes, StorageType[] creationFallbacks, StorageType[] replicationFallbacks) { this(id, name, storageTypes, creationFallbacks, replicationFallbacks, false); } @VisibleForTesting public BlockStoragePolicy(byte id, String name, StorageType[] storageTypes, StorageType[] creationFallbacks, StorageType[] replicationFallbacks, boolean copyOnCreateFile) { this.id = id; this.name = name; this.storageTypes = storageTypes; this.creationFallbacks = creationFallbacks; this.replicationFallbacks = replicationFallbacks; this.copyOnCreateFile = copyOnCreateFile; } /** * @return a list of {@link StorageType}s for storing the replicas of a block. */ public List<StorageType> chooseStorageTypes(final short replication) { final List<StorageType> types = new LinkedList<>(); int i = 0, j = 0; // Do not return transient storage types. We will not have accurate // usage information for transient types. for (; i < replication && j < storageTypes.length; ++j) { if (!storageTypes[j].isTransient()) { types.add(storageTypes[j]); ++i; } } final StorageType last = storageTypes[storageTypes.length - 1]; if (!last.isTransient()) { for (; i < replication; i++) { types.add(last); } } return types; } /** * Choose the storage types for storing the remaining replicas, given the * replication number and the storage types of the chosen replicas. * * @param replication the replication number. * @param chosen the storage types of the chosen replicas. * @return a list of {@link StorageType}s for storing the replicas of a block. */ public List<StorageType> chooseStorageTypes(final short replication, final Iterable<StorageType> chosen) { return chooseStorageTypes(replication, chosen, null); } private List<StorageType> chooseStorageTypes(final short replication, final Iterable<StorageType> chosen, final List<StorageType> excess) { final List<StorageType> types = chooseStorageTypes(replication); diff(types, chosen, excess); return types; } /** * Choose the storage types for storing the remaining replicas, given the * replication number, the storage types of the chosen replicas and * the unavailable storage types. It uses fallback storage in case that * the desired storage type is unavailable. * * @param replication the replication number. * @param chosen the storage types of the chosen replicas. * @param unavailables the unavailable storage types. * @param isNewBlock Is it for new block creation? * @return a list of {@link StorageType}s for storing the replicas of a block. */ public List<StorageType> chooseStorageTypes(final short replication, final Iterable<StorageType> chosen, final EnumSet<StorageType> unavailables, final boolean isNewBlock) { final List<StorageType> excess = new LinkedList<>(); final List<StorageType> storageTypes = chooseStorageTypes(replication, chosen, excess); final int expectedSize = storageTypes.size() - excess.size(); final List<StorageType> removed = new LinkedList<>(); for (int i = storageTypes.size() - 1; i >= 0; i--) { // replace/remove unavailable storage types. final StorageType t = storageTypes.get(i); if (unavailables.contains(t)) { final StorageType fallback = isNewBlock ? getCreationFallback(unavailables) : getReplicationFallback(unavailables); if (fallback == null) { removed.add(storageTypes.remove(i)); } else { storageTypes.set(i, fallback); } } } // remove excess storage types after fallback replacement. diff(storageTypes, excess, null); if (storageTypes.size() < expectedSize) { LOG.warn( "Failed to place enough replicas: expected size is {}" + " but only {} storage types can be selected (replication={}," + " selected={}, unavailable={}" + ", removed={}" + ", policy={}" + ")", expectedSize, storageTypes.size(), replication, storageTypes, unavailables, removed, this); } return storageTypes; } /** * Compute the difference between two lists t and c so that after the diff * computation we have: t = t - c; * Further, if e is not null, set e = e + c - t; */ private static void diff(List<StorageType> t, Iterable<StorageType> c, List<StorageType> e) { for (StorageType storagetype : c) { final int i = t.indexOf(storagetype); if (i >= 0) { t.remove(i); } else if (e != null) { e.add(storagetype); } } } /** * Choose excess storage types for deletion, given the * replication number and the storage types of the chosen replicas. * * @param replication the replication number. * @param chosen the storage types of the chosen replicas. * @return a list of {@link StorageType}s for deletion. */ public List<StorageType> chooseExcess(final short replication, final Iterable<StorageType> chosen) { final List<StorageType> types = chooseStorageTypes(replication); final List<StorageType> excess = new LinkedList<>(); diff(types, chosen, excess); return excess; } /** @return the fallback {@link StorageType} for creation. */ public StorageType getCreationFallback(EnumSet<StorageType> unavailables) { return getFallback(unavailables, creationFallbacks); } /** @return the fallback {@link StorageType} for replication. */ public StorageType getReplicationFallback(EnumSet<StorageType> unavailables) { return getFallback(unavailables, replicationFallbacks); } @Override public int hashCode() { return Byte.valueOf(id).hashCode(); } @Override public boolean equals(Object obj) { if (obj == this) { return true; } else if (obj == null || !(obj instanceof BlockStoragePolicy)) { return false; } final BlockStoragePolicy that = (BlockStoragePolicy) obj; return this.id == that.id; } @Override public String toString() { return getClass().getSimpleName() + "{" + name + ":" + id + ", storageTypes=" + Arrays.asList(storageTypes) + ", creationFallbacks=" + Arrays.asList(creationFallbacks) + ", replicationFallbacks=" + Arrays.asList(replicationFallbacks) + "}"; } public byte getId() { return id; } @Override public String getName() { return name; } @Override public StorageType[] getStorageTypes() { return this.storageTypes; } @Override public StorageType[] getCreationFallbacks() { return this.creationFallbacks; } @Override public StorageType[] getReplicationFallbacks() { return this.replicationFallbacks; } private static StorageType getFallback(EnumSet<StorageType> unavailables, StorageType[] fallbacks) { for (StorageType fb : fallbacks) { if (!unavailables.contains(fb)) { return fb; } } return null; } @Override public boolean isCopyOnCreateFile() { return copyOnCreateFile; } }