Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs.server.namenode; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.ReconfigurableBase; import org.apache.hadoop.conf.ReconfigurationException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; import org.apache.hadoop.ha.HAServiceStatus; import org.apache.hadoop.ha.HealthCheckFailedException; import org.apache.hadoop.ha.ServiceFailedException; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.StoragePolicySatisfierMode; import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState; import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby; import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; import org.apache.hadoop.hdfs.server.namenode.ha.HAState; import org.apache.hadoop.hdfs.server.namenode.ha.StandbyState; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgressMetrics; import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.JournalProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.ipc.ExternalCall; import org.apache.hadoop.ipc.RefreshCallQueueProtocol; import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol; import org.apache.hadoop.tools.GetUserMappingsProtocol; import org.apache.hadoop.tracing.TraceAdminProtocol; import org.apache.hadoop.tracing.TraceUtils; import org.apache.hadoop.tracing.TracerConfigurationManager; import org.apache.hadoop.util.ExitUtil.ExitException; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ServicePlugin; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.htrace.core.Tracer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.management.ObjectName; import java.io.IOException; import java.io.PrintStream; import java.net.InetSocketAddress; import java.net.URI; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.TreeSet; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_SERVICE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_EDITS_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_BIND_HOST_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_BIND_HOST_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_RPC_BIND_HOST_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_PLUGINS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.FS_PROTECTED_DIRECTORIES; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY; import static org.apache.hadoop.util.ExitUtil.terminate; import static org.apache.hadoop.util.ToolRunner.confirmPrompt; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_NAMESPACE; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT; /********************************************************** * NameNode serves as both directory namespace manager and * "inode table" for the Hadoop DFS. There is a single NameNode * running in any DFS deployment. (Well, except when there * is a second backup/failover NameNode, or when using federated NameNodes.) * * The NameNode controls two critical tables: * 1) filename->blocksequence (namespace) * 2) block->machinelist ("inodes") * * The first table is stored on disk and is very precious. * The second table is rebuilt every time the NameNode comes up. * * 'NameNode' refers to both this class as well as the 'NameNode server'. * The 'FSNamesystem' class actually performs most of the filesystem * management. The majority of the 'NameNode' class itself is concerned * with exposing the IPC interface and the HTTP server to the outside world, * plus some configuration management. * * NameNode implements the * {@link org.apache.hadoop.hdfs.protocol.ClientProtocol} interface, which * allows clients to ask for DFS services. * {@link org.apache.hadoop.hdfs.protocol.ClientProtocol} is not designed for * direct use by authors of DFS client code. End-users should instead use the * {@link org.apache.hadoop.fs.FileSystem} class. * * NameNode also implements the * {@link org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol} interface, * used by DataNodes that actually store DFS data blocks. These * methods are invoked repeatedly and automatically by all the * DataNodes in a DFS deployment. * * NameNode also implements the * {@link org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol} interface, * used by secondary namenodes or rebalancing processes to get partial * NameNode state, for example partial blocksMap etc. **********************************************************/ @InterfaceAudience.Private public class NameNode extends ReconfigurableBase implements NameNodeStatusMXBean { static { HdfsConfiguration.init(); } private InMemoryLevelDBAliasMapServer levelDBAliasMapServer; /** * Categories of operations supported by the namenode. */ public enum OperationCategory { /** Operations that are state agnostic */ UNCHECKED, /** Read operation that does not change the namespace state */ READ, /** Write operation that changes the namespace state */ WRITE, /** Operations related to checkpointing */ CHECKPOINT, /** Operations related to {@link JournalProtocol} */ JOURNAL } /** * HDFS configuration can have three types of parameters: * <ol> * <li>Parameters that are common for all the name services in the cluster.</li> * <li>Parameters that are specific to a name service. These keys are suffixed * with nameserviceId in the configuration. For example, * "dfs.namenode.rpc-address.nameservice1".</li> * <li>Parameters that are specific to a single name node. These keys are suffixed * with nameserviceId and namenodeId in the configuration. for example, * "dfs.namenode.rpc-address.nameservice1.namenode1"</li> * </ol> * * In the latter cases, operators may specify the configuration without * any suffix, with a nameservice suffix, or with a nameservice and namenode * suffix. The more specific suffix will take precedence. * * These keys are specific to a given namenode, and thus may be configured * globally, for a nameservice, or for a specific namenode within a nameservice. */ public static final String[] NAMENODE_SPECIFIC_KEYS = { DFS_NAMENODE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_BIND_HOST_KEY, DFS_NAMENODE_NAME_DIR_KEY, DFS_NAMENODE_EDITS_DIR_KEY, DFS_NAMENODE_SHARED_EDITS_DIR_KEY, DFS_NAMENODE_CHECKPOINT_DIR_KEY, DFS_NAMENODE_CHECKPOINT_EDITS_DIR_KEY, DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY, DFS_NAMENODE_LIFELINE_RPC_BIND_HOST_KEY, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY, DFS_NAMENODE_HTTP_ADDRESS_KEY, DFS_NAMENODE_HTTPS_ADDRESS_KEY, DFS_NAMENODE_HTTP_BIND_HOST_KEY, DFS_NAMENODE_HTTPS_BIND_HOST_KEY, DFS_NAMENODE_KEYTAB_FILE_KEY, DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY, DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY, DFS_NAMENODE_BACKUP_ADDRESS_KEY, DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY, DFS_NAMENODE_BACKUP_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY, DFS_HA_FENCE_METHODS_KEY, DFS_HA_ZKFC_PORT_KEY, }; /** * @see #NAMENODE_SPECIFIC_KEYS * These keys are specific to a nameservice, but may not be overridden * for a specific namenode. */ public static final String[] NAMESERVICE_SPECIFIC_KEYS = { DFS_HA_AUTO_FAILOVER_ENABLED_KEY }; private String ipcClientRPCBackoffEnable; /** A list of property that are reconfigurable at runtime. */ private final TreeSet<String> reconfigurableProperties = Sets.newTreeSet(Lists.newArrayList( DFS_HEARTBEAT_INTERVAL_KEY, DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, FS_PROTECTED_DIRECTORIES, HADOOP_CALLER_CONTEXT_ENABLED_KEY, DFS_STORAGE_POLICY_SATISFIER_MODE_KEY)); private static final String USAGE = "Usage: hdfs namenode [" + StartupOption.BACKUP.getName() + "] | \n\t[" + StartupOption.CHECKPOINT.getName() + "] | \n\t[" + StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() + " cid ] [" + StartupOption.FORCE.getName() + "] [" + StartupOption.NONINTERACTIVE.getName() + "] ] | \n\t[" + StartupOption.UPGRADE.getName() + " [" + StartupOption.CLUSTERID.getName() + " cid]" + " [" + StartupOption.RENAMERESERVED.getName() + "<k-v pairs>] ] | \n\t[" + StartupOption.UPGRADEONLY.getName() + " [" + StartupOption.CLUSTERID.getName() + " cid]" + " [" + StartupOption.RENAMERESERVED.getName() + "<k-v pairs>] ] | \n\t[" + StartupOption.ROLLBACK.getName() + "] | \n\t[" + StartupOption.ROLLINGUPGRADE.getName() + " " + RollingUpgradeStartupOption.getAllOptionString() + " ] | \n\t[" + StartupOption.IMPORT.getName() + "] | \n\t[" + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | \n\t[" + StartupOption.BOOTSTRAPSTANDBY.getName() + " [" + StartupOption.FORCE.getName() + "] [" + StartupOption.NONINTERACTIVE.getName() + "] [" + StartupOption.SKIPSHAREDEDITSCHECK.getName() + "] ] | \n\t[" + StartupOption.RECOVER.getName() + " [ " + StartupOption.FORCE.getName() + "] ] | \n\t[" + StartupOption.METADATAVERSION.getName() + " ]"; public long getProtocolVersion(String protocol, long clientVersion) throws IOException { if (protocol.equals(ClientProtocol.class.getName())) { return ClientProtocol.versionID; } else if (protocol.equals(DatanodeProtocol.class.getName())) { return DatanodeProtocol.versionID; } else if (protocol.equals(NamenodeProtocol.class.getName())) { return NamenodeProtocol.versionID; } else if (protocol.equals(RefreshAuthorizationPolicyProtocol.class.getName())) { return RefreshAuthorizationPolicyProtocol.versionID; } else if (protocol.equals(RefreshUserMappingsProtocol.class.getName())) { return RefreshUserMappingsProtocol.versionID; } else if (protocol.equals(RefreshCallQueueProtocol.class.getName())) { return RefreshCallQueueProtocol.versionID; } else if (protocol.equals(GetUserMappingsProtocol.class.getName())) { return GetUserMappingsProtocol.versionID; } else if (protocol.equals(TraceAdminProtocol.class.getName())) { return TraceAdminProtocol.versionID; } else { throw new IOException("Unknown protocol to name node: " + protocol); } } /** * @deprecated Use {@link HdfsClientConfigKeys#DFS_NAMENODE_RPC_PORT_DEFAULT} * instead. */ @Deprecated public static final int DEFAULT_PORT = DFS_NAMENODE_RPC_PORT_DEFAULT; public static final Logger LOG = LoggerFactory.getLogger(NameNode.class.getName()); public static final Logger stateChangeLog = LoggerFactory.getLogger("org.apache.hadoop.hdfs.StateChange"); public static final Logger blockStateChangeLog = LoggerFactory.getLogger("BlockStateChange"); public static final HAState ACTIVE_STATE = new ActiveState(); public static final HAState STANDBY_STATE = new StandbyState(); private static final String NAMENODE_HTRACE_PREFIX = "namenode.htrace."; public static final Log MetricsLog = LogFactory.getLog("NameNodeMetricsLog"); protected FSNamesystem namesystem; protected final NamenodeRole role; private volatile HAState state; private final boolean haEnabled; private final HAContext haContext; protected final boolean allowStaleStandbyReads; private AtomicBoolean started = new AtomicBoolean(false); private final static int HEALTH_MONITOR_WARN_THRESHOLD_MS = 5000; /** httpServer */ protected NameNodeHttpServer httpServer; private Thread emptier; /** only used for testing purposes */ protected boolean stopRequested = false; /** Registration information of this name-node */ protected NamenodeRegistration nodeRegistration; /** Activated plug-ins. */ private List<ServicePlugin> plugins; private NameNodeRpcServer rpcServer; private JvmPauseMonitor pauseMonitor; private ObjectName nameNodeStatusBeanName; protected final Tracer tracer; protected final TracerConfigurationManager tracerConfigurationManager; ScheduledThreadPoolExecutor metricsLoggerTimer; /** * The namenode address that clients will use to access this namenode * or the name service. For HA configurations using logical URI, it * will be the logical address. */ private String clientNamenodeAddress; /** Format a new filesystem. Destroys any filesystem that may already * exist at this location. **/ public static void format(Configuration conf) throws IOException { format(conf, true, true); } static NameNodeMetrics metrics; private static final StartupProgress startupProgress = new StartupProgress(); /** Return the {@link FSNamesystem} object. * @return {@link FSNamesystem} object. */ public FSNamesystem getNamesystem() { return namesystem; } public NamenodeProtocols getRpcServer() { return rpcServer; } public void queueExternalCall(ExternalCall<?> extCall) throws IOException, InterruptedException { if (rpcServer == null) { throw new RetriableException("Namenode is in startup mode"); } rpcServer.getClientRpcServer().queueCall(extCall); } public static void initMetrics(Configuration conf, NamenodeRole role) { metrics = NameNodeMetrics.create(conf, role); } public static NameNodeMetrics getNameNodeMetrics() { return metrics; } /** * Returns object used for reporting namenode startup progress. * * @return StartupProgress for reporting namenode startup progress */ public static StartupProgress getStartupProgress() { return startupProgress; } /** * Return the service name of the issued delegation token. * * @return The name service id in HA-mode, or the rpc address in non-HA mode */ public String getTokenServiceName() { return getClientNamenodeAddress(); } /** * Get the namenode address to be used by clients. * @return nn address */ public String getClientNamenodeAddress() { return clientNamenodeAddress; } /** * Set the configuration property for the service rpc address * to address */ public static void setServiceAddress(Configuration conf, String address) { LOG.info("Setting ADDRESS {}", address); conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, address); } /** * Fetches the address for services to use when connecting to namenode * based on the value of fallback returns null if the special * address is not specified or returns the default namenode address * to be used by both clients and services. * Services here are datanodes, backup node, any non client connection */ public static InetSocketAddress getServiceAddress(Configuration conf, boolean fallback) { String addr = conf.getTrimmed(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY); if (addr == null || addr.isEmpty()) { return fallback ? DFSUtilClient.getNNAddress(conf) : null; } return DFSUtilClient.getNNAddress(addr); } // // Common NameNode methods implementation for the active name-node role. // public NamenodeRole getRole() { return role; } boolean isRole(NamenodeRole that) { return role.equals(that); } public static String composeNotStartedMessage(NamenodeRole role) { return role + " still not started"; } /** * Given a configuration get the address of the lifeline RPC server. * If the lifeline RPC is not configured returns null. * * @param conf configuration * @return address or null */ InetSocketAddress getLifelineRpcServerAddress(Configuration conf) { String addr = getTrimmedOrNull(conf, DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY); if (addr == null) { return null; } return NetUtils.createSocketAddr(addr); } /** * Given a configuration get the address of the service rpc server * If the service rpc is not configured returns null */ protected InetSocketAddress getServiceRpcServerAddress(Configuration conf) { return NameNode.getServiceAddress(conf, false); } protected InetSocketAddress getRpcServerAddress(Configuration conf) { return DFSUtilClient.getNNAddress(conf); } /** * Given a configuration get the bind host of the lifeline RPC server. * If the bind host is not configured returns null. * * @param conf configuration * @return bind host or null */ String getLifelineRpcServerBindHost(Configuration conf) { return getTrimmedOrNull(conf, DFS_NAMENODE_LIFELINE_RPC_BIND_HOST_KEY); } /** Given a configuration get the bind host of the service rpc server * If the bind host is not configured returns null. */ protected String getServiceRpcServerBindHost(Configuration conf) { return getTrimmedOrNull(conf, DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY); } /** Given a configuration get the bind host of the client rpc server * If the bind host is not configured returns null. */ protected String getRpcServerBindHost(Configuration conf) { return getTrimmedOrNull(conf, DFS_NAMENODE_RPC_BIND_HOST_KEY); } /** * Gets a trimmed value from configuration, or null if no value is defined. * * @param conf configuration * @param key configuration key to get * @return trimmed value, or null if no value is defined */ private static String getTrimmedOrNull(Configuration conf, String key) { String addr = conf.getTrimmed(key); if (addr == null || addr.isEmpty()) { return null; } return addr; } /** * Modifies the configuration to contain the lifeline RPC address setting. * * @param conf configuration to modify * @param lifelineRPCAddress lifeline RPC address */ void setRpcLifelineServerAddress(Configuration conf, InetSocketAddress lifelineRPCAddress) { LOG.info("Setting lifeline RPC address {}", lifelineRPCAddress); conf.set(DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY, NetUtils.getHostPortString(lifelineRPCAddress)); } /** * Modifies the configuration passed to contain the service rpc address setting */ protected void setRpcServiceServerAddress(Configuration conf, InetSocketAddress serviceRPCAddress) { setServiceAddress(conf, NetUtils.getHostPortString(serviceRPCAddress)); } protected void setRpcServerAddress(Configuration conf, InetSocketAddress rpcAddress) { FileSystem.setDefaultUri(conf, DFSUtilClient.getNNUri(rpcAddress)); } protected InetSocketAddress getHttpServerAddress(Configuration conf) { return getHttpAddress(conf); } /** * HTTP server address for binding the endpoint. This method is * for use by the NameNode and its derivatives. It may return * a different address than the one that should be used by clients to * connect to the NameNode. See * {@link DFSConfigKeys#DFS_NAMENODE_HTTP_BIND_HOST_KEY} * * @param conf * @return */ protected InetSocketAddress getHttpServerBindAddress(Configuration conf) { InetSocketAddress bindAddress = getHttpServerAddress(conf); // If DFS_NAMENODE_HTTP_BIND_HOST_KEY exists then it overrides the // host name portion of DFS_NAMENODE_HTTP_ADDRESS_KEY. final String bindHost = conf.getTrimmed(DFS_NAMENODE_HTTP_BIND_HOST_KEY); if (bindHost != null && !bindHost.isEmpty()) { bindAddress = new InetSocketAddress(bindHost, bindAddress.getPort()); } return bindAddress; } /** @return the NameNode HTTP address. */ public static InetSocketAddress getHttpAddress(Configuration conf) { return NetUtils.createSocketAddr( conf.getTrimmed(DFS_NAMENODE_HTTP_ADDRESS_KEY, DFS_NAMENODE_HTTP_ADDRESS_DEFAULT)); } protected void loadNamesystem(Configuration conf) throws IOException { this.namesystem = FSNamesystem.loadFromDisk(conf); } NamenodeRegistration getRegistration() { return nodeRegistration; } NamenodeRegistration setRegistration() { nodeRegistration = new NamenodeRegistration(NetUtils.getHostPortString(getNameNodeAddress()), NetUtils.getHostPortString(getHttpAddress()), getFSImage().getStorage(), getRole()); return nodeRegistration; } /* optimize ugi lookup for RPC operations to avoid a trip through * UGI.getCurrentUser which is synch'ed */ public static UserGroupInformation getRemoteUser() throws IOException { UserGroupInformation ugi = Server.getRemoteUser(); return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser(); } /** * Login as the configured user for the NameNode. */ void loginAsNameNodeUser(Configuration conf) throws IOException { InetSocketAddress socAddr = getRpcServerAddress(conf); SecurityUtil.login(conf, DFS_NAMENODE_KEYTAB_FILE_KEY, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName()); } /** * Initialize name-node. * * @param conf the configuration */ protected void initialize(Configuration conf) throws IOException { if (conf.get(HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS) == null) { String intervals = conf.get(DFS_METRICS_PERCENTILES_INTERVALS_KEY); if (intervals != null) { conf.set(HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS, intervals); } } UserGroupInformation.setConfiguration(conf); loginAsNameNodeUser(conf); NameNode.initMetrics(conf, this.getRole()); StartupProgressMetrics.register(startupProgress); pauseMonitor = new JvmPauseMonitor(); pauseMonitor.init(conf); pauseMonitor.start(); metrics.getJvmMetrics().setPauseMonitor(pauseMonitor); if (NamenodeRole.NAMENODE == role) { startHttpServer(conf); } loadNamesystem(conf); startAliasMapServerIfNecessary(conf); rpcServer = createRpcServer(conf); initReconfigurableBackoffKey(); if (clientNamenodeAddress == null) { // This is expected for MiniDFSCluster. Set it now using // the RPC server's bind address. clientNamenodeAddress = NetUtils.getHostPortString(getNameNodeAddress()); LOG.info("Clients are to use " + clientNamenodeAddress + " to access" + " this namenode/service."); } if (NamenodeRole.NAMENODE == role) { httpServer.setNameNodeAddress(getNameNodeAddress()); httpServer.setFSImage(getFSImage()); } startCommonServices(conf); startMetricsLogger(conf); } private void startAliasMapServerIfNecessary(Configuration conf) throws IOException { if (conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED, DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED_DEFAULT) && conf.getBoolean(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_ENABLED, DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_ENABLED_DEFAULT)) { levelDBAliasMapServer = new InMemoryLevelDBAliasMapServer(InMemoryAliasMap::init, namesystem.getBlockPoolId()); levelDBAliasMapServer.setConf(conf); levelDBAliasMapServer.start(); } } private void initReconfigurableBackoffKey() { ipcClientRPCBackoffEnable = buildBackoffEnableKey(rpcServer.getClientRpcServer().getPort()); reconfigurableProperties.add(ipcClientRPCBackoffEnable); } static String buildBackoffEnableKey(final int port) { // format used to construct backoff enable key, e.g. ipc.8020.backoff.enable String format = "%s.%d.%s"; return String.format(format, IPC_NAMESPACE, port, IPC_BACKOFF_ENABLE); } /** * Start a timer to periodically write NameNode metrics to the log * file. This behavior can be disabled by configuration. * @param conf */ protected void startMetricsLogger(Configuration conf) { long metricsLoggerPeriodSec = conf.getInt(DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_KEY, DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT); if (metricsLoggerPeriodSec <= 0) { return; } MetricsLoggerTask.makeMetricsLoggerAsync(MetricsLog); // Schedule the periodic logging. metricsLoggerTimer = new ScheduledThreadPoolExecutor(1); metricsLoggerTimer.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); metricsLoggerTimer.scheduleWithFixedDelay(new MetricsLoggerTask(MetricsLog, "NameNode", (short) 128), metricsLoggerPeriodSec, metricsLoggerPeriodSec, TimeUnit.SECONDS); } protected void stopMetricsLogger() { if (metricsLoggerTimer != null) { metricsLoggerTimer.shutdown(); metricsLoggerTimer = null; } } /** * Create the RPC server implementation. Used as an extension point for the * BackupNode. */ protected NameNodeRpcServer createRpcServer(Configuration conf) throws IOException { return new NameNodeRpcServer(conf, this); } /** Start the services common to active and standby states */ private void startCommonServices(Configuration conf) throws IOException { namesystem.startCommonServices(conf, haContext); registerNNSMXBean(); if (NamenodeRole.NAMENODE != role) { startHttpServer(conf); httpServer.setNameNodeAddress(getNameNodeAddress()); httpServer.setFSImage(getFSImage()); } rpcServer.start(); try { plugins = conf.getInstances(DFS_NAMENODE_PLUGINS_KEY, ServicePlugin.class); } catch (RuntimeException e) { String pluginsValue = conf.get(DFS_NAMENODE_PLUGINS_KEY); LOG.error("Unable to load NameNode plugins. Specified list of plugins: " + pluginsValue, e); throw e; } for (ServicePlugin p : plugins) { try { p.start(this); } catch (Throwable t) { LOG.warn("ServicePlugin " + p + " could not be started", t); } } LOG.info(getRole() + " RPC up at: " + getNameNodeAddress()); if (rpcServer.getServiceRpcAddress() != null) { LOG.info(getRole() + " service RPC up at: " + rpcServer.getServiceRpcAddress()); } } private void stopCommonServices() { if (rpcServer != null) rpcServer.stop(); if (namesystem != null) namesystem.close(); if (pauseMonitor != null) pauseMonitor.stop(); if (plugins != null) { for (ServicePlugin p : plugins) { try { p.stop(); } catch (Throwable t) { LOG.warn("ServicePlugin " + p + " could not be stopped", t); } } } stopHttpServer(); } private void startTrashEmptier(final Configuration conf) throws IOException { long trashInterval = conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT); if (trashInterval == 0) { return; } else if (trashInterval < 0) { throw new IOException("Cannot start trash emptier with negative interval." + " Set " + FS_TRASH_INTERVAL_KEY + " to a positive value."); } // This may be called from the transitionToActive code path, in which // case the current user is the administrator, not the NN. The trash // emptier needs to run as the NN. See HDFS-3972. FileSystem fs = SecurityUtil.doAsLoginUser(new PrivilegedExceptionAction<FileSystem>() { @Override public FileSystem run() throws IOException { return FileSystem.get(conf); } }); this.emptier = new Thread(new Trash(fs, conf).getEmptier(), "Trash Emptier"); this.emptier.setDaemon(true); this.emptier.start(); } private void stopTrashEmptier() { if (this.emptier != null) { emptier.interrupt(); emptier = null; } } private void startHttpServer(final Configuration conf) throws IOException { httpServer = new NameNodeHttpServer(conf, this, getHttpServerBindAddress(conf)); httpServer.start(); httpServer.setStartupProgress(startupProgress); } private void stopHttpServer() { try { if (httpServer != null) httpServer.stop(); } catch (Exception e) { LOG.error("Exception while stopping httpserver", e); } } /** * Start NameNode. * <p> * The name-node can be started with one of the following startup options: * <ul> * <li>{@link StartupOption#REGULAR REGULAR} - normal name node startup</li> * <li>{@link StartupOption#FORMAT FORMAT} - format name node</li> * <li>{@link StartupOption#BACKUP BACKUP} - start backup node</li> * <li>{@link StartupOption#CHECKPOINT CHECKPOINT} - start checkpoint node</li> * <li>{@link StartupOption#UPGRADE UPGRADE} - start the cluster * <li>{@link StartupOption#UPGRADEONLY UPGRADEONLY} - upgrade the cluster * upgrade and create a snapshot of the current file system state</li> * <li>{@link StartupOption#RECOVER RECOVERY} - recover name node * metadata</li> * <li>{@link StartupOption#ROLLBACK ROLLBACK} - roll the * cluster back to the previous state</li> * <li>{@link StartupOption#IMPORT IMPORT} - import checkpoint</li> * </ul> * The option is passed via configuration field: * <tt>dfs.namenode.startup</tt> * * The conf will be modified to reflect the actual ports on which * the NameNode is up and running if the user passes the port as * <code>zero</code> in the conf. * * @param conf confirguration * @throws IOException */ public NameNode(Configuration conf) throws IOException { this(conf, NamenodeRole.NAMENODE); } protected NameNode(Configuration conf, NamenodeRole role) throws IOException { super(conf); this.tracer = new Tracer.Builder("NameNode").conf(TraceUtils.wrapHadoopConf(NAMENODE_HTRACE_PREFIX, conf)) .build(); this.tracerConfigurationManager = new TracerConfigurationManager(NAMENODE_HTRACE_PREFIX, conf); this.role = role; String nsId = getNameServiceId(conf); String namenodeId = HAUtil.getNameNodeId(conf, nsId); clientNamenodeAddress = NameNodeUtils.getClientNamenodeAddress(conf, nsId); if (clientNamenodeAddress != null) { LOG.info("Clients should use {} to access" + " this namenode/service.", clientNamenodeAddress); } this.haEnabled = HAUtil.isHAEnabled(conf, nsId); state = createHAState(getStartupOption(conf)); this.allowStaleStandbyReads = HAUtil.shouldAllowStandbyReads(conf); this.haContext = createHAContext(); try { initializeGenericKeys(conf, nsId, namenodeId); initialize(getConf()); try { haContext.writeLock(); state.prepareToEnterState(haContext); state.enterState(haContext); } finally { haContext.writeUnlock(); } } catch (IOException e) { this.stopAtException(e); throw e; } catch (HadoopIllegalArgumentException e) { this.stopAtException(e); throw e; } this.started.set(true); } private void stopAtException(Exception e) { try { this.stop(); } catch (Exception ex) { LOG.warn("Encountered exception when handling exception (" + e.getMessage() + "):", ex); } } protected HAState createHAState(StartupOption startOpt) { if (!haEnabled || startOpt == StartupOption.UPGRADE || startOpt == StartupOption.UPGRADEONLY) { return ACTIVE_STATE; } else { return STANDBY_STATE; } } protected HAContext createHAContext() { return new NameNodeHAContext(); } /** * Wait for service to finish. * (Normally, it runs forever.) */ public void join() { try { rpcServer.join(); } catch (InterruptedException ie) { LOG.info("Caught interrupted exception ", ie); } } /** * Stop all NameNode threads and wait for all to finish. */ public void stop() { synchronized (this) { if (stopRequested) return; stopRequested = true; } try { if (state != null) { state.exitState(haContext); } } catch (ServiceFailedException e) { LOG.warn("Encountered exception while exiting state ", e); } finally { stopMetricsLogger(); stopCommonServices(); if (metrics != null) { metrics.shutdown(); } if (namesystem != null) { namesystem.shutdown(); } if (nameNodeStatusBeanName != null) { MBeans.unregister(nameNodeStatusBeanName); nameNodeStatusBeanName = null; } if (levelDBAliasMapServer != null) { levelDBAliasMapServer.close(); } } tracer.close(); } synchronized boolean isStopRequested() { return stopRequested; } /** * Is the cluster currently in safe mode? */ public boolean isInSafeMode() { return namesystem.isInSafeMode(); } /** get FSImage */ @VisibleForTesting public FSImage getFSImage() { return namesystem.getFSImage(); } /** * @return NameNode RPC address */ public InetSocketAddress getNameNodeAddress() { return rpcServer.getRpcAddress(); } /** * @return NameNode RPC address in "host:port" string form */ public String getNameNodeAddressHostPortString() { return NetUtils.getHostPortString(getNameNodeAddress()); } /** * @return NameNode service RPC address if configured, the * NameNode RPC address otherwise */ public InetSocketAddress getServiceRpcAddress() { final InetSocketAddress serviceAddr = rpcServer.getServiceRpcAddress(); return serviceAddr == null ? getNameNodeAddress() : serviceAddr; } /** * @return NameNode HTTP address, used by the Web UI, image transfer, * and HTTP-based file system clients like WebHDFS */ public InetSocketAddress getHttpAddress() { return httpServer.getHttpAddress(); } /** * @return NameNode HTTPS address, used by the Web UI, image transfer, * and HTTP-based file system clients like WebHDFS */ public InetSocketAddress getHttpsAddress() { return httpServer.getHttpsAddress(); } /** * @return NameNodeHttpServer, used by unit tests to ensure a full shutdown, * so that no bind exception is thrown during restart. */ @VisibleForTesting public void joinHttpServer() { if (httpServer != null) { try { httpServer.join(); } catch (InterruptedException e) { LOG.info("Caught InterruptedException joining NameNodeHttpServer", e); Thread.currentThread().interrupt(); } } } /** * Verify that configured directories exist, then * Interactively confirm that formatting is desired * for each existing directory and format them. * * @param conf configuration to use * @param force if true, format regardless of whether dirs exist * @return true if formatting was aborted, false otherwise * @throws IOException */ private static boolean format(Configuration conf, boolean force, boolean isInteractive) throws IOException { String nsId = DFSUtil.getNamenodeNameServiceId(conf); String namenodeId = HAUtil.getNameNodeId(conf, nsId); initializeGenericKeys(conf, nsId, namenodeId); checkAllowFormat(conf); if (UserGroupInformation.isSecurityEnabled()) { InetSocketAddress socAddr = DFSUtilClient.getNNAddress(conf); SecurityUtil.login(conf, DFS_NAMENODE_KEYTAB_FILE_KEY, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName()); } Collection<URI> nameDirsToFormat = FSNamesystem.getNamespaceDirs(conf); List<URI> sharedDirs = FSNamesystem.getSharedEditsDirs(conf); List<URI> dirsToPrompt = new ArrayList<URI>(); dirsToPrompt.addAll(nameDirsToFormat); dirsToPrompt.addAll(sharedDirs); List<URI> editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf); // if clusterID is not provided - see if you can find the current one String clusterId = StartupOption.FORMAT.getClusterId(); if (clusterId == null || clusterId.equals("")) { //Generate a new cluster id clusterId = NNStorage.newClusterID(); } System.out.println("Formatting using clusterid: " + clusterId); FSImage fsImage = new FSImage(conf, nameDirsToFormat, editDirsToFormat); try { FSNamesystem fsn = new FSNamesystem(conf, fsImage); fsImage.getEditLog().initJournalsForWrite(); // Abort NameNode format if reformat is disabled and if // meta-dir already exists if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) { force = false; isInteractive = false; for (StorageDirectory sd : fsImage.storage.dirIterable(null)) { if (sd.hasSomeData()) { throw new NameNodeFormatException( "NameNode format aborted as reformat is disabled for " + "this cluster."); } } } if (!fsImage.confirmFormat(force, isInteractive)) { return true; // aborted } fsImage.format(fsn, clusterId, force); } catch (IOException ioe) { LOG.warn("Encountered exception during format: ", ioe); fsImage.close(); throw ioe; } return false; } public static void checkAllowFormat(Configuration conf) throws IOException { if (!conf.getBoolean(DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY, DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT)) { throw new IOException("The option " + DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY + " is set to false for this filesystem, so it " + "cannot be formatted. You will need to set " + DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY + " parameter " + "to true in order to format this filesystem"); } } @VisibleForTesting public static boolean initializeSharedEdits(Configuration conf) throws IOException { return initializeSharedEdits(conf, true); } @VisibleForTesting public static boolean initializeSharedEdits(Configuration conf, boolean force) throws IOException { return initializeSharedEdits(conf, force, false); } /** * Clone the supplied configuration but remove the shared edits dirs. * * @param conf Supplies the original configuration. * @return Cloned configuration without the shared edit dirs. * @throws IOException on failure to generate the configuration. */ private static Configuration getConfigurationWithoutSharedEdits(Configuration conf) throws IOException { List<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(conf, false); String editsDirsString = Joiner.on(",").join(editsDirs); Configuration confWithoutShared = new Configuration(conf); confWithoutShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); confWithoutShared.setStrings(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, editsDirsString); return confWithoutShared; } /** * Format a new shared edits dir and copy in enough edit log segments so that * the standby NN can start up. * * @param conf configuration * @param force format regardless of whether or not the shared edits dir exists * @param interactive prompt the user when a dir exists * @return true if the command aborts, false otherwise */ private static boolean initializeSharedEdits(Configuration conf, boolean force, boolean interactive) throws IOException { String nsId = DFSUtil.getNamenodeNameServiceId(conf); String namenodeId = HAUtil.getNameNodeId(conf, nsId); initializeGenericKeys(conf, nsId, namenodeId); if (conf.get(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY) == null) { LOG.error("No shared edits directory configured for namespace " + nsId + " namenode " + namenodeId); return false; } if (UserGroupInformation.isSecurityEnabled()) { InetSocketAddress socAddr = DFSUtilClient.getNNAddress(conf); SecurityUtil.login(conf, DFS_NAMENODE_KEYTAB_FILE_KEY, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName()); } NNStorage existingStorage = null; FSImage sharedEditsImage = null; try { FSNamesystem fsns = FSNamesystem.loadFromDisk(getConfigurationWithoutSharedEdits(conf)); existingStorage = fsns.getFSImage().getStorage(); NamespaceInfo nsInfo = existingStorage.getNamespaceInfo(); List<URI> sharedEditsDirs = FSNamesystem.getSharedEditsDirs(conf); sharedEditsImage = new FSImage(conf, Lists.<URI>newArrayList(), sharedEditsDirs); sharedEditsImage.getEditLog().initJournalsForWrite(); if (!sharedEditsImage.confirmFormat(force, interactive)) { return true; // abort } NNStorage newSharedStorage = sharedEditsImage.getStorage(); // Call Storage.format instead of FSImage.format here, since we don't // actually want to save a checkpoint - just prime the dirs with // the existing namespace info newSharedStorage.format(nsInfo); sharedEditsImage.getEditLog().formatNonFileJournals(nsInfo, force); // Need to make sure the edit log segments are in good shape to initialize // the shared edits dir. fsns.getFSImage().getEditLog().close(); fsns.getFSImage().getEditLog().initJournalsForWrite(); fsns.getFSImage().getEditLog().recoverUnclosedStreams(); copyEditLogSegmentsToSharedDir(fsns, sharedEditsDirs, newSharedStorage, conf); } catch (IOException ioe) { LOG.error("Could not initialize shared edits dir", ioe); return true; // aborted } finally { if (sharedEditsImage != null) { try { sharedEditsImage.close(); } catch (IOException ioe) { LOG.warn("Could not close sharedEditsImage", ioe); } } // Have to unlock storage explicitly for the case when we're running in a // unit test, which runs in the same JVM as NNs. if (existingStorage != null) { try { existingStorage.unlockAll(); } catch (IOException ioe) { LOG.warn("Could not unlock storage directories", ioe); return true; // aborted } } } return false; // did not abort } private static void copyEditLogSegmentsToSharedDir(FSNamesystem fsns, Collection<URI> sharedEditsDirs, NNStorage newSharedStorage, Configuration conf) throws IOException { Preconditions.checkArgument(!sharedEditsDirs.isEmpty(), "No shared edits specified"); // Copy edit log segments into the new shared edits dir. List<URI> sharedEditsUris = new ArrayList<URI>(sharedEditsDirs); FSEditLog newSharedEditLog = new FSEditLog(conf, newSharedStorage, sharedEditsUris); newSharedEditLog.initJournalsForWrite(); newSharedEditLog.recoverUnclosedStreams(); FSEditLog sourceEditLog = fsns.getFSImage().editLog; long fromTxId = fsns.getFSImage().getMostRecentCheckpointTxId(); Collection<EditLogInputStream> streams = null; try { streams = sourceEditLog.selectInputStreams(fromTxId + 1, 0); // Set the nextTxid to the CheckpointTxId+1 newSharedEditLog.setNextTxId(fromTxId + 1); // Copy all edits after last CheckpointTxId to shared edits dir for (EditLogInputStream stream : streams) { LOG.debug("Beginning to copy stream " + stream + " to shared edits"); FSEditLogOp op; boolean segmentOpen = false; while ((op = stream.readOp()) != null) { if (LOG.isTraceEnabled()) { LOG.trace("copying op: " + op); } if (!segmentOpen) { newSharedEditLog.startLogSegment(op.txid, false, fsns.getEffectiveLayoutVersion()); segmentOpen = true; } newSharedEditLog.logEdit(op); if (op.opCode == FSEditLogOpCodes.OP_END_LOG_SEGMENT) { newSharedEditLog.endCurrentLogSegment(false); LOG.debug("ending log segment because of END_LOG_SEGMENT op in " + stream); segmentOpen = false; } } if (segmentOpen) { LOG.debug("ending log segment because of end of stream in " + stream); newSharedEditLog.logSync(); newSharedEditLog.endCurrentLogSegment(false); segmentOpen = false; } } } finally { if (streams != null) { FSEditLog.closeAllStreams(streams); } } } @VisibleForTesting public static boolean doRollback(Configuration conf, boolean isConfirmationNeeded) throws IOException { String nsId = DFSUtil.getNamenodeNameServiceId(conf); String namenodeId = HAUtil.getNameNodeId(conf, nsId); initializeGenericKeys(conf, nsId, namenodeId); FSNamesystem nsys = new FSNamesystem(conf, new FSImage(conf)); System.err.print("\"rollBack\" will remove the current state of the file system,\n" + "returning you to the state prior to initiating your recent.\n" + "upgrade. This action is permanent and cannot be undone. If you\n" + "are performing a rollback in an HA environment, you should be\n" + "certain that no NameNode process is running on any host."); if (isConfirmationNeeded) { if (!confirmPrompt("Roll back file system state?")) { System.err.println("Rollback aborted."); return true; } } nsys.getFSImage().doRollback(nsys); return false; } private static void printUsage(PrintStream out) { out.println(USAGE + "\n"); } @VisibleForTesting static StartupOption parseArguments(String args[]) { int argsLen = (args == null) ? 0 : args.length; StartupOption startOpt = StartupOption.REGULAR; for (int i = 0; i < argsLen; i++) { String cmd = args[i]; if (StartupOption.FORMAT.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.FORMAT; for (i = i + 1; i < argsLen; i++) { if (args[i].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { i++; if (i >= argsLen) { // if no cluster id specified, return null LOG.error("Must specify a valid cluster ID after the " + StartupOption.CLUSTERID.getName() + " flag"); return null; } String clusterId = args[i]; // Make sure an id is specified and not another flag if (clusterId.isEmpty() || clusterId.equalsIgnoreCase(StartupOption.FORCE.getName()) || clusterId.equalsIgnoreCase(StartupOption.NONINTERACTIVE.getName())) { LOG.error("Must specify a valid cluster ID after the " + StartupOption.CLUSTERID.getName() + " flag"); return null; } startOpt.setClusterId(clusterId); } if (args[i].equalsIgnoreCase(StartupOption.FORCE.getName())) { startOpt.setForceFormat(true); } if (args[i].equalsIgnoreCase(StartupOption.NONINTERACTIVE.getName())) { startOpt.setInteractiveFormat(false); } } } else if (StartupOption.GENCLUSTERID.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.GENCLUSTERID; } else if (StartupOption.REGULAR.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.REGULAR; } else if (StartupOption.BACKUP.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.BACKUP; } else if (StartupOption.CHECKPOINT.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.CHECKPOINT; } else if (StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd) || StartupOption.UPGRADEONLY.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd) ? StartupOption.UPGRADE : StartupOption.UPGRADEONLY; /* Can be followed by CLUSTERID with a required parameter or * RENAMERESERVED with an optional parameter */ while (i + 1 < argsLen) { String flag = args[i + 1]; if (flag.equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { if (i + 2 < argsLen) { i += 2; startOpt.setClusterId(args[i]); } else { LOG.error("Must specify a valid cluster ID after the " + StartupOption.CLUSTERID.getName() + " flag"); return null; } } else if (flag.equalsIgnoreCase(StartupOption.RENAMERESERVED.getName())) { if (i + 2 < argsLen) { FSImageFormat.setRenameReservedPairs(args[i + 2]); i += 2; } else { FSImageFormat.useDefaultRenameReservedPairs(); i += 1; } } else { LOG.error("Unknown upgrade flag " + flag); return null; } } } else if (StartupOption.ROLLINGUPGRADE.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.ROLLINGUPGRADE; ++i; if (i >= argsLen) { LOG.error("Must specify a rolling upgrade startup option " + RollingUpgradeStartupOption.getAllOptionString()); return null; } startOpt.setRollingUpgradeStartupOption(args[i]); } else if (StartupOption.ROLLBACK.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.ROLLBACK; } else if (StartupOption.IMPORT.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.IMPORT; } else if (StartupOption.BOOTSTRAPSTANDBY.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.BOOTSTRAPSTANDBY; return startOpt; } else if (StartupOption.INITIALIZESHAREDEDITS.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.INITIALIZESHAREDEDITS; for (i = i + 1; i < argsLen; i++) { if (StartupOption.NONINTERACTIVE.getName().equals(args[i])) { startOpt.setInteractiveFormat(false); } else if (StartupOption.FORCE.getName().equals(args[i])) { startOpt.setForceFormat(true); } else { LOG.error("Invalid argument: " + args[i]); return null; } } return startOpt; } else if (StartupOption.RECOVER.getName().equalsIgnoreCase(cmd)) { if (startOpt != StartupOption.REGULAR) { throw new RuntimeException("Can't combine -recover with " + "other startup options."); } startOpt = StartupOption.RECOVER; while (++i < argsLen) { if (args[i].equalsIgnoreCase(StartupOption.FORCE.getName())) { startOpt.setForce(MetaRecoveryContext.FORCE_FIRST_CHOICE); } else { throw new RuntimeException( "Error parsing recovery options: " + "can't understand option \"" + args[i] + "\""); } } } else if (StartupOption.METADATAVERSION.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.METADATAVERSION; } else { return null; } } return startOpt; } private static void setStartupOption(Configuration conf, StartupOption opt) { conf.set(DFS_NAMENODE_STARTUP_KEY, opt.name()); } public static StartupOption getStartupOption(Configuration conf) { return StartupOption.valueOf(conf.get(DFS_NAMENODE_STARTUP_KEY, StartupOption.REGULAR.toString())); } private static void doRecovery(StartupOption startOpt, Configuration conf) throws IOException { String nsId = DFSUtil.getNamenodeNameServiceId(conf); String namenodeId = HAUtil.getNameNodeId(conf, nsId); initializeGenericKeys(conf, nsId, namenodeId); if (startOpt.getForce() < MetaRecoveryContext.FORCE_ALL) { if (!confirmPrompt("You have selected Metadata Recovery mode. " + "This mode is intended to recover lost metadata on a corrupt " + "filesystem. Metadata recovery mode often permanently deletes " + "data from your HDFS filesystem. Please back up your edit log " + "and fsimage before trying this!\n\n" + "Are you ready to proceed? (Y/N)\n")) { System.err.println("Recovery aborted at user request.\n"); return; } } MetaRecoveryContext.LOG.info("starting recovery..."); UserGroupInformation.setConfiguration(conf); NameNode.initMetrics(conf, startOpt.toNodeRole()); FSNamesystem fsn = null; try { fsn = FSNamesystem.loadFromDisk(conf); fsn.getFSImage().saveNamespace(fsn); MetaRecoveryContext.LOG.info("RECOVERY COMPLETE"); } catch (IOException e) { MetaRecoveryContext.LOG.info("RECOVERY FAILED: caught exception", e); throw e; } catch (RuntimeException e) { MetaRecoveryContext.LOG.info("RECOVERY FAILED: caught exception", e); throw e; } finally { if (fsn != null) fsn.close(); } } /** * Verify that configured directories exist, then print the metadata versions * of the software and the image. * * @param conf configuration to use * @throws IOException */ private static boolean printMetadataVersion(Configuration conf) throws IOException { final String nsId = DFSUtil.getNamenodeNameServiceId(conf); final String namenodeId = HAUtil.getNameNodeId(conf, nsId); NameNode.initializeGenericKeys(conf, nsId, namenodeId); final FSImage fsImage = new FSImage(conf); final FSNamesystem fs = new FSNamesystem(conf, fsImage, false); return fsImage.recoverTransitionRead(StartupOption.METADATAVERSION, fs, null); } public static NameNode createNameNode(String argv[], Configuration conf) throws IOException { LOG.info("createNameNode " + Arrays.asList(argv)); if (conf == null) conf = new HdfsConfiguration(); // Parse out some generic args into Configuration. GenericOptionsParser hParser = new GenericOptionsParser(conf, argv); argv = hParser.getRemainingArgs(); // Parse the rest, NN specific args. StartupOption startOpt = parseArguments(argv); if (startOpt == null) { printUsage(System.err); return null; } setStartupOption(conf, startOpt); boolean aborted = false; switch (startOpt) { case FORMAT: aborted = format(conf, startOpt.getForceFormat(), startOpt.getInteractiveFormat()); terminate(aborted ? 1 : 0); return null; // avoid javac warning case GENCLUSTERID: System.err.println("Generating new cluster id:"); System.out.println(NNStorage.newClusterID()); terminate(0); return null; case ROLLBACK: aborted = doRollback(conf, true); terminate(aborted ? 1 : 0); return null; // avoid warning case BOOTSTRAPSTANDBY: String[] toolArgs = Arrays.copyOfRange(argv, 1, argv.length); int rc = BootstrapStandby.run(toolArgs, conf); terminate(rc); return null; // avoid warning case INITIALIZESHAREDEDITS: aborted = initializeSharedEdits(conf, startOpt.getForceFormat(), startOpt.getInteractiveFormat()); terminate(aborted ? 1 : 0); return null; // avoid warning case BACKUP: case CHECKPOINT: NamenodeRole role = startOpt.toNodeRole(); DefaultMetricsSystem.initialize(role.toString().replace(" ", "")); return new BackupNode(conf, role); case RECOVER: NameNode.doRecovery(startOpt, conf); return null; case METADATAVERSION: printMetadataVersion(conf); terminate(0); return null; // avoid javac warning case UPGRADEONLY: DefaultMetricsSystem.initialize("NameNode"); new NameNode(conf); terminate(0); return null; default: DefaultMetricsSystem.initialize("NameNode"); return new NameNode(conf); } } /** * In federation configuration is set for a set of * namenode and secondary namenode/backup/checkpointer, which are * grouped under a logical nameservice ID. The configuration keys specific * to them have suffix set to configured nameserviceId. * * This method copies the value from specific key of format key.nameserviceId * to key, to set up the generic configuration. Once this is done, only * generic version of the configuration is read in rest of the code, for * backward compatibility and simpler code changes. * * @param conf * Configuration object to lookup specific key and to set the value * to the key passed. Note the conf object is modified * @param nameserviceId name service Id (to distinguish federated NNs) * @param namenodeId the namenode ID (to distinguish HA NNs) * @see DFSUtil#setGenericConf(Configuration, String, String, String...) */ public static void initializeGenericKeys(Configuration conf, String nameserviceId, String namenodeId) { if ((nameserviceId != null && !nameserviceId.isEmpty()) || (namenodeId != null && !namenodeId.isEmpty())) { if (nameserviceId != null) { conf.set(DFS_NAMESERVICE_ID, nameserviceId); } if (namenodeId != null) { conf.set(DFS_HA_NAMENODE_ID_KEY, namenodeId); } DFSUtil.setGenericConf(conf, nameserviceId, namenodeId, NAMENODE_SPECIFIC_KEYS); DFSUtil.setGenericConf(conf, nameserviceId, null, NAMESERVICE_SPECIFIC_KEYS); } // If the RPC address is set use it to (re-)configure the default FS if (conf.get(DFS_NAMENODE_RPC_ADDRESS_KEY) != null) { URI defaultUri = URI .create(HdfsConstants.HDFS_URI_SCHEME + "://" + conf.get(DFS_NAMENODE_RPC_ADDRESS_KEY)); conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); if (LOG.isDebugEnabled()) { LOG.debug("Setting " + FS_DEFAULT_NAME_KEY + " to " + defaultUri.toString()); } } } /** * Get the name service Id for the node * @return name service Id or null if federation is not configured */ protected String getNameServiceId(Configuration conf) { return DFSUtil.getNamenodeNameServiceId(conf); } /** */ public static void main(String argv[]) throws Exception { if (DFSUtil.parseHelpArgument(argv, NameNode.USAGE, System.out, true)) { System.exit(0); } try { StringUtils.startupShutdownMessage(NameNode.class, argv, LOG); NameNode namenode = createNameNode(argv, null); if (namenode != null) { namenode.join(); } } catch (Throwable e) { LOG.error("Failed to start namenode.", e); terminate(1, e); } } synchronized void monitorHealth() throws HealthCheckFailedException, AccessControlException { namesystem.checkSuperuserPrivilege(); if (!haEnabled) { return; // no-op, if HA is not enabled } long start = Time.monotonicNow(); getNamesystem().checkAvailableResources(); long end = Time.monotonicNow(); if (end - start >= HEALTH_MONITOR_WARN_THRESHOLD_MS) { // log a warning if it take >= 5 seconds. LOG.warn("Remote IP {} checking available resources took {}ms", Server.getRemoteIp(), end - start); } if (!getNamesystem().nameNodeHasResourcesAvailable()) { throw new HealthCheckFailedException("The NameNode has no resources available"); } } synchronized void transitionToActive() throws ServiceFailedException, AccessControlException { namesystem.checkSuperuserPrivilege(); if (!haEnabled) { throw new ServiceFailedException("HA for namenode is not enabled"); } state.setState(haContext, ACTIVE_STATE); } synchronized void transitionToStandby() throws ServiceFailedException, AccessControlException { namesystem.checkSuperuserPrivilege(); if (!haEnabled) { throw new ServiceFailedException("HA for namenode is not enabled"); } state.setState(haContext, STANDBY_STATE); } synchronized HAServiceStatus getServiceStatus() throws ServiceFailedException, AccessControlException { if (!haEnabled) { throw new ServiceFailedException("HA for namenode is not enabled"); } if (state == null) { return new HAServiceStatus(HAServiceState.INITIALIZING); } HAServiceState retState = state.getServiceState(); HAServiceStatus ret = new HAServiceStatus(retState); if (retState == HAServiceState.STANDBY) { if (namesystem.isInSafeMode()) { ret.setNotReadyToBecomeActive("The NameNode is in safemode. " + namesystem.getSafeModeTip()); } else { ret.setReadyToBecomeActive(); } } else if (retState == HAServiceState.ACTIVE) { ret.setReadyToBecomeActive(); } else { ret.setNotReadyToBecomeActive("State is " + state); } return ret; } synchronized HAServiceState getServiceState() { if (state == null) { return HAServiceState.INITIALIZING; } return state.getServiceState(); } /** * Register NameNodeStatusMXBean */ private void registerNNSMXBean() { nameNodeStatusBeanName = MBeans.register("NameNode", "NameNodeStatus", this); } @Override // NameNodeStatusMXBean public String getNNRole() { String roleStr = ""; NamenodeRole role = getRole(); if (null != role) { roleStr = role.toString(); } return roleStr; } @Override // NameNodeStatusMXBean public String getState() { String servStateStr = ""; HAServiceState servState = getServiceState(); if (null != servState) { servStateStr = servState.toString(); } return servStateStr; } @Override // NameNodeStatusMXBean public String getHostAndPort() { return getNameNodeAddressHostPortString(); } @Override // NameNodeStatusMXBean public boolean isSecurityEnabled() { return UserGroupInformation.isSecurityEnabled(); } @Override // NameNodeStatusMXBean public long getLastHATransitionTime() { return state.getLastHATransitionTime(); } @Override //NameNodeStatusMXBean public long getBytesWithFutureGenerationStamps() { return getNamesystem().getBytesInFuture(); } @Override public String getSlowPeersReport() { return namesystem.getBlockManager().getDatanodeManager().getSlowPeersReport(); } @Override //NameNodeStatusMXBean public String getSlowDisksReport() { return namesystem.getBlockManager().getDatanodeManager().getSlowDisksReport(); } /** * Shutdown the NN immediately in an ungraceful way. Used when it would be * unsafe for the NN to continue operating, e.g. during a failed HA state * transition. * * @param t exception which warrants the shutdown. Printed to the NN log * before exit. * @throws ExitException thrown only for testing. */ protected synchronized void doImmediateShutdown(Throwable t) throws ExitException { String message = "Error encountered requiring NN shutdown. " + "Shutting down immediately."; try { LOG.error(message, t); } catch (Throwable ignored) { // This is unlikely to happen, but there's nothing we can do if it does. } terminate(1, t); } /** * Class used to expose {@link NameNode} as context to {@link HAState} */ protected class NameNodeHAContext implements HAContext { @Override public void setState(HAState s) { state = s; } @Override public HAState getState() { return state; } @Override public void startActiveServices() throws IOException { try { namesystem.startActiveServices(); startTrashEmptier(getConf()); } catch (Throwable t) { doImmediateShutdown(t); } } @Override public void stopActiveServices() throws IOException { try { if (namesystem != null) { namesystem.stopActiveServices(); } stopTrashEmptier(); } catch (Throwable t) { doImmediateShutdown(t); } } @Override public void startStandbyServices() throws IOException { try { namesystem.startStandbyServices(getConf()); } catch (Throwable t) { doImmediateShutdown(t); } } @Override public void prepareToStopStandbyServices() throws ServiceFailedException { try { namesystem.prepareToStopStandbyServices(); } catch (Throwable t) { doImmediateShutdown(t); } } @Override public void stopStandbyServices() throws IOException { try { if (namesystem != null) { namesystem.stopStandbyServices(); } } catch (Throwable t) { doImmediateShutdown(t); } } @Override public void writeLock() { namesystem.writeLock(); namesystem.lockRetryCache(); } @Override public void writeUnlock() { namesystem.unlockRetryCache(); namesystem.writeUnlock(); } /** Check if an operation of given category is allowed */ @Override public void checkOperation(final OperationCategory op) throws StandbyException { state.checkOperation(haContext, op); } @Override public boolean allowStaleReads() { return allowStaleStandbyReads; } } public boolean isStandbyState() { return (state.equals(STANDBY_STATE)); } public boolean isActiveState() { return (state.equals(ACTIVE_STATE)); } /** * Returns whether the NameNode is completely started */ boolean isStarted() { return this.started.get(); } /** * Check that a request to change this node's HA state is valid. * In particular, verifies that, if auto failover is enabled, non-forced * requests from the HAAdmin CLI are rejected, and vice versa. * * @param req the request to check * @throws AccessControlException if the request is disallowed */ void checkHaStateChange(StateChangeRequestInfo req) throws AccessControlException { boolean autoHaEnabled = getConf().getBoolean(DFS_HA_AUTO_FAILOVER_ENABLED_KEY, DFS_HA_AUTO_FAILOVER_ENABLED_DEFAULT); switch (req.getSource()) { case REQUEST_BY_USER: if (autoHaEnabled) { throw new AccessControlException( "Manual HA control for this NameNode is disallowed, because " + "automatic HA is enabled."); } break; case REQUEST_BY_USER_FORCED: if (autoHaEnabled) { LOG.warn("Allowing manual HA control from " + Server.getRemoteAddress() + " even though automatic HA is enabled, because the user " + "specified the force flag"); } break; case REQUEST_BY_ZKFC: if (!autoHaEnabled) { throw new AccessControlException("Request from ZK failover controller at " + Server.getRemoteAddress() + " denied since automatic HA " + "is not enabled"); } break; } } /* * {@inheritDoc} * */ @Override // ReconfigurableBase public Collection<String> getReconfigurableProperties() { return reconfigurableProperties; } /* * {@inheritDoc} * */ @Override // ReconfigurableBase protected String reconfigurePropertyImpl(String property, String newVal) throws ReconfigurationException { final DatanodeManager datanodeManager = namesystem.getBlockManager().getDatanodeManager(); if (property.equals(DFS_HEARTBEAT_INTERVAL_KEY)) { return reconfHeartbeatInterval(datanodeManager, property, newVal); } else if (property.equals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY)) { return reconfHeartbeatRecheckInterval(datanodeManager, property, newVal); } else if (property.equals(FS_PROTECTED_DIRECTORIES)) { return reconfProtectedDirectories(newVal); } else if (property.equals(HADOOP_CALLER_CONTEXT_ENABLED_KEY)) { return reconfCallerContextEnabled(newVal); } else if (property.equals(ipcClientRPCBackoffEnable)) { return reconfigureIPCBackoffEnabled(newVal); } else if (property.equals(DFS_STORAGE_POLICY_SATISFIER_MODE_KEY)) { return reconfigureSPSModeEvent(newVal, property); } else { throw new ReconfigurationException(property, newVal, getConf().get(property)); } } private String reconfHeartbeatInterval(final DatanodeManager datanodeManager, final String property, final String newVal) throws ReconfigurationException { namesystem.writeLock(); try { if (newVal == null) { // set to default datanodeManager.setHeartbeatInterval(DFS_HEARTBEAT_INTERVAL_DEFAULT); return String.valueOf(DFS_HEARTBEAT_INTERVAL_DEFAULT); } else { long newInterval = getConf().getTimeDurationHelper(DFS_HEARTBEAT_INTERVAL_KEY, newVal, TimeUnit.SECONDS); datanodeManager.setHeartbeatInterval(newInterval); return String.valueOf(datanodeManager.getHeartbeatInterval()); } } catch (NumberFormatException nfe) { throw new ReconfigurationException(property, newVal, getConf().get(property), nfe); } finally { namesystem.writeUnlock(); LOG.info("RECONFIGURE* changed heartbeatInterval to " + datanodeManager.getHeartbeatInterval()); } } private String reconfHeartbeatRecheckInterval(final DatanodeManager datanodeManager, final String property, final String newVal) throws ReconfigurationException { namesystem.writeLock(); try { if (newVal == null) { // set to default datanodeManager.setHeartbeatRecheckInterval(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); return String.valueOf(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); } else { datanodeManager.setHeartbeatRecheckInterval(Integer.parseInt(newVal)); return String.valueOf(datanodeManager.getHeartbeatRecheckInterval()); } } catch (NumberFormatException nfe) { throw new ReconfigurationException(property, newVal, getConf().get(property), nfe); } finally { namesystem.writeUnlock(); LOG.info("RECONFIGURE* changed heartbeatRecheckInterval to " + datanodeManager.getHeartbeatRecheckInterval()); } } private String reconfProtectedDirectories(String newVal) { return getNamesystem().getFSDirectory().setProtectedDirectories(newVal); } private String reconfCallerContextEnabled(String newVal) { Boolean callerContextEnabled; if (newVal == null) { callerContextEnabled = HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT; } else { callerContextEnabled = Boolean.parseBoolean(newVal); } namesystem.setCallerContextEnabled(callerContextEnabled); return Boolean.toString(callerContextEnabled); } String reconfigureIPCBackoffEnabled(String newVal) { boolean clientBackoffEnabled; if (newVal == null) { clientBackoffEnabled = IPC_BACKOFF_ENABLE_DEFAULT; } else { clientBackoffEnabled = Boolean.parseBoolean(newVal); } rpcServer.getClientRpcServer().setClientBackoffEnabled(clientBackoffEnabled); return Boolean.toString(clientBackoffEnabled); } String reconfigureSPSModeEvent(String newVal, String property) throws ReconfigurationException { if (newVal == null || StoragePolicySatisfierMode.fromString(newVal) == null) { throw new ReconfigurationException(property, newVal, getConf().get(property), new HadoopIllegalArgumentException("For enabling or disabling storage policy satisfier, must " + "pass either internal/external/none string value only")); } if (!isActiveState()) { throw new ReconfigurationException(property, newVal, getConf().get(property), new HadoopIllegalArgumentException("Enabling or disabling storage policy satisfier service on " + state + " NameNode is not allowed")); } StoragePolicySatisfierMode mode = StoragePolicySatisfierMode.fromString(newVal); if (mode == StoragePolicySatisfierMode.NONE) { // disabling sps service if (namesystem.getBlockManager().getSPSManager() != null) { namesystem.getBlockManager().getSPSManager().changeModeEvent(mode); namesystem.getBlockManager().disableSPS(); } } else { // enabling sps service boolean spsCreated = (namesystem.getBlockManager().getSPSManager() != null); if (!spsCreated) { spsCreated = namesystem.getBlockManager().createSPSManager(getConf(), newVal); } if (spsCreated) { namesystem.getBlockManager().getSPSManager().changeModeEvent(mode); } } return newVal; } @Override // ReconfigurableBase protected Configuration getNewConf() { return new HdfsConfiguration(); } }