List of usage examples for org.apache.hadoop.hdfs DFSTestUtil formatNameNode
public static void formatNameNode(Configuration conf) throws IOException
From source file:com.mellanox.r4h.MiniDFSCluster.java
License:Apache License
private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy, boolean format, StartupOption operation, String clusterId, Configuration conf) throws IOException { Preconditions.checkArgument(nnTopology.countNameNodes() > 0, "empty NN topology: no namenodes specified!"); if (!federation && nnTopology.countNameNodes() == 1) { NNConf onlyNN = nnTopology.getOnlyNameNode(); // we only had one NN, set DEFAULT_NAME for it. If not explicitly // specified initially, the port will be 0 to make NN bind to any // available port. It will be set to the right address after // NN is started. conf.set(FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:" + MiniDFSClusterBridge.getNNConf_ipcPort(onlyNN)); }/*from www . j a v a 2 s . c o m*/ List<String> allNsIds = Lists.newArrayList(); for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) { if (nameservice.getId() != null) { allNsIds.add(nameservice.getId()); } } if (!allNsIds.isEmpty()) { conf.set(DFS_NAMESERVICES, Joiner.on(",").join(allNsIds)); } int nnCounter = 0; for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) { String nsId = nameservice.getId(); String lastDefaultFileSystem = null; Preconditions.checkArgument(!federation || nsId != null, "if there is more than one NS, they must have names"); // First set up the configuration which all of the NNs // need to have - have to do this a priori before starting // *any* of the NNs, so they know to come up in standby. List<String> nnIds = Lists.newArrayList(); // Iterate over the NNs in this nameservice for (NNConf nn : nameservice.getNNs()) { nnIds.add(MiniDFSClusterBridge.getNNConf_nnId(nn)); initNameNodeAddress(conf, nameservice.getId(), nn); } // If HA is enabled on this nameservice, enumerate all the namenodes // in the configuration. Also need to set a shared edits dir if (nnIds.size() > 1) { conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, nameservice.getId()), Joiner.on(",").join(nnIds)); if (manageNameDfsSharedDirs) { URI sharedEditsUri = getSharedEditsDir(nnCounter, nnCounter + nnIds.size() - 1); conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, sharedEditsUri.toString()); // Clean out the shared edits dir completely, including all subdirectories. FileUtil.fullyDelete(new File(sharedEditsUri)); } } // Now format first NN and copy the storage directory from that node to the others. int i = 0; Collection<URI> prevNNDirs = null; int nnCounterForFormat = nnCounter; for (NNConf nn : nameservice.getNNs()) { initNameNodeConf(conf, nsId, MiniDFSClusterBridge.getNNConf_nnId(nn), manageNameDfsDirs, enableManagedDfsDirsRedundancy, nnCounterForFormat); Collection<URI> namespaceDirs = FSNamesystem.getNamespaceDirs(conf); if (format) { for (URI nameDirUri : namespaceDirs) { File nameDir = new File(nameDirUri); if (nameDir.exists() && !FileUtil.fullyDelete(nameDir)) { throw new IOException("Could not fully delete " + nameDir); } } Collection<URI> checkpointDirs = Util.stringCollectionAsURIs( conf.getTrimmedStringCollection(DFS_NAMENODE_CHECKPOINT_DIR_KEY)); for (URI checkpointDirUri : checkpointDirs) { File checkpointDir = new File(checkpointDirUri); if (checkpointDir.exists() && !FileUtil.fullyDelete(checkpointDir)) { throw new IOException("Could not fully delete " + checkpointDir); } } } boolean formatThisOne = format; if (format && i++ > 0) { // Don't format the second NN in an HA setup - that // would result in it having a different clusterID, // block pool ID, etc. Instead, copy the name dirs // from the first one. formatThisOne = false; assert (null != prevNNDirs); copyNameDirs(prevNNDirs, namespaceDirs, conf); } nnCounterForFormat++; if (formatThisOne) { // Allow overriding clusterID for specific NNs to test // misconfiguration. if (MiniDFSClusterBridge.getNNConf_cliusterId(nn) == null) { StartupOption.FORMAT.setClusterId(clusterId); } else { StartupOption.FORMAT.setClusterId(MiniDFSClusterBridge.getNNConf_cliusterId(nn)); } DFSTestUtil.formatNameNode(conf); } prevNNDirs = namespaceDirs; } // Start all Namenodes for (NNConf nn : nameservice.getNNs()) { initNameNodeConf(conf, nsId, MiniDFSClusterBridge.getNNConf_nnId(nn), manageNameDfsDirs, enableManagedDfsDirsRedundancy, nnCounter); createNameNode(nnCounter, conf, numDataNodes, false, operation, clusterId, nsId, MiniDFSClusterBridge.getNNConf_nnId(nn)); // Record the last namenode uri if (nameNodes[nnCounter] != null && nameNodes[nnCounter].conf != null) { lastDefaultFileSystem = nameNodes[nnCounter].conf.get(FS_DEFAULT_NAME_KEY); } nnCounter++; } if (!federation && lastDefaultFileSystem != null) { // Set the default file system to the actual bind address of NN. conf.set(FS_DEFAULT_NAME_KEY, lastDefaultFileSystem); } } }
From source file:com.mellanox.r4h.MiniDFSCluster.java
License:Apache License
private void createNameNode(int nnIndex, Configuration conf, int numDataNodes, boolean format, StartupOption operation, String clusterId, String nameserviceId, String nnId) throws IOException { // Format and clean out DataNode directories if (format) { DFSTestUtil.formatNameNode(conf); }/*from w w w . jav a 2 s . c om*/ if (operation == StartupOption.UPGRADE) { operation.setClusterId(clusterId); } // Start the NameNode after saving the default file system. String originalDefaultFs = conf.get(FS_DEFAULT_NAME_KEY); String[] args = createArgs(operation); NameNode nn = NameNode.createNameNode(args, conf); if (operation == StartupOption.RECOVER) { return; } // After the NN has started, set back the bound ports into // the conf conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, nameserviceId, nnId), nn.getNameNodeAddressHostPortString()); if (nn.getHttpAddress() != null) { conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY, nameserviceId, nnId), NetUtils.getHostPortString(nn.getHttpAddress())); } if (nn.getHttpsAddress() != null) { conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTPS_ADDRESS_KEY, nameserviceId, nnId), NetUtils.getHostPortString(nn.getHttpsAddress())); } DFSUtil.setGenericConf(conf, nameserviceId, nnId, DFS_NAMENODE_HTTP_ADDRESS_KEY); nameNodes[nnIndex] = new NameNodeInfo(nn, nameserviceId, nnId, operation, new Configuration(conf)); // Restore the default fs name if (originalDefaultFs == null) { conf.set(FS_DEFAULT_NAME_KEY, ""); } else { conf.set(FS_DEFAULT_NAME_KEY, originalDefaultFs); } }