List of usage examples for org.apache.hadoop.hdfs.server.namenode FSNamesystem getNamespaceDirs
public static Collection<URI> getNamespaceDirs(Configuration conf)
From source file:com.mellanox.r4h.MiniDFSCluster.java
License:Apache License
private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy, boolean format, StartupOption operation, String clusterId, Configuration conf) throws IOException { Preconditions.checkArgument(nnTopology.countNameNodes() > 0, "empty NN topology: no namenodes specified!"); if (!federation && nnTopology.countNameNodes() == 1) { NNConf onlyNN = nnTopology.getOnlyNameNode(); // we only had one NN, set DEFAULT_NAME for it. If not explicitly // specified initially, the port will be 0 to make NN bind to any // available port. It will be set to the right address after // NN is started. conf.set(FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:" + MiniDFSClusterBridge.getNNConf_ipcPort(onlyNN)); }//from w w w . ja v a 2 s . c om List<String> allNsIds = Lists.newArrayList(); for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) { if (nameservice.getId() != null) { allNsIds.add(nameservice.getId()); } } if (!allNsIds.isEmpty()) { conf.set(DFS_NAMESERVICES, Joiner.on(",").join(allNsIds)); } int nnCounter = 0; for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) { String nsId = nameservice.getId(); String lastDefaultFileSystem = null; Preconditions.checkArgument(!federation || nsId != null, "if there is more than one NS, they must have names"); // First set up the configuration which all of the NNs // need to have - have to do this a priori before starting // *any* of the NNs, so they know to come up in standby. List<String> nnIds = Lists.newArrayList(); // Iterate over the NNs in this nameservice for (NNConf nn : nameservice.getNNs()) { nnIds.add(MiniDFSClusterBridge.getNNConf_nnId(nn)); initNameNodeAddress(conf, nameservice.getId(), nn); } // If HA is enabled on this nameservice, enumerate all the namenodes // in the configuration. Also need to set a shared edits dir if (nnIds.size() > 1) { conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, nameservice.getId()), Joiner.on(",").join(nnIds)); if (manageNameDfsSharedDirs) { URI sharedEditsUri = getSharedEditsDir(nnCounter, nnCounter + nnIds.size() - 1); conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, sharedEditsUri.toString()); // Clean out the shared edits dir completely, including all subdirectories. FileUtil.fullyDelete(new File(sharedEditsUri)); } } // Now format first NN and copy the storage directory from that node to the others. int i = 0; Collection<URI> prevNNDirs = null; int nnCounterForFormat = nnCounter; for (NNConf nn : nameservice.getNNs()) { initNameNodeConf(conf, nsId, MiniDFSClusterBridge.getNNConf_nnId(nn), manageNameDfsDirs, enableManagedDfsDirsRedundancy, nnCounterForFormat); Collection<URI> namespaceDirs = FSNamesystem.getNamespaceDirs(conf); if (format) { for (URI nameDirUri : namespaceDirs) { File nameDir = new File(nameDirUri); if (nameDir.exists() && !FileUtil.fullyDelete(nameDir)) { throw new IOException("Could not fully delete " + nameDir); } } Collection<URI> checkpointDirs = Util.stringCollectionAsURIs( conf.getTrimmedStringCollection(DFS_NAMENODE_CHECKPOINT_DIR_KEY)); for (URI checkpointDirUri : checkpointDirs) { File checkpointDir = new File(checkpointDirUri); if (checkpointDir.exists() && !FileUtil.fullyDelete(checkpointDir)) { throw new IOException("Could not fully delete " + checkpointDir); } } } boolean formatThisOne = format; if (format && i++ > 0) { // Don't format the second NN in an HA setup - that // would result in it having a different clusterID, // block pool ID, etc. Instead, copy the name dirs // from the first one. formatThisOne = false; assert (null != prevNNDirs); copyNameDirs(prevNNDirs, namespaceDirs, conf); } nnCounterForFormat++; if (formatThisOne) { // Allow overriding clusterID for specific NNs to test // misconfiguration. if (MiniDFSClusterBridge.getNNConf_cliusterId(nn) == null) { StartupOption.FORMAT.setClusterId(clusterId); } else { StartupOption.FORMAT.setClusterId(MiniDFSClusterBridge.getNNConf_cliusterId(nn)); } DFSTestUtil.formatNameNode(conf); } prevNNDirs = namespaceDirs; } // Start all Namenodes for (NNConf nn : nameservice.getNNs()) { initNameNodeConf(conf, nsId, MiniDFSClusterBridge.getNNConf_nnId(nn), manageNameDfsDirs, enableManagedDfsDirsRedundancy, nnCounter); createNameNode(nnCounter, conf, numDataNodes, false, operation, clusterId, nsId, MiniDFSClusterBridge.getNNConf_nnId(nn)); // Record the last namenode uri if (nameNodes[nnCounter] != null && nameNodes[nnCounter].conf != null) { lastDefaultFileSystem = nameNodes[nnCounter].conf.get(FS_DEFAULT_NAME_KEY); } nnCounter++; } if (!federation && lastDefaultFileSystem != null) { // Set the default file system to the actual bind address of NN. conf.set(FS_DEFAULT_NAME_KEY, lastDefaultFileSystem); } } }
From source file:com.mellanox.r4h.MiniDFSCluster.java
License:Apache License
/** * Get the directories where the namenode stores its image. */// w ww . j a v a2 s .c om public Collection<URI> getNameDirs(int nnIndex) { return FSNamesystem.getNamespaceDirs(nameNodes[nnIndex].conf); }
From source file:common.NameNode.java
License:Apache License
/** * Verify that configured directories exist, then * Interactively confirm that formatting is desired * for each existing directory and format them. * /*from w ww . j av a 2 s .c o m*/ * @param conf * @param isConfirmationNeeded * @return true if formatting was aborted, false otherwise * @throws IOException */ private static boolean format(Configuration conf, boolean isConfirmationNeeded) throws IOException { Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(conf); Collection<URI> editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf); for (Iterator<URI> it = dirsToFormat.iterator(); it.hasNext();) { File curDir = new File(it.next().getPath()); if (!curDir.exists()) continue; if (isConfirmationNeeded) { System.err.print("Re-format filesystem in " + curDir + " ? (Y or N) "); if (!(System.in.read() == 'Y')) { System.err.println("Format aborted in " + curDir); return true; } while (System.in.read() != '\n') ; // discard the enter-key } } FSNamesystem nsys = new FSNamesystem(new FSImage(dirsToFormat, editDirsToFormat), conf); nsys.dir.fsImage.format(); return false; }
From source file:common.NameNode.java
License:Apache License
private static boolean finalize(Configuration conf, boolean isConfirmationNeeded) throws IOException { Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(conf); Collection<URI> editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf); FSNamesystem nsys = new FSNamesystem(new FSImage(dirsToFormat, editDirsToFormat), conf); System.err.print("\"finalize\" will remove the previous state of the files system.\n" + "Recent upgrade will become permanent.\n" + "Rollback option will not be available anymore.\n"); if (isConfirmationNeeded) { System.err.print("Finalize filesystem state ? (Y or N) "); if (!(System.in.read() == 'Y')) { System.err.println("Finalize aborted."); return true; }/*from w ww. j av a 2 s. c o m*/ while (System.in.read() != '\n') ; // discard the enter-key } nsys.dir.fsImage.finalizeUpgrade(); return false; }
From source file:io.fabric8.hadoop.commands.NameNodeFormat.java
License:Apache License
@Override protected void doExecute(Configuration conf) throws Exception { Collection<File> dirsToFormat = FSNamesystem.getNamespaceDirs(conf); for (Iterator<File> it = dirsToFormat.iterator(); it.hasNext();) { File curDir = it.next();//from ww w. ja v a2 s .com if (!curDir.exists()) { continue; } if (!force) { System.err.print("Re-format filesystem in " + curDir + " ? (Y or N) "); System.err.flush(); if (!(System.in.read() == 'Y')) { System.err.println("Format aborted in " + curDir); return; } while (System.in.read() != '\n') ; // discard the enter-key } } NameNode.format(conf); }
From source file:io.fabric8.hadoop.hdfs.NameNodeFactory.java
License:Apache License
@Override protected NameNode doCreate(Dictionary properties) throws Exception { Configuration conf = new Configuration(); for (Enumeration e = properties.keys(); e.hasMoreElements();) { Object key = e.nextElement(); Object val = properties.get(key); conf.set(key.toString(), val.toString()); }/* w w w . ja v a2s.c o m*/ boolean exists = false; for (File file : FSNamesystem.getNamespaceDirs(conf)) { exists |= file.exists(); } if (!exists) { NameNode.format(conf); } NameNode nameNode = NameNode.createNameNode(null, conf); return nameNode; }