Example usage for org.apache.hadoop.hdfs MiniDFSNNTopology countNameNodes

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs MiniDFSNNTopology countNameNodes.

Prototype

public int countNameNodes() 

Source Link

Usage

From source file:com.mellanox.r4h.MiniDFSCluster.java

License:Apache License

private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, boolean manageNameDfsDirs,
        boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy, boolean format,
        StartupOption operation, String clusterId, Configuration conf) throws IOException {
    Preconditions.checkArgument(nnTopology.countNameNodes() > 0, "empty NN topology: no namenodes specified!");

    if (!federation && nnTopology.countNameNodes() == 1) {
        NNConf onlyNN = nnTopology.getOnlyNameNode();
        // we only had one NN, set DEFAULT_NAME for it. If not explicitly
        // specified initially, the port will be 0 to make NN bind to any
        // available port. It will be set to the right address after
        // NN is started.
        conf.set(FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:" + MiniDFSClusterBridge.getNNConf_ipcPort(onlyNN));
    }//from  ww w .  j a v  a 2 s  .c o m

    List<String> allNsIds = Lists.newArrayList();
    for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) {
        if (nameservice.getId() != null) {
            allNsIds.add(nameservice.getId());
        }
    }
    if (!allNsIds.isEmpty()) {
        conf.set(DFS_NAMESERVICES, Joiner.on(",").join(allNsIds));
    }

    int nnCounter = 0;
    for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) {
        String nsId = nameservice.getId();
        String lastDefaultFileSystem = null;

        Preconditions.checkArgument(!federation || nsId != null,
                "if there is more than one NS, they must have names");

        // First set up the configuration which all of the NNs
        // need to have - have to do this a priori before starting
        // *any* of the NNs, so they know to come up in standby.
        List<String> nnIds = Lists.newArrayList();
        // Iterate over the NNs in this nameservice
        for (NNConf nn : nameservice.getNNs()) {
            nnIds.add(MiniDFSClusterBridge.getNNConf_nnId(nn));

            initNameNodeAddress(conf, nameservice.getId(), nn);
        }

        // If HA is enabled on this nameservice, enumerate all the namenodes
        // in the configuration. Also need to set a shared edits dir
        if (nnIds.size() > 1) {
            conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, nameservice.getId()),
                    Joiner.on(",").join(nnIds));
            if (manageNameDfsSharedDirs) {
                URI sharedEditsUri = getSharedEditsDir(nnCounter, nnCounter + nnIds.size() - 1);
                conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, sharedEditsUri.toString());
                // Clean out the shared edits dir completely, including all subdirectories.
                FileUtil.fullyDelete(new File(sharedEditsUri));
            }
        }

        // Now format first NN and copy the storage directory from that node to the others.
        int i = 0;
        Collection<URI> prevNNDirs = null;
        int nnCounterForFormat = nnCounter;
        for (NNConf nn : nameservice.getNNs()) {
            initNameNodeConf(conf, nsId, MiniDFSClusterBridge.getNNConf_nnId(nn), manageNameDfsDirs,
                    enableManagedDfsDirsRedundancy, nnCounterForFormat);
            Collection<URI> namespaceDirs = FSNamesystem.getNamespaceDirs(conf);
            if (format) {
                for (URI nameDirUri : namespaceDirs) {
                    File nameDir = new File(nameDirUri);
                    if (nameDir.exists() && !FileUtil.fullyDelete(nameDir)) {
                        throw new IOException("Could not fully delete " + nameDir);
                    }
                }
                Collection<URI> checkpointDirs = Util.stringCollectionAsURIs(
                        conf.getTrimmedStringCollection(DFS_NAMENODE_CHECKPOINT_DIR_KEY));
                for (URI checkpointDirUri : checkpointDirs) {
                    File checkpointDir = new File(checkpointDirUri);
                    if (checkpointDir.exists() && !FileUtil.fullyDelete(checkpointDir)) {
                        throw new IOException("Could not fully delete " + checkpointDir);
                    }
                }
            }

            boolean formatThisOne = format;
            if (format && i++ > 0) {
                // Don't format the second NN in an HA setup - that
                // would result in it having a different clusterID,
                // block pool ID, etc. Instead, copy the name dirs
                // from the first one.
                formatThisOne = false;
                assert (null != prevNNDirs);
                copyNameDirs(prevNNDirs, namespaceDirs, conf);
            }

            nnCounterForFormat++;
            if (formatThisOne) {
                // Allow overriding clusterID for specific NNs to test
                // misconfiguration.
                if (MiniDFSClusterBridge.getNNConf_cliusterId(nn) == null) {
                    StartupOption.FORMAT.setClusterId(clusterId);
                } else {
                    StartupOption.FORMAT.setClusterId(MiniDFSClusterBridge.getNNConf_cliusterId(nn));
                }
                DFSTestUtil.formatNameNode(conf);
            }
            prevNNDirs = namespaceDirs;
        }

        // Start all Namenodes
        for (NNConf nn : nameservice.getNNs()) {
            initNameNodeConf(conf, nsId, MiniDFSClusterBridge.getNNConf_nnId(nn), manageNameDfsDirs,
                    enableManagedDfsDirsRedundancy, nnCounter);
            createNameNode(nnCounter, conf, numDataNodes, false, operation, clusterId, nsId,
                    MiniDFSClusterBridge.getNNConf_nnId(nn));
            // Record the last namenode uri
            if (nameNodes[nnCounter] != null && nameNodes[nnCounter].conf != null) {
                lastDefaultFileSystem = nameNodes[nnCounter].conf.get(FS_DEFAULT_NAME_KEY);
            }
            nnCounter++;
        }
        if (!federation && lastDefaultFileSystem != null) {
            // Set the default file system to the actual bind address of NN.
            conf.set(FS_DEFAULT_NAME_KEY, lastDefaultFileSystem);
        }
    }

}

---