Example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_NAMENODE_REPLICATION_MIN_KEY

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_NAMENODE_REPLICATION_MIN_KEY.

Prototype

String DFS_NAMENODE_REPLICATION_MIN_KEY

To view the source code for org.apache.hadoop.hdfs DFSConfigKeys DFS_NAMENODE_REPLICATION_MIN_KEY.

Click Source Link

Usage

From source file:org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl.java

License:Apache License

/**
 * @param config/*w w  w. j a va2  s. co  m*/
 *          initial configuration
 */
@SuppressWarnings("deprecation")
public MiniAccumuloClusterImpl(MiniAccumuloConfigImpl config) throws IOException {

    this.config = config.initialize();

    mkdirs(config.getConfDir());
    mkdirs(config.getLogDir());
    mkdirs(config.getLibDir());
    mkdirs(config.getLibExtDir());

    if (!config.useExistingInstance()) {
        if (!config.useExistingZooKeepers())
            mkdirs(config.getZooKeeperDir());
        mkdirs(config.getWalogDir());
        mkdirs(config.getAccumuloDir());
    }

    if (config.useMiniDFS()) {
        File nn = new File(config.getAccumuloDir(), "nn");
        mkdirs(nn);
        File dn = new File(config.getAccumuloDir(), "dn");
        mkdirs(dn);
        File dfs = new File(config.getAccumuloDir(), "dfs");
        mkdirs(dfs);
        Configuration conf = new Configuration();
        conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nn.getAbsolutePath());
        conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dn.getAbsolutePath());
        conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "1");
        conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, "1");
        conf.set("dfs.support.append", "true");
        conf.set("dfs.datanode.synconclose", "true");
        conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission());
        String oldTestBuildData = System.setProperty("test.build.data", dfs.getAbsolutePath());
        miniDFS = new MiniDFSCluster.Builder(conf).build();
        if (oldTestBuildData == null)
            System.clearProperty("test.build.data");
        else
            System.setProperty("test.build.data", oldTestBuildData);
        miniDFS.waitClusterUp();
        InetSocketAddress dfsAddress = miniDFS.getNameNode().getNameNodeAddress();
        dfsUri = "hdfs://" + dfsAddress.getHostName() + ":" + dfsAddress.getPort();
        File coreFile = new File(config.getConfDir(), "core-site.xml");
        writeConfig(coreFile, Collections.singletonMap("fs.default.name", dfsUri).entrySet());
        File hdfsFile = new File(config.getConfDir(), "hdfs-site.xml");
        writeConfig(hdfsFile, conf);

        Map<String, String> siteConfig = config.getSiteConfig();
        siteConfig.put(Property.INSTANCE_DFS_URI.getKey(), dfsUri);
        siteConfig.put(Property.INSTANCE_DFS_DIR.getKey(), "/accumulo");
        config.setSiteConfig(siteConfig);
    } else if (config.useExistingInstance()) {
        dfsUri = CachedConfiguration.getInstance().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY);
    } else {
        dfsUri = "file:///";
    }

    File clientConfFile = config.getClientConfFile();
    // Write only the properties that correspond to ClientConfiguration properties
    writeConfigProperties(clientConfFile, Maps.filterEntries(config.getSiteConfig(),
            v -> ClientConfiguration.ClientProperty.getPropertyByKey(v.getKey()) != null));

    File siteFile = new File(config.getConfDir(), "accumulo-site.xml");
    writeConfig(siteFile, config.getSiteConfig().entrySet());

    if (!config.useExistingInstance() && !config.useExistingZooKeepers()) {
        zooCfgFile = new File(config.getConfDir(), "zoo.cfg");
        FileWriter fileWriter = new FileWriter(zooCfgFile);

        // zookeeper uses Properties to read its config, so use that to write in order to properly escape things like Windows paths
        Properties zooCfg = new Properties();
        zooCfg.setProperty("tickTime", "2000");
        zooCfg.setProperty("initLimit", "10");
        zooCfg.setProperty("syncLimit", "5");
        zooCfg.setProperty("clientPortAddress", "127.0.0.1");
        zooCfg.setProperty("clientPort", config.getZooKeeperPort() + "");
        zooCfg.setProperty("maxClientCnxns", "1000");
        zooCfg.setProperty("dataDir", config.getZooKeeperDir().getAbsolutePath());
        zooCfg.store(fileWriter, null);

        fileWriter.close();
    }

    // disable audit logging for mini....
    InputStream auditStream = this.getClass().getResourceAsStream("/auditLog.xml");

    if (auditStream != null) {
        FileUtils.copyInputStreamToFile(auditStream, new File(config.getConfDir(), "auditLog.xml"));
    }

    clusterControl = new MiniAccumuloClusterControl(this);
}

---

From source file:org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl.java

License:Apache License

/**
 * @param config/*  w w w  . j  a  v  a 2 s . c  om*/
 *          initial configuration
 */
@SuppressWarnings("deprecation")
public MiniAccumuloClusterImpl(MiniAccumuloConfigImpl config) throws IOException {

    this.config = config.initialize();

    mkdirs(config.getConfDir());
    mkdirs(config.getLogDir());
    mkdirs(config.getLibDir());
    mkdirs(config.getLibExtDir());

    if (!config.useExistingInstance()) {
        if (!config.useExistingZooKeepers())
            mkdirs(config.getZooKeeperDir());
        mkdirs(config.getAccumuloDir());
    }

    if (config.useMiniDFS()) {
        File nn = new File(config.getAccumuloDir(), "nn");
        mkdirs(nn);
        File dn = new File(config.getAccumuloDir(), "dn");
        mkdirs(dn);
        File dfs = new File(config.getAccumuloDir(), "dfs");
        mkdirs(dfs);
        Configuration conf = new Configuration();
        conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nn.getAbsolutePath());
        conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dn.getAbsolutePath());
        conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "1");
        conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, "1");
        conf.set("dfs.support.append", "true");
        conf.set("dfs.datanode.synconclose", "true");
        conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission());
        String oldTestBuildData = System.setProperty("test.build.data", dfs.getAbsolutePath());
        miniDFS = new MiniDFSCluster.Builder(conf).build();
        if (oldTestBuildData == null)
            System.clearProperty("test.build.data");
        else
            System.setProperty("test.build.data", oldTestBuildData);
        miniDFS.waitClusterUp();
        InetSocketAddress dfsAddress = miniDFS.getNameNode().getNameNodeAddress();
        dfsUri = "hdfs://" + dfsAddress.getHostName() + ":" + dfsAddress.getPort();
        File coreFile = new File(config.getConfDir(), "core-site.xml");
        writeConfig(coreFile, Collections.singletonMap("fs.default.name", dfsUri).entrySet());
        File hdfsFile = new File(config.getConfDir(), "hdfs-site.xml");
        writeConfig(hdfsFile, conf);

        Map<String, String> siteConfig = config.getSiteConfig();
        siteConfig.put(Property.INSTANCE_DFS_URI.getKey(), dfsUri);
        siteConfig.put(Property.INSTANCE_DFS_DIR.getKey(), "/accumulo");
        config.setSiteConfig(siteConfig);
    } else if (config.useExistingInstance()) {
        dfsUri = config.getHadoopConfiguration().get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY);
    } else {
        dfsUri = "file:///";
    }

    File clientConfFile = config.getClientConfFile();
    // Write only the properties that correspond to ClientConfiguration properties
    writeConfigProperties(clientConfFile,
            Maps.filterEntries(config.getSiteConfig(),
                    v -> org.apache.accumulo.core.client.ClientConfiguration.ClientProperty
                            .getPropertyByKey(v.getKey()) != null));

    Map<String, String> clientProps = config.getClientProps();
    clientProps.put(ClientProperty.INSTANCE_ZOOKEEPERS.getKey(), config.getZooKeepers());
    clientProps.put(ClientProperty.INSTANCE_NAME.getKey(), config.getInstanceName());
    if (!clientProps.containsKey(ClientProperty.AUTH_TYPE.getKey())) {
        clientProps.put(ClientProperty.AUTH_TYPE.getKey(), "password");
        clientProps.put(ClientProperty.AUTH_PRINCIPAL.getKey(), config.getRootUserName());
        clientProps.put(ClientProperty.AUTH_TOKEN.getKey(), config.getRootPassword());
    }

    File clientPropsFile = config.getClientPropsFile();
    writeConfigProperties(clientPropsFile, clientProps);

    File siteFile = new File(config.getConfDir(), "accumulo.properties");
    writeConfigProperties(siteFile, config.getSiteConfig());
    siteConfig = new SiteConfiguration(siteFile);

    if (!config.useExistingInstance() && !config.useExistingZooKeepers()) {
        zooCfgFile = new File(config.getConfDir(), "zoo.cfg");
        FileWriter fileWriter = new FileWriter(zooCfgFile);

        // zookeeper uses Properties to read its config, so use that to write in order to properly
        // escape things like Windows paths
        Properties zooCfg = new Properties();
        zooCfg.setProperty("tickTime", "2000");
        zooCfg.setProperty("initLimit", "10");
        zooCfg.setProperty("syncLimit", "5");
        zooCfg.setProperty("clientPortAddress", "127.0.0.1");
        zooCfg.setProperty("clientPort", config.getZooKeeperPort() + "");
        zooCfg.setProperty("maxClientCnxns", "1000");
        zooCfg.setProperty("dataDir", config.getZooKeeperDir().getAbsolutePath());
        zooCfg.store(fileWriter, null);

        fileWriter.close();
    }
    clusterControl = new MiniAccumuloClusterControl(this);
}

---