Example usage for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR

List of usage examples for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.hdfs.*
  5. MiniDFSCluster
  6. HDFS_MINIDFS_BASEDIR

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR.

Prototype

String HDFS_MINIDFS_BASEDIR

To view the source code for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR.

Click Source Link

Document

Configuration option to set the data dir:

Usage

From source file:org.apache.tez.test.TestPipelinedShuffle.java

License:Apache License

@BeforeClass
public static void setupDFSCluster() throws Exception {
    conf = new Configuration();
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH, false);
    EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
    miniDFSCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
    fs = miniDFSCluster.getFileSystem();
    conf.set("fs.defaultFS", fs.getUri().toString());
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, false);
}

From source file:org.apache.tez.test.TestRecovery.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    LOG.info("Starting mini clusters");
    try {//from   w ww .  j  a v a 2 s.c  o m
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }
    if (miniTezCluster == null) {
        miniTezCluster = new MiniTezCluster(TestRecovery.class.getName(), 1, 1, 1);
        Configuration miniTezconf = new Configuration(conf);
        miniTezconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 4);
        miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
        miniTezCluster.init(miniTezconf);
        miniTezCluster.start();
    }
}

From source file:org.apache.tez.test.TestSecureShuffle.java

License:Apache License

@BeforeClass
public static void setupDFSCluster() throws Exception {
    conf = new Configuration();
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH, false);
    EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
    miniDFSCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
    fs = miniDFSCluster.getFileSystem();
    conf.set("fs.defaultFS", fs.getUri().toString());
}

From source file:org.apache.tez.test.TestTezJobs.java

License:Apache License

@BeforeClass
public static void setup() throws IOException {
    localFs = FileSystem.getLocal(conf);
    try {/*from   ww w  .  java 2  s. co  m*/
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }

    if (mrrTezCluster == null) {
        mrrTezCluster = new MiniTezCluster(TestTezJobs.class.getName(), 1, 1, 1);
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
        mrrTezCluster.init(conf);
        mrrTezCluster.start();
    }

}

From source file:org.apache.tez.tests.ExternalTezServiceTestHelper.java

License:Apache License

/**
 * Current usage: Create. setupSharedTezClient - during setup (beforeClass). Invoke tearDownAll when done (afterClass)
 * Alternately tearDown the sharedTezClient independently
 *//*from w  ww  .  j  a v a 2 s  . com*/
public ExternalTezServiceTestHelper(String testRootDir) throws IOException {
    try {
        clusterConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testRootDir);
        dfsCluster = new MiniDFSCluster.Builder(clusterConf).numDataNodes(1).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
        LOG.info("MiniDFSCluster started");
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }

    tezCluster = new MiniTezCluster(TestExternalTezServices.class.getName(), 1, 1, 1);
    Configuration conf = new Configuration();
    conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
    tezCluster.init(conf);
    tezCluster.start();
    LOG.info("MiniTezCluster started");

    clusterConf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
    for (Map.Entry<String, String> entry : tezCluster.getConfig()) {
        clusterConf.set(entry.getKey(), entry.getValue());
    }
    long jvmMax = Runtime.getRuntime().maxMemory();

    tezTestServiceCluster = MiniTezTestServiceCluster.create(TestExternalTezServices.class.getSimpleName(), 3,
            ((long) (jvmMax * 0.5d)), 1);
    tezTestServiceCluster.init(clusterConf);
    tezTestServiceCluster.start();
    LOG.info("MiniTezTestServer started");

    confForJobs = new Configuration(clusterConf);
    for (Map.Entry<String, String> entry : tezTestServiceCluster.getClusterSpecificConfiguration()) {
        confForJobs.set(entry.getKey(), entry.getValue());
    }

    Path stagingDirPath = new Path("/tmp/tez-staging-dir");
    remoteFs.mkdirs(stagingDirPath);
    // This is currently configured to push tasks into the Service, and then use the standard RPC
    confForJobs.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    confForJobs.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, false);
}

From source file:org.apache.twill.filesystem.FileContextLocationTest.java

License:Apache License

@BeforeClass
public static void init() throws IOException {
    Configuration conf = new Configuration();
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
}

From source file:org.apache.twill.yarn.TwillTester.java

License:Apache License

@Override
protected void before() throws Throwable {
    tmpFolder.create();/* w w  w.  j  a va  2 s. co  m*/

    // Starts Zookeeper
    zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
    zkServer.startAndWait();

    // Start YARN mini cluster
    File miniDFSDir = tmpFolder.newFolder();
    LOG.info("Starting Mini DFS on path {}", miniDFSDir);
    Configuration fsConf = new HdfsConfiguration(new Configuration());
    fsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, miniDFSDir.getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(fsConf).numDataNodes(1).build();

    Configuration conf = new YarnConfiguration(dfsCluster.getFileSystem().getConf());

    if (YarnUtils.getHadoopVersion().equals(YarnUtils.HadoopVersions.HADOOP_20)) {
        conf.set("yarn.resourcemanager.scheduler.class",
                "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler");
    } else {
        conf.set("yarn.resourcemanager.scheduler.class",
                "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
        conf.set("yarn.scheduler.capacity.resource-calculator",
                "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
        conf.setBoolean("yarn.scheduler.include-port-in-node-name", true);
    }
    conf.set("yarn.nodemanager.vmem-pmem-ratio", "20.1");
    conf.set("yarn.nodemanager.vmem-check-enabled", "false");
    conf.set("yarn.scheduler.minimum-allocation-mb", "128");
    conf.set("yarn.nodemanager.delete.debug-delay-sec", "3600");

    cluster = new MiniYARNCluster("test-cluster", 3, 1, 1);
    cluster.init(conf);
    cluster.start();

    this.config = new YarnConfiguration(cluster.getConfig());

    twillRunner = createTwillRunnerService();
    twillRunner.start();

    yarnAppClient = new VersionDetectYarnAppClientFactory().create(conf);
    yarnAppClient.startAndWait();
}

From source file:org.apache.twill.yarn.YarnTestUtils.java

License:Apache License

private static final void init(File folder) throws IOException {
    // Starts Zookeeper
    zkServer = InMemoryZKServer.builder().build();
    zkServer.startAndWait();//from  w w  w .  j  av  a 2s.co  m

    // Start YARN mini cluster
    LOG.info("Starting Mini DFS on path {}", folder);
    Configuration fsConf = new HdfsConfiguration(new Configuration());
    fsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, folder.getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(fsConf).numDataNodes(1).build();

    Configuration conf = new YarnConfiguration(dfsCluster.getFileSystem().getConf());

    if (YarnUtils.getHadoopVersion().equals(YarnUtils.HadoopVersions.HADOOP_20)) {
        conf.set("yarn.resourcemanager.scheduler.class",
                "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler");
    } else {
        conf.set("yarn.resourcemanager.scheduler.class",
                "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
        conf.set("yarn.scheduler.capacity.resource-calculator",
                "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
        conf.setBoolean("yarn.scheduler.include-port-in-node-name", true);
    }
    conf.set("yarn.nodemanager.vmem-pmem-ratio", "20.1");
    conf.set("yarn.nodemanager.vmem-check-enabled", "false");
    conf.set("yarn.scheduler.minimum-allocation-mb", "128");
    conf.set("yarn.nodemanager.delete.debug-delay-sec", "3600");

    cluster = new MiniYARNCluster("test-cluster", 3, 1, 1);
    cluster.init(conf);
    cluster.start();

    config = new YarnConfiguration(cluster.getConfig());

    runnerService = createTwillRunnerService();
    runnerService.startAndWait();

    yarnAppClient = new VersionDetectYarnAppClientFactory().create(conf);
    yarnAppClient.start();
}

From source file:org.dkpro.bigdata.io.hadoop.HdfsResourceLoaderLocatorTest.java

License:Apache License

@Before
public void startCluster() throws Exception {
    // Start dummy HDFS
    File target = folder.newFolder("hdfs");
    hadoopTmp = folder.newFolder("hadoop");

    File baseDir = new File(target, "hdfs").getAbsoluteFile();
    FileUtil.fullyDelete(baseDir);/* w ww. ja  v  a 2  s  .co  m*/
    Configuration conf = new Configuration();
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    conf.set("hadoop.tmp.dir", hadoopTmp.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
    hdfsCluster = builder.build();
    hdfsCluster.waitActive();
}

From source file:org.elasticsearch.hadoop.integration.yarn.YarnTestCluster.java

License:Apache License

@Override
protected void before() throws Throwable {
    if (yarnCluster != null) {
        return;/*  ww w . j  a  v a 2 s. c  o m*/
    }

    System.out.println("Starting YARN cluster...");

    //tempDir = System.getProperty("java.io.tmpdir");
    //System.setProperty("java.io.tmpdir", "/tmp-yarn");

    //        FileContext.DEFAULT_PERM.fromShort((short) 777);
    //        FileContext.DIR_DEFAULT_PERM.fromShort((short) 777);
    //        FileContext.FILE_DEFAULT_PERM.fromShort((short) 777);
    //ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION.fromShort((short) 777);

    // make sure to use IP discovery
    Configuration.addDefaultResource("yarn-test-site.xml");

    cfg.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "build/es-yarn/" + CLUSTER_NAME + "-dfs");

    dfsCluster = new MiniDFSCluster.Builder(cfg).numDataNodes(nodes).build();
    System.out.println("Started DFS cluster...");

    cfg.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    yarnCluster = new MiniYARNCluster(CLUSTER_NAME, nodes, 1, 1);
    yarnCluster.init(cfg);
    yarnCluster.start();
    System.out.println("Started YARN cluster...");
}