Example usage for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR

List of usage examples for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.hdfs.*
  5. MiniDFSCluster
  6. HDFS_MINIDFS_BASEDIR

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR.

Prototype

String HDFS_MINIDFS_BASEDIR

To view the source code for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR.

Click Source Link

Document

Configuration option to set the data dir:

Usage

From source file:org.apache.tez.history.TestATSFileParser.java

License:Apache License

@BeforeClass
public static void setupCluster() throws Exception {
    conf = new Configuration();
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_NOEDITLOGCHANNELFLUSH, false);
    EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
    miniDFSCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
    fs = miniDFSCluster.getFileSystem();
    conf.set("fs.defaultFS", fs.getUri().toString());

    setupTezCluster();//from w ww . j av a  2s .c  om
}

From source file:org.apache.tez.mapreduce.hadoop.TestMRInputHelpers.java

License:Apache License

@BeforeClass
public static void setup() throws IOException {
    try {/*  ww  w . ja  va2 s.  c om*/
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }

    Configuration testConf = new YarnConfiguration(dfsCluster.getFileSystem().getConf());

    File testConfFile = new File(TEST_ROOT_DIR, "test.xml");
    try {
        testConfFile.createNewFile();
        testConf.writeXml(new FileOutputStream(testConfFile));
        testConfFile.deleteOnExit();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        throw new RuntimeException(e);
    }

    remoteFs.mkdirs(new Path("/tmp/input/"));
    remoteFs.mkdirs(new Path("/tmp/splitsDirNew/"));
    remoteFs.mkdirs(new Path("/tmp/splitsDirOld/"));
    testFilePath = remoteFs.makeQualified(new Path("/tmp/input/test.xml"));
    remoteFs.copyFromLocalFile(new Path(testConfFile.getAbsolutePath()), testFilePath);
    FileStatus fsStatus = remoteFs.getFileStatus(testFilePath);
    Assert.assertTrue(fsStatus.getLen() > 0);

    oldSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirOld/"));
    newSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirNew/"));
}

From source file:org.apache.tez.mapreduce.TestMRRJobs.java

License:Apache License

@BeforeClass
public static void setup() throws IOException {
    try {/*from   ww w  .  j  a v a  2s  .c  o m*/
        conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }

    if (!(new File(MiniTezCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniTezCluster.APPJAR + " not found. Not running test.");
        return;
    }

    if (mrrTezCluster == null) {
        mrrTezCluster = new MiniTezCluster(TestMRRJobs.class.getName(), 1, 1, 1);
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
        conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir");
        conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l);
        mrrTezCluster.init(conf);
        mrrTezCluster.start();
    }

}

From source file:org.apache.tez.mapreduce.TestMRRJobsDAGApi.java

License:Apache License

@BeforeClass
public static void setup() throws IOException {
    try {//from  w w  w  .j a  v  a2  s  .  c om
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }

    if (mrrTezCluster == null) {
        mrrTezCluster = new MiniTezCluster(TestMRRJobsDAGApi.class.getName(), 1, 1, 1);
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
        conf.setInt("yarn.nodemanager.delete.debug-delay-sec", 20000);
        mrrTezCluster.init(conf);
        mrrTezCluster.start();
    }

}

From source file:org.apache.tez.runtime.metrics.TestFileSystemStatisticUpdater2.java

License:Apache License

@BeforeClass
public static void setup() throws IOException {
    try {//from  www.  jav  a  2s .  com
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }
}

From source file:org.apache.tez.test.TestAMRecovery.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    LOG.info("Starting mini clusters");
    try {/*from  w ww  .jav  a2s .  c  o  m*/
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }
    if (miniTezCluster == null) {
        miniTezCluster = new MiniTezCluster(TestAMRecovery.class.getName(), 1, 1, 1);
        Configuration miniTezconf = new Configuration(conf);
        miniTezconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, MAX_AM_ATTEMPT);
        miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
        miniTezCluster.init(miniTezconf);
        miniTezCluster.start();
    }
}

From source file:org.apache.tez.test.TestDAGRecovery.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    LOG.info("Starting mini clusters");
    try {//from  w w  w  .ja v  a  2  s.  co m
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }
    if (miniTezCluster == null) {
        miniTezCluster = new MiniTezCluster(TestDAGRecovery.class.getName(), 1, 1, 1);
        Configuration miniTezconf = new Configuration(conf);
        miniTezconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 4);
        miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
        miniTezCluster.init(miniTezconf);
        miniTezCluster.start();
    }
}

From source file:org.apache.tez.test.TestDAGRecovery2.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    LOG.info("Starting mini clusters");
    try {//from  www.  j  a  v a 2s . co  m
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }
    if (miniTezCluster == null) {
        miniTezCluster = new MiniTezCluster(TestDAGRecovery2.class.getName(), 1, 1, 1);
        Configuration miniTezconf = new Configuration(conf);
        miniTezconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 4);
        miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
        miniTezCluster.init(miniTezconf);
        miniTezCluster.start();
    }
}

From source file:org.apache.tez.test.TestExceptionPropagation.java

License:Apache License

private void startMiniTezCluster() {
    LOG.info("Starting mini clusters");
    try {//  w w w. ja  v  a2  s  .  c  o  m
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }
    miniTezCluster = new MiniTezCluster(TestExceptionPropagation.class.getName(), 1, 1, 1);
    Configuration miniTezconf = new Configuration(conf);
    miniTezconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 4);
    miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
    miniTezCluster.init(miniTezconf);
    miniTezCluster.start();
}

From source file:org.apache.tez.test.TestFaultTolerance.java

License:Apache License

@BeforeClass
public static void setup() throws Exception {
    LOG.info("Starting mini clusters");
    FileSystem remoteFs = null;/*from w w  w.j  a  v a2 s .  c  o m*/
    try {
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }
    if (miniTezCluster == null) {
        miniTezCluster = new MiniTezCluster(TestFaultTolerance.class.getName(), 4, 1, 1);
        Configuration miniTezconf = new Configuration(conf);
        miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
        miniTezCluster.init(miniTezconf);
        miniTezCluster.start();

        Path remoteStagingDir = remoteFs
                .makeQualified(new Path(TEST_ROOT_DIR, String.valueOf(new Random().nextInt(100000))));
        TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

        TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
        tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);

        tezSession = TezClient.create("TestFaultTolerance", tezConf, true);
        tezSession.start();
    }
}