Example usage for org.apache.hadoop.hdfs MiniDFSCluster MiniDFSCluster

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs MiniDFSCluster MiniDFSCluster.

Prototype

@Deprecated 
public MiniDFSCluster(Configuration conf, int numDataNodes, boolean format, String[] racks) throws IOException

Source Link

Document

Modify the config and start up the servers.

Usage

From source file:com.trace.hadoop.TestDFSRename.java

License:Apache License

@Override
protected void setUp() throws Exception {
    cluster = new MiniDFSCluster(CONF, 2, true, null);
}

From source file:com.trace.hadoop.TestDFSRename.java

License:Apache License

private void restartCluster() throws IOException {
    if (cluster != null) {
        cluster.shutdown();/*from  w ww. ja  va2 s.  c  om*/
        cluster = null;
    }
    cluster = new MiniDFSCluster(CONF, 1, false, null);
    cluster.waitClusterUp();
}

From source file:crunch.MaxTemperature.java

License:Apache License

  @Before
public void setUp() throws IOException { // XXX set  up FileSystem in setUp
  Configuration conf = new Configuration();
  if (System.getProperty("test.build.data") == null) {
    System.setProperty("test.build.data", "/tmp");
  }//from  w  w w . ja  v  a2s .  com
  cluster = new MiniDFSCluster(conf, 1, true, null); // XXX
  fs = cluster.getFileSystem(); // XXX
}

From source file:crunch.MaxTemperature.java

License:Apache License

@Before // XXX
    public void setUp() throws IOException {
        Configuration conf = new Configuration(); // XXX hadoop.conf.Configuration
        if (System.getProperty("test.build.data") == null) {
            System.setProperty("test.build.data", "/tmp");
        }//w  ww .j a va 2  s.c o m
        cluster = new MiniDFSCluster(conf, 1, true, null); // XXX
        fs = cluster.getFileSystem(); // XXX
        OutputStream out = fs.create(new Path("/dir/file")); // XXX hadoop.fs.Path
        out.write("content".getBytes("UTF-8"));
        out.close();
    }

From source file:crunch.MaxTemperature.java

License:Apache License

@BeforeClass
    public static void setUp() throws IOException {
        Configuration conf = new Configuration();
        if (System.getProperty("test.build.data") == null) {
            System.setProperty("test.build.data", "/tmp");
        }// ww w. j  a  va2 s. c  om
        cluster = new MiniDFSCluster(conf, 1, true, null);
        fs = cluster.getFileSystem();
    }

From source file:csc555.ebratt.depaul.edu.Test_RCWordCountDriver.java

License:Open Source License

/**
 * @throws java.lang.Exception// w ww .j av a2s  .c  o m
 */
@Before
public void setUp() throws Exception {
    // create a new mini dfs cluster
    cluster = new MiniDFSCluster(CONF, NUM_DATA_NODES, true, null);
    String hdfsURI = "hdfs://localhost:" + cluster.getNameNodePort() + "/";
    LOG.info("HDFS URI: " + hdfsURI);
    cluster.waitActive();
    namesystem = FSNamesystem.getFSNamesystem();
    fs = (DistributedFileSystem) cluster.getFileSystem();
    nn = cluster.getNameNode();
}

From source file:datafu.hourglass.test.TestBase.java

License:Apache License

@SuppressWarnings("deprecation")
public void beforeClass() throws Exception {
    // make sure the log folder exists or it will fail
    new File("test-logs").mkdirs();
    System.setProperty("hadoop.log.dir", "test-logs");

    if (localFS) {
        _fileSystem = FileSystem.get(new JobConf());
        _log.info("*** Using local file system: " + _fileSystem.getUri());
    } else {//from w ww . j a  va 2  s . c o  m
        _log.info("*** Starting Mini DFS Cluster");
        _dfsCluster = new MiniDFSCluster(new JobConf(), dataNodes, true, null);
        _fileSystem = _dfsCluster.getFileSystem();
    }

    if (localMR) {
        _log.info("*** Using local MR Cluster");
    } else {
        _log.info("*** Starting Mini MR Cluster");
        _mrCluster = new MiniMRCluster(taskTrackers, _fileSystem.getName(), 1);
    }
}

From source file:edu.berkeley.chukwa_xtrace.TestXtrExtract.java

License:Apache License

public void testArchiving() throws Exception {

    System.out.println("starting archive test");
    Configuration conf = new Configuration();
    System.setProperty("hadoop.log.dir", System.getProperty("test.build.data", "/tmp"));
    MiniDFSCluster dfs = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null);
    FileSystem fileSys = dfs.getFileSystem();
    fileSys.delete(OUTPUT_DIR, true);//nuke output dir

    writeASinkFile(conf, fileSys, INPUT_DIR, 1000);

    FileStatus fstat = fileSys.getFileStatus(INPUT_DIR);
    assertTrue(fstat.getLen() > 10);/*from  ww  w.  ja  va2s .c  o m*/

    System.out.println("filesystem is " + fileSys.getUri());
    conf.set("fs.default.name", fileSys.getUri().toString());
    conf.setInt("io.sort.mb", 1);
    conf.setInt("io.sort.factor", 5);
    conf.setInt("mapred.tasktracker.map.tasks.maximum", 2);
    conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 2);

    MiniMRCluster mr = new MiniMRCluster(NUM_HADOOP_SLAVES, fileSys.getUri().toString(), 1);
    String[] archiveArgs = { INPUT_DIR.toString(), fileSys.getUri().toString() + OUTPUT_DIR.toString() };

    JobConf jc = mr.createJobConf(new JobConf(conf));
    assertEquals("true", jc.get("archive.groupByClusterName"));
    assertEquals(1, jc.getInt("io.sort.mb", 5));

    int returnVal = ToolRunner.run(jc, new XtrExtract(), archiveArgs);
    assertEquals(0, returnVal);
    fstat = fileSys.getFileStatus(new Path("/chukwa/archives/foocluster/HadoopLogProcessor_2008_05_29.arc"));
    assertTrue(fstat.getLen() > 10);

    Thread.sleep(1000);

    System.out.println("done!");
}

From source file:edu.uci.ics.hyracks.hdfs.dataflow.DataflowTest.java

License:Apache License

/**
 * Start the HDFS cluster and setup the data files
 * /*from w  w  w.  ja  va 2 s  .c  om*/
 * @throws IOException
 */
private void startHDFS() throws IOException {
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));

    FileSystem lfs = FileSystem.getLocal(new Configuration());
    lfs.delete(new Path("build"), true);
    System.setProperty("hadoop.log.dir", "logs");
    dfsCluster = new MiniDFSCluster(conf, numberOfNC, true, null);
    FileSystem dfs = FileSystem.get(conf);
    Path src = new Path(DATA_PATH);
    Path dest = new Path(HDFS_INPUT_PATH);
    Path result = new Path(HDFS_OUTPUT_PATH);
    dfs.mkdirs(dest);
    dfs.mkdirs(result);
    dfs.copyFromLocalFile(src, dest);

    DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
    conf.writeXml(confOutput);
    confOutput.flush();
    confOutput.close();
}

From source file:edu.uci.ics.pregelix.example.asterixdb.ConnectorTest.java

License:Apache License

@SuppressWarnings("deprecation")
private static void startHDFS() throws IOException {
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
    FileSystem lfs = FileSystem.getLocal(new Configuration());
    lfs.delete(new Path("build"), true);
    System.setProperty("hadoop.log.dir", "logs");
    dfsCluster = new MiniDFSCluster(conf, numberOfNC, true, null);
    DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
    conf.writeXml(confOutput);/*from   w  w  w  .  j a va  2  s  .c om*/
    confOutput.flush();
    confOutput.close();
}