Example usage for org.apache.hadoop.mapred MiniMRCluster createJobConf

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred MiniMRCluster createJobConf.

Prototype

public JobConf createJobConf() 

Source Link

Usage

From source file:JaqlShell.java

License:Apache License

/**
 * @param mrc/*from  www  . j  a  va  2  s  .c o m*/
 * @param conf
 * @throws Exception
 */
private static void setupOverride(MiniMRCluster mrc, Configuration conf) throws Exception {
    File overrideDir = new File(System.getProperty("hadoop.conf.override"));
    if (!overrideDir.exists()) {
        overrideDir.mkdirs();
    }

    // write out the JobConf from MiniMR to the override dir
    JobConf jc = mrc.createJobConf();
    conf.set("mapred.job.tracker", jc.get("mapred.job.tracker", null));
    String name = "mapred.job.tracker.info.port";
    String addr = jc.get(name, null);
    if (addr == null) {
        name = "mapred.job.tracker.http.address";
        addr = jc.get(name, null);
    }
    conf.set(name, addr);
    OutputStream outCore = new FileOutputStream(
            overrideDir.getCanonicalPath() + File.separator + "core-default.xml");
    OutputStream outMapred = new FileOutputStream(
            overrideDir.getCanonicalPath() + File.separator + "mapred-default.xml");
    OutputStream outHdfs = new FileOutputStream(
            overrideDir.getCanonicalPath() + File.separator + "hdfs-default.xml");
    conf.writeXml(outCore);
    conf.writeXml(outMapred);
    conf.writeXml(outHdfs);
    outCore.close();
    outMapred.close();
    outHdfs.close();
}

---

From source file:com.hadoop.mapreduce.TestLzoLazyLoading.java

License:Open Source License

public void testWithLocal() throws Exception {
    MiniMRCluster mr = null;
    try {/*from   w  ww.  j a v  a2  s. c o m*/
        JobConf jconf = new JobConf();
        jconf.set("mapred.queue.names", "default");
        mr = new MiniMRCluster(2, "file:///", 3, null, null, jconf);
        Configuration cf = mr.createJobConf();
        cf.set("io.compression.codecs", LzoCodec.class.getName());
        runWordCount(cf, false, false);
        runWordCount(cf, false, true);
        runWordCount(cf, true, false);
    } finally {
        if (mr != null) {
            mr.shutdown();
        }
    }
}

---

From source file:com.ngdata.hbaseindexer.mr.MRTestUtil.java

License:Apache License

/**
 * Start a mini MapReduce cluster in either HBase 0.94 or HBase 0.95/96 mode.
 *///from  w w  w  .  ja v a2s .  com
public void startMrCluster() throws Exception {
    // Handle compatibility between HBase 0.94 and HBase 0.95
    Method startMrClusterMethod = hbaseTestUtil.getClass().getMethod("startMiniMapReduceCluster");

    if (Void.TYPE.equals(startMrClusterMethod.getReturnType())) {
        // HBase 0.94.x doesn't return a MR cluster, and puts the JobTracker
        // information directly in its own configuration
        hbaseTestUtil.startMiniMapReduceCluster();
    } else {
        // HBase 0.95.x returns a MR cluster, and we have to manually
        // copy the job tracker address into our configuration
        MiniMRCluster mrCluster = (MiniMRCluster) startMrClusterMethod.invoke(hbaseTestUtil);
        Configuration conf = hbaseTestUtil.getConfiguration();
        conf.set("mapred.job.tracker", mrCluster.createJobConf().get("mapred.job.tracker"));
    }
}

---

From source file:org.apache.crunch.io.hbase.HFileTargetIT.java

License:Apache License

/**
 * We need to set the address of JobHistory server, as it randomly picks a unused port
 * to listen. Unfortunately, HBaseTestingUtility neither does that nor provides a way
 * for us to know the picked address. We have to access it using reflection.
 *
 * This is necessary when testing with MRv2, but does no harm to MRv1.
 *///from   w w  w  .java2 s  .  c o  m
private static void dirtyFixForJobHistoryServerAddress() {
    try {
        // Retrieve HBASE_TEST_UTILITY.mrCluster via reflection, as it is private.
        Field mrClusterField = HBaseTestingUtility.class.getDeclaredField("mrCluster");
        mrClusterField.setAccessible(true);
        MiniMRCluster mrCluster = (MiniMRCluster) mrClusterField.get(HBASE_TEST_UTILITY);
        JobConf jobConf = mrCluster.createJobConf();
        Configuration conf = HBASE_TEST_UTILITY.getConfiguration();
        String proprety = "mapreduce.jobhistory.address";
        String value = jobConf.get(proprety);
        if (value != null) { // maybe null if we're running MRv1
            conf.set(proprety, value);
        }
    } catch (IllegalAccessException e) {
        throw new AssertionError(e);
    } catch (NoSuchFieldException e) {
        throw new AssertionError(e);
    }
}

---