Example usage for org.apache.hadoop.mapreduce JobID JobID

List of usage examples for org.apache.hadoop.mapreduce JobID JobID

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. JobID
  6. JobID

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce JobID JobID.

Prototype

public JobID(String jtIdentifier, int id)

Source Link

Document

Constructs a JobID object

Usage

From source file:com.asakusafw.runtime.compatibility.hadoop.JobCompatibilityHadoop.java

License:Apache License

/**
 * Creates a new job ID./*from  w  w w.  j ava 2 s . co  m*/
 * @param id the local ID
 * @return the created ID
 */
public JobID newJobId(int id) {
    return new JobID(NAME_DUMMY_JOB, id);
}

From source file:com.bosscs.spark.commons.extractor.impl.GenericHadoopExtractor.java

License:Apache License

@Override
public Partition[] getPartitions(S config) {

    int id = config.getRddId();

    jobId = new JobID(jobTrackerId, id);

    Configuration conf = getHadoopConfig(config);

    JobContext jobContext = SparkHadoopMapReduceUtil.newJobContext(conf, jobId);

    try {/*www. j  a  v  a  2 s. c o m*/
        List<InputSplit> splits = inputFormat.getSplits(jobContext);

        Partition[] partitions = new Partition[(splits.size())];
        for (int i = 0; i < splits.size(); i++) {
            partitions[i] = new NewHadoopPartition(id, i, splits.get(i));
        }

        return partitions;

    } catch (IOException | InterruptedException | RuntimeException e) {
        LOG.error("Impossible to calculate partitions " + e.getMessage());
        throw new GenericException("Impossible to calculate partitions ", e);
    }

}

From source file:com.google.appengine.tools.mapreduce.AppEngineJobContext.java

License:Apache License

/**
 * Generates a new unique Hadoop job ID.
 *
 * There's a whole idiom for how JobID is constructed.
 * See/*from   w  w  w .  j  a v  a2 s  . c om*/
 * <a href="http://hadoop.apache.org/common/docs/r0.20.0/api/org/apache/hadoop/mapreduce/TaskAttemptID.html">
 * TaskAttemptID</a> and the linked classes for details.
 *
 * In the interest of making everyone happy, we pretend like we're the world's
 * worst job tracker. It restarts each time we start a new MR. On the plus
 * side, every job is job #1!
 */
protected static JobID generateNewJobID() {
    return new JobID(("" + System.currentTimeMillis() + UUID.randomUUID()).replace("-", ""), 1);
}

From source file:com.google.appengine.tools.mapreduce.AppEngineJobContextTest.java

License:Apache License

@Override
public void setUp() throws Exception {
    super.setUp();
    helper.setUp();/*from  ww w.  j  a va 2s .co m*/
    ds = DatastoreServiceFactory.getDatastoreService();
    jobId = new JobID("foo", 1);
}

From source file:com.google.appengine.tools.mapreduce.AppEngineJobContextTest.java

License:Apache License

public void testGetJobContextFromRequest() {
    JobID jobId = new JobID("foo", 1);
    HttpServletRequest req = createMockMapReduceRequest(jobId);
    replay(req);/*from  w w  w  .  j  a v  a 2 s .c  o  m*/

    Configuration conf = ConfigurationXmlUtil.getConfigurationFromXml(SIMPLE_CONF_XML);
    persistMRState(jobId, conf);

    JobContext context = new AppEngineJobContext(req);
    assertEquals("/tmp/foo", context.getConfiguration().get("foo.bar"));
    assertEquals(jobId.toString(), context.getJobID().toString());
    verify(req);
}

From source file:com.google.appengine.tools.mapreduce.DatastoreInputFormatTest.java

License:Apache License

/**
 * Create splits with sensible defaults for testing.
 *//*from ww  w  . j av  a 2s. com*/
private List<InputSplit> getEntityKindSplitsFromCount(int shardCount) throws IOException {
    DatastoreInputFormat inputFormat = new DatastoreInputFormat();
    Configuration conf = new Configuration();
    conf.set(DatastoreInputFormat.ENTITY_KIND_KEY, ENTITY_KIND_NAME);
    conf.set(DatastoreInputFormat.SHARD_COUNT_KEY, "" + shardCount);

    JobContext context = new JobContext(conf, new JobID("Foo", 1));

    List<InputSplit> splits = inputFormat.getSplits(context);

    assertEquals(shardCount, splits.size());

    return splits;
}

From source file:com.google.appengine.tools.mapreduce.DatastoreInputFormatTest.java

License:Apache License

public void testGetSplitWithNoData() throws Exception {
    DatastoreInputFormat inputFormat = new DatastoreInputFormat();
    Configuration conf = new Configuration();
    conf.set(DatastoreInputFormat.ENTITY_KIND_KEY, ENTITY_KIND_NAME);
    conf.set(DatastoreInputFormat.SHARD_COUNT_KEY, "1");

    JobContext context = new JobContext(conf, new JobID("Foo", 1));

    List<InputSplit> splits = inputFormat.getSplits(context);
    assertEquals(0, splits.size());//from w  w  w.ja  v a2  s  . co m
}

From source file:com.google.appengine.tools.mapreduce.MapReduceServletTest.java

License:Apache License

public void testControllerCSRF() {
    JobID jobId = new JobID("foo", 1);
    // Send it as an AJAX request but not a task queue request - should be denied.
    HttpServletRequest request = createMockRequest(MapReduceServlet.CONTROLLER_PATH, false, true);
    expect(request.getParameter(AppEngineJobContext.SLICE_NUMBER_PARAMETER_NAME)).andReturn("" + 0).anyTimes();
    expect(request.getParameter(AppEngineJobContext.JOB_ID_PARAMETER_NAME)).andReturn("" + jobId).anyTimes();
    HttpServletResponse response = createMock(HttpServletResponse.class);
    try {//from  w  w w . ja va 2  s.c  o  m
        response.sendError(403, "Received unexpected non-task queue request.");
    } catch (IOException ioe) {
        // Can't actually be sent in mock setup
    }
    replay(request, response);
    servlet.doPost(request, response);
    verify(request, response);
}

From source file:com.google.appengine.tools.mapreduce.MapReduceServletTest.java

License:Apache License

public void testGetJobDetailCSRF() {
    JobID jobId = new JobID("foo", 1);
    // Send it as a task queue request but not an ajax request - should be denied.
    HttpServletRequest request = createMockRequest(
            MapReduceServlet.COMMAND_PATH + "/" + MapReduceServlet.GET_JOB_DETAIL_PATH, true, false);
    expect(request.getMethod()).andReturn("POST").anyTimes();
    expect(request.getParameter(AppEngineJobContext.JOB_ID_PARAMETER_NAME)).andReturn("" + jobId).anyTimes();

    HttpServletResponse response = createMock(HttpServletResponse.class);

    // Set before error and last one wins, so this is harmless.
    response.setContentType("application/json");
    EasyMock.expectLastCall().anyTimes();

    try {/* w  ww . j  a v a2  s.  c  o m*/
        response.sendError(403, "Received unexpected non-XMLHttpRequest command.");
    } catch (IOException ioe) {
        // Can't actually be sent in mock setup
    }
    replay(request, response);
    servlet.doGet(request, response);
    verify(request, response);
}

From source file:com.google.appengine.tools.mapreduce.MapReduceServletTest.java

License:Apache License

/**
 * Test that handleController has reasonable behavior when there are still
 * active workers.//from w w w  .  j a v a 2  s .  c  o  m
 *
 * @throws EntityNotFoundException
 */
public void testHandleController_withContinue() throws EntityNotFoundException {
    JobID jobId = new JobID("foo", 1);
    HttpServletRequest request = createMockControllerRequest(0, jobId);
    replay(request);

    Configuration sampleConf = getSampleMapReduceConfiguration();

    persistMRState(jobId, sampleConf);

    ShardState shardState1 = ShardState.generateInitializedShardState(ds,
            new TaskAttemptID(new TaskID(jobId, true, 1), 1));
    Counters counters1 = new Counters();
    counters1.findCounter("a", "z").increment(1);
    shardState1.setCounters(counters1);
    shardState1.setInputSplit(sampleConf, new StubInputSplit(1));
    shardState1.setRecordReader(sampleConf, new StubRecordReader());
    shardState1.persist();

    ShardState shardState2 = ShardState.generateInitializedShardState(ds,
            new TaskAttemptID(new TaskID(jobId, true, 2), 1));
    Counters counters2 = new Counters();
    counters2.findCounter("a", "z").increment(1);
    shardState2.setCounters(counters2);
    shardState2.setInputSplit(sampleConf, new StubInputSplit(2));
    shardState2.setRecordReader(sampleConf, new StubRecordReader());
    shardState2.setDone();
    shardState2.persist();

    // doPost should call handleCallback()
    // resp is never used
    servlet.doPost(request, null);

    MapReduceState mrState = MapReduceState.getMapReduceStateFromJobID(ds, jobId);

    // Check result of aggregateState()
    assertEquals(2, mrState.getCounters().findCounter("a", "z").getValue());

    // Check the result of refillQuota()
    // Should fill the active thread but not the done one.
    assertEquals(1000, new QuotaManager(MemcacheServiceFactory.getMemcacheService())
            .get("" + shardState1.getTaskAttemptID()));
    assertEquals(0, new QuotaManager(MemcacheServiceFactory.getMemcacheService())
            .get("" + shardState2.getTaskAttemptID()));

    // Check that the next controller task got enqueued.
    QueueStateInfo defaultQueue = getDefaultQueueInfo();
    assertEquals(1, defaultQueue.getCountTasks());
    TaskStateInfo firstTask = defaultQueue.getTaskInfo().get(0);
    assertEquals("/mapreduce/" + MapReduceServlet.CONTROLLER_PATH, firstTask.getUrl());
    assertTrue(firstTask.getBody(), firstTask.getBody().indexOf("jobID=job_foo_0001") != -1);

    assertEquals(1, mrState.getActiveShardCount());
    assertEquals(2, mrState.getShardCount());

    verify(request);
}