Example usage for org.apache.hadoop.mapreduce JobID JobID

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce JobID JobID.

Prototype

public JobID(String jtIdentifier, int id) 

Source Link

Document

Constructs a JobID object

Usage

From source file:org.apache.falcon.oozie.logging.TaskLogRetrieverYarnTest.java

License:Apache License

@DataProvider(name = "testData")
public Object[][] testData() throws IOException, InterruptedException {
    int samples = getRandomValueInRange(10) + 1;
    Object[][] resultSet = new Object[samples][2];
    for (int count = 0; count < samples; count++) {
        List<String> expectedResult = new ArrayList<String>();
        Cluster cluster = getCluster(getConf());
        String jobId = new JobID("job", count).toString();
        boolean success = random.nextBoolean();
        JobID jobID = JobID.forName(jobId);
        int numEvents = getRandomValueInRange(10) + 1;
        TaskCompletionEvent[] events = getTaskCompletionEvents(numEvents, jobID);
        Job job = mock(Job.class);
        when(cluster.getJob(jobID)).thenReturn(job);
        when(job.getTaskCompletionEvents(0)).thenReturn(events);
        for (TaskCompletionEvent event : events) {
            if (success) {
                LogParams params = getLogParams();
                when(cluster.getLogParams(jobID, event.getTaskAttemptId())).thenReturn(params);
                String url = SCHEME + getConf().get(YARN_LOG_SERVER_URL) + "/" + event.getTaskTrackerHttp()
                        + "/" + params.getContainerId() + "/" + params.getApplicationId() + "/"
                        + params.getOwner() + "?start=0";
                expectedResult.add(url);
            } else {
                when(cluster.getJob(jobID)).thenReturn(null);
                expectedResult = null;//from w ww.  j a v  a2s.  c  om
                break;
            }
        }
        resultSet[count] = new Object[] { jobId, expectedResult };
    }
    return resultSet;
}

---

From source file:org.apache.hive.hcatalog.pig.TestE2EScenarios.java

License:Apache License

private TaskAttemptContext createTaskAttemptContext(Configuration tconf) {
    Configuration conf = (tconf == null) ? (new Configuration()) : tconf;
    TaskAttemptID taskId = HCatMapRedUtil.createTaskAttemptID(new JobID("200908190029", 1), false, 1, 1);
    conf.setInt("mapred.task.partition", taskId.getId());
    conf.set("mapred.task.id", taskId.toString());
    TaskAttemptContext rtaskContext = HCatMapRedUtil.createTaskAttemptContext(conf, taskId);
    return rtaskContext;
}

---

From source file:org.apache.ignite.client.hadoop.GridHadoopClientProtocol.java

License:Apache License

/** {@inheritDoc} */
@Override//from  w  w w  .ja va 2s  .  c o  m
public JobID getNewJobID() throws IOException, InterruptedException {
    try {
        conf.setLong(REQ_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());

        GridHadoopJobId jobID = cli.compute().execute(GridHadoopProtocolNextTaskIdTask.class.getName(), null);

        conf.setLong(RESPONSE_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());

        return new JobID(jobID.globalId().toString(), jobID.localId());
    } catch (GridClientException e) {
        throw new IOException("Failed to get new job ID.", e);
    }
}

---

From source file:org.apache.ignite.client.hadoop.GridHadoopClientProtocolSelfTest.java

License:Apache License

/**
 * Tests job counters retrieval for unknown job id.
 *
 * @throws Exception If failed./*from   w ww. j  av a2  s . co m*/
 */
private void tstUnknownJobCounters() throws Exception {
    GridHadoopClientProtocolProvider provider = provider();

    ClientProtocol proto = provider.create(config(GridHadoopAbstractSelfTest.REST_PORT));

    try {
        proto.getJobCounters(new JobID(UUID.randomUUID().toString(), -1));
        fail("exception must be thrown");
    } catch (Exception e) {
        assert e instanceof IOException : "wrong error has been thrown";
    }
}

---

From source file:org.apache.ignite.client.hadoop.HadoopClientProtocolSelfTest.java

License:Apache License

/**
 * Tests job counters retrieval for unknown job id.
 *
 * @throws Exception If failed.//w  ww. j  a  v a2  s  .  c  o  m
 */
private void tstUnknownJobCounters() throws Exception {
    IgniteHadoopClientProtocolProvider provider = provider();

    ClientProtocol proto = provider.create(config(HadoopAbstractSelfTest.REST_PORT));

    try {
        proto.getJobCounters(new JobID(UUID.randomUUID().toString(), -1));
        fail("exception must be thrown");
    } catch (Exception e) {
        assert e instanceof IOException : "wrong error has been thrown";
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.GridHadoopUtils.java

License:Apache License

/**
 * Convert GG job status to Hadoop job status.
 *
 * @param status GG job status./*from w  ww.j  av a  2 s . co m*/
 * @return Hadoop job status.
 */
public static JobStatus status(GridHadoopJobStatus status, Configuration conf) {
    JobID jobId = new JobID(status.jobId().globalId().toString(), status.jobId().localId());

    float setupProgress = 0;
    float mapProgress = 0;
    float reduceProgress = 0;
    float cleanupProgress = 0;

    JobStatus.State state = JobStatus.State.RUNNING;

    switch (status.jobPhase()) {
    case PHASE_SETUP:
        setupProgress = 0.42f;

        break;

    case PHASE_MAP:
        setupProgress = 1;
        mapProgress = 1f - status.pendingMapperCnt() / (float) status.totalMapperCnt();

        break;

    case PHASE_REDUCE:
        assert status.totalReducerCnt() > 0;

        setupProgress = 1;
        mapProgress = 1;
        reduceProgress = 1f - status.pendingReducerCnt() / (float) status.totalReducerCnt();

        break;

    case PHASE_CANCELLING:
    case PHASE_COMPLETE:
        if (!status.isFailed()) {
            setupProgress = 1;
            mapProgress = 1;
            reduceProgress = 1;
            cleanupProgress = 1;

            state = JobStatus.State.SUCCEEDED;
        } else
            state = JobStatus.State.FAILED;

        break;

    default:
        assert false;
    }

    return new JobStatus(jobId, setupProgress, mapProgress, reduceProgress, cleanupProgress, state,
            JobPriority.NORMAL, status.user(), status.jobName(), jobFile(conf, status.user(), jobId).toString(),
            "N/A");
}

---

From source file:org.apache.ignite.internal.processors.hadoop.HadoopUtils.java

License:Apache License

/**
 * Convert Ignite job status to Hadoop job status.
 *
 * @param status Ignite job status./*from  w w  w . jav  a 2  s.  com*/
 * @return Hadoop job status.
 */
public static JobStatus status(HadoopJobStatus status, Configuration conf) {
    JobID jobId = new JobID(status.jobId().globalId().toString(), status.jobId().localId());

    float setupProgress = 0;
    float mapProgress = 0;
    float reduceProgress = 0;
    float cleanupProgress = 0;

    JobStatus.State state = JobStatus.State.RUNNING;

    switch (status.jobPhase()) {
    case PHASE_SETUP:
        setupProgress = 0.42f;

        break;

    case PHASE_MAP:
        setupProgress = 1;
        mapProgress = 1f - status.pendingMapperCnt() / (float) status.totalMapperCnt();

        break;

    case PHASE_REDUCE:
        assert status.totalReducerCnt() > 0;

        setupProgress = 1;
        mapProgress = 1;
        reduceProgress = 1f - status.pendingReducerCnt() / (float) status.totalReducerCnt();

        break;

    case PHASE_CANCELLING:
    case PHASE_COMPLETE:
        if (!status.isFailed()) {
            setupProgress = 1;
            mapProgress = 1;
            reduceProgress = 1;
            cleanupProgress = 1;

            state = JobStatus.State.SUCCEEDED;
        } else
            state = JobStatus.State.FAILED;

        break;

    default:
        assert false;
    }

    return new JobStatus(jobId, setupProgress, mapProgress, reduceProgress, cleanupProgress, state,
            JobPriority.NORMAL, status.user(), status.jobName(), jobFile(conf, status.user(), jobId).toString(),
            "N/A");
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.java

License:Apache License

/**
 * Convert Ignite job status to Hadoop job status.
 *
 * @param status Ignite job status./*ww  w. j a  v  a2  s .  c  o m*/
 * @return Hadoop job status.
 */
public static JobStatus status(HadoopJobStatus status, Configuration conf) {
    JobID jobId = new JobID(status.jobId().globalId().toString(), status.jobId().localId());

    float setupProgress = 0;
    float mapProgress = 0;
    float reduceProgress = 0;
    float cleanupProgress = 0;

    JobStatus.State state = JobStatus.State.RUNNING;

    switch (status.jobPhase()) {
    case PHASE_SETUP:
        setupProgress = 0.42f;

        break;

    case PHASE_MAP:
        setupProgress = 1;
        mapProgress = 1f - status.pendingMapperCnt() / (float) status.totalMapperCnt();

        break;

    case PHASE_REDUCE:
        setupProgress = 1;
        mapProgress = 1;

        if (status.totalReducerCnt() > 0)
            reduceProgress = 1f - status.pendingReducerCnt() / (float) status.totalReducerCnt();
        else
            reduceProgress = 1f;

        break;

    case PHASE_CANCELLING:
    case PHASE_COMPLETE:
        if (!status.isFailed()) {
            setupProgress = 1;
            mapProgress = 1;
            reduceProgress = 1;
            cleanupProgress = 1;

            state = JobStatus.State.SUCCEEDED;
        } else
            state = JobStatus.State.FAILED;

        break;

    default:
        assert false;
    }

    return new JobStatus(jobId, setupProgress, mapProgress, reduceProgress, cleanupProgress, state,
            JobPriority.NORMAL, status.user(), status.jobName(), jobFile(conf, status.user(), jobId).toString(),
            "N/A");
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.proto.HadoopClientProtocol.java

License:Apache License

/** {@inheritDoc} */
@Override//from  w w  w .j  a  v a 2 s.  co m
public JobID getNewJobID() throws IOException, InterruptedException {
    try {
        conf.setLong(HadoopCommonUtils.REQ_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());

        HadoopJobId jobID = execute(HadoopProtocolNextTaskIdTask.class);

        conf.setLong(HadoopCommonUtils.RESPONSE_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());

        return new JobID(jobID.globalId().toString(), jobID.localId());
    } catch (GridClientException e) {
        throw new IOException("Failed to get new job ID.", e);
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.proto.HadoopClientProtocol.java

License:Apache License

/** {@inheritDoc} */
@Override/*from w  w  w .  j  a va  2s .co  m*/
public JobID getNewJobID() throws IOException, InterruptedException {
    try {
        conf.setLong(REQ_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());

        HadoopJobId jobID = cli.compute().execute(HadoopProtocolNextTaskIdTask.class.getName(), null);

        conf.setLong(RESPONSE_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());

        return new JobID(jobID.globalId().toString(), jobID.localId());
    } catch (GridClientException e) {
        throw new IOException("Failed to get new job ID.", e);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• »