Example usage for org.apache.hadoop.mapreduce Job getTaskCompletionEvents

List of usage examples for org.apache.hadoop.mapreduce Job getTaskCompletionEvents

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. getTaskCompletionEvents

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job getTaskCompletionEvents.

Prototype

public org.apache.hadoop.mapred.TaskCompletionEvent[] getTaskCompletionEvents(final int startFrom)
        throws IOException

Source Link

Document

Get events indicating completion (success/failure) of component tasks.

Usage

From source file:cascading.stats.hadoop.HadoopStepStats.java

License:Open Source License

/** Method captureDetail captures statistics task details and completion events. */
@Override/*from  ww w.  ja  v a2s  .co m*/
public synchronized void captureDetail(Type depth) {
    if (!getType().isChild(depth))
        return;

    Job runningJob = getJob(getJobStatusClient());

    if (runningJob == null)
        return;

    try {
        mapperNodeStats.captureDetail(depth);

        if (reducerNodeStats != null)
            reducerNodeStats.captureDetail(depth);

        int count = 0;

        while (depth == Type.ATTEMPT) {
            // todo: we may be able to continue where we left off if we retain the count
            TaskCompletionEvent[] events = runningJob.getTaskCompletionEvents(count);

            if (events.length == 0)
                break;

            addAttemptsToTaskStats(events);
            count += events.length;
        }
    } catch (IOException exception) {
        LOG.warn("unable to get task stats", exception);
    }
}

From source file:org.apache.falcon.logging.TaskLogRetrieverYarn.java

License:Apache License

@Override
public List<String> retrieveTaskLogURL(String jobIdStr) throws IOException {
    List<String> taskLogUrls = new ArrayList<String>();
    Configuration conf = getConf();
    Cluster cluster = getCluster(conf);/*from w w  w. ja v  a2  s  . c  o  m*/
    JobID jobID = JobID.forName(jobIdStr);
    if (jobID == null) {
        LOG.warn("External id for workflow action is null");
        return null;
    }

    if (conf.get(YARN_LOG_SERVER_URL) == null) {
        LOG.warn("YARN log Server is null");
        return null;
    }

    try {
        Job job = cluster.getJob(jobID);
        if (job != null) {
            TaskCompletionEvent[] events = job.getTaskCompletionEvents(0);
            for (TaskCompletionEvent event : events) {
                LogParams params = cluster.getLogParams(jobID, event.getTaskAttemptId());
                String url = (conf.get(YARN_LOG_SERVER_URL).startsWith(SCHEME) ? conf.get(YARN_LOG_SERVER_URL)
                        : SCHEME + conf.get(YARN_LOG_SERVER_URL)) + "/" + event.getTaskTrackerHttp() + "/"
                        + params.getContainerId() + "/" + params.getApplicationId() + "/" + params.getOwner()
                        + "?start=0";
                LOG.info("Task Log URL for the job {} is {}" + jobIdStr, url);
                taskLogUrls.add(url);
            }
            return taskLogUrls;
        }
        LOG.warn("Unable to find the job in cluster {}" + jobIdStr);
        return null;
    } catch (InterruptedException e) {
        throw new IOException(e);
    }
}

From source file:org.apache.falcon.logging.v2.TaskLogRetrieverYarn.java

License:Apache License

@Override
public List<String> retrieveTaskLogURL(String jobIdStr) throws IOException {
    List<String> taskLogUrls = new ArrayList<String>();
    Configuration conf = getConf();
    Cluster cluster = getCluster(conf);//from w  w w  .  j  a  v a2  s.  co  m
    JobID jobID = JobID.forName(jobIdStr);
    if (jobID == null) {
        LOG.warn("External id for workflow action is null");
        return null;
    }
    try {
        Job job = cluster.getJob(jobID);
        if (job != null) {
            TaskCompletionEvent[] events = job.getTaskCompletionEvents(0);
            for (TaskCompletionEvent event : events) {
                LogParams params = cluster.getLogParams(jobID, event.getTaskAttemptId());
                String url = SCHEME + conf.get(YARN_LOG_SERVER_URL) + "/" + event.getTaskTrackerHttp() + "/"
                        + params.getContainerId() + "/" + params.getApplicationId() + "/" + params.getOwner()
                        + "?start=0";
                LOG.info("Task Log URL for the job {} is {}" + jobIdStr, url);
                taskLogUrls.add(url);
            }
            return taskLogUrls;
        }
        LOG.warn("Unable to find the job in cluster {}" + jobIdStr);
        return null;
    } catch (InterruptedException e) {
        throw new IOException(e);
    }
}

From source file:org.apache.falcon.logging.v2.TaskLogRetrieverYarnTest.java

License:Apache License

@DataProvider(name = "testData")
public Object[][] testData() throws IOException, InterruptedException {
    int samples = getRandomValueInRange(10) + 1;
    Object[][] resultSet = new Object[samples][2];
    for (int count = 0; count < samples; count++) {
        List<String> expectedResult = new ArrayList<String>();
        Cluster cluster = getCluster(getConf());
        String jobId = new JobID("job", RANDOM.nextInt(1000)).toString();
        boolean success = RANDOM.nextBoolean();
        JobID jobID = JobID.forName(jobId);
        int numEvents = getRandomValueInRange(10) + 1;
        TaskCompletionEvent[] events = getTaskCompletionEvents(numEvents, jobID);
        Job job = mock(Job.class);
        when(cluster.getJob(jobID)).thenReturn(job);
        when(job.getTaskCompletionEvents(0)).thenReturn(events);
        for (TaskCompletionEvent event : events) {
            if (success) {
                LogParams params = getLogParams();
                when(cluster.getLogParams(jobID, event.getTaskAttemptId())).thenReturn(params);
                String url = SCHEME + getConf().get(YARN_LOG_SERVER_URL) + "/" + event.getTaskTrackerHttp()
                        + "/" + params.getContainerId() + "/" + params.getApplicationId() + "/"
                        + params.getOwner() + "?start=0";
                expectedResult.add(url);
            } else {
                when(cluster.getJob(jobID)).thenReturn(null);
                expectedResult = null;//w  w w . j  ava 2 s. co  m
            }
            resultSet[count] = new Object[] { jobId, expectedResult };
        }
    }
    return resultSet;
}

From source file:org.apache.falcon.oozie.logging.TaskLogRetrieverYarnTest.java

License:Apache License

@DataProvider(name = "testData")
public Object[][] testData() throws IOException, InterruptedException {
    int samples = getRandomValueInRange(10) + 1;
    Object[][] resultSet = new Object[samples][2];
    for (int count = 0; count < samples; count++) {
        List<String> expectedResult = new ArrayList<String>();
        Cluster cluster = getCluster(getConf());
        String jobId = new JobID("job", count).toString();
        boolean success = random.nextBoolean();
        JobID jobID = JobID.forName(jobId);
        int numEvents = getRandomValueInRange(10) + 1;
        TaskCompletionEvent[] events = getTaskCompletionEvents(numEvents, jobID);
        Job job = mock(Job.class);
        when(cluster.getJob(jobID)).thenReturn(job);
        when(job.getTaskCompletionEvents(0)).thenReturn(events);
        for (TaskCompletionEvent event : events) {
            if (success) {
                LogParams params = getLogParams();
                when(cluster.getLogParams(jobID, event.getTaskAttemptId())).thenReturn(params);
                String url = SCHEME + getConf().get(YARN_LOG_SERVER_URL) + "/" + event.getTaskTrackerHttp()
                        + "/" + params.getContainerId() + "/" + params.getApplicationId() + "/"
                        + params.getOwner() + "?start=0";
                expectedResult.add(url);
            } else {
                when(cluster.getJob(jobID)).thenReturn(null);
                expectedResult = null;/*  w w w  .  ja  v  a 2  s .  co  m*/
                break;
            }
        }
        resultSet[count] = new Object[] { jobId, expectedResult };
    }
    return resultSet;
}

From source file:org.apache.gobblin.compaction.mapreduce.CompactionJobConfigurator.java

License:Apache License

private static List<TaskCompletionEvent> getAllTaskCompletionEvent(Job completedJob) {
    List<TaskCompletionEvent> completionEvents = new LinkedList<>();

    while (true) {
        try {//  w  w  w .j  a  va2  s  .c o  m
            TaskCompletionEvent[] bunchOfEvents;
            bunchOfEvents = completedJob.getTaskCompletionEvents(completionEvents.size());
            if (bunchOfEvents == null || bunchOfEvents.length == 0) {
                break;
            }
            completionEvents.addAll(Arrays.asList(bunchOfEvents));
        } catch (IOException e) {
            break;
        }
    }

    return completionEvents;
}

From source file:weka.distributed.hadoop.HadoopJob.java

License:Open Source License

/**
 * Output task messages for the currently running job
 * /* w  w  w .  j a  v  a2s .  c o m*/
 * @param job the job to output messages for
 * @param startIndex the index to start outputting messages from
 * @return the index of the last message output
 * @throws IOException if a problem occurs
 */
protected int logTaskMessages(Job job, int startIndex) throws IOException {
    TaskCompletionEvent[] tcEvents = job.getTaskCompletionEvents(startIndex);

    // StringBuilder taskMessages = new StringBuilder();
    for (TaskCompletionEvent tcEvent : tcEvents) {
        logMessage(tcEvent.toString());
        // taskMessages.append(tcEvent.toString()).append("\n");
    }

    // logMessage(taskMessages.toString());

    return tcEvents.length;
}