Example usage for org.apache.hadoop.mapred Counters incrAllCounters

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred Counters incrAllCounters.

Prototype

public synchronized void incrAllCounters(Counters other) 

Source Link

Document

Increments multiple counters by their amounts in another Counters instance.

Usage

From source file:org.apache.oozie.action.hadoop.SqoopActionExecutor.java

License:Apache License

/**
 * We will gather counters from all executed action Hadoop jobs (e.g. jobs
 * that moved data, not the launcher itself) and merge them together. There
 * will be only one job most of the time. The only exception is
 * import-all-table option that will execute one job per one exported table.
 *
 * @param context Action context/*from w  w w .ja  v  a 2 s  . com*/
 * @param action Workflow action
 * @throws ActionExecutorException
 */
@Override
public void end(Context context, WorkflowAction action) throws ActionExecutorException {
    super.end(context, action);
    JobClient jobClient = null;

    boolean exception = false;
    try {
        if (action.getStatus() == WorkflowAction.Status.OK) {
            Element actionXml = XmlUtils.parseXml(action.getConf());
            JobConf jobConf = createBaseHadoopConf(context, actionXml);
            jobClient = createJobClient(context, jobConf);

            // Cumulative counters for all Sqoop mapreduce jobs
            Counters counters = null;

            // Sqoop do not have to create mapreduce job each time
            String externalIds = action.getExternalChildIDs();
            if (externalIds != null && !externalIds.trim().isEmpty()) {
                String[] jobIds = externalIds.split(",");

                for (String jobId : jobIds) {
                    RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
                    if (runningJob == null) {
                        throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "SQOOP001",
                                "Unknown hadoop job [{0}] associated with action [{1}].  Failing this action!",
                                action.getExternalId(), action.getId());
                    }

                    Counters taskCounters = runningJob.getCounters();
                    if (taskCounters != null) {
                        if (counters == null) {
                            counters = taskCounters;
                        } else {
                            counters.incrAllCounters(taskCounters);
                        }
                    } else {
                        XLog.getLog(getClass()).warn("Could not find Hadoop Counters for job: [{0}]", jobId);
                    }
                }
            }

            if (counters != null) {
                ActionStats stats = new MRStats(counters);
                String statsJsonString = stats.toJSON();
                context.setVar(MapReduceActionExecutor.HADOOP_COUNTERS, statsJsonString);

                // If action stats write property is set to false by user or
                // size of stats is greater than the maximum allowed size,
                // do not store the action stats
                if (Boolean.parseBoolean(
                        evaluateConfigurationProperty(actionXml, OOZIE_ACTION_EXTERNAL_STATS_WRITE, "true"))
                        && (statsJsonString.getBytes().length <= getMaxExternalStatsSize())) {
                    context.setExecutionStats(statsJsonString);
                    LOG.debug("Printing stats for sqoop action as a JSON string : [{0}]", statsJsonString);
                }
            } else {
                context.setVar(MapReduceActionExecutor.HADOOP_COUNTERS, "");
                XLog.getLog(getClass()).warn("Can't find any associated Hadoop job counters");
            }
        }
    } catch (Exception ex) {
        exception = true;
        throw convertException(ex);
    } finally {
        if (jobClient != null) {
            try {
                jobClient.close();
            } catch (Exception e) {
                if (exception) {
                    LOG.error("JobClient error: ", e);
                } else {
                    throw convertException(e);
                }
            }
        }
    }
}

---