Example usage for org.apache.hadoop.mapred JobClient getConf

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobClient getConf.

Prototype

@Override
    public Configuration getConf() 

Source Link

Usage

From source file:com.netflix.lipstick.warnings.JobWarnings.java

License:Apache License

public List<ReducerDuration> enumerateReducerRunTimesAccending(JobClient jobClient, String jobId) {
    if (!jobClient.getConf().getBoolean("pig.stats.notaskreport", false)) {
        try {//  ww w.  j a v  a2  s.c om
            TaskReport[] reduceTasks = jobClient.getReduceTaskReports(jobId);
            return enumerateReducerRunTimesAccending(reduceTasks);
        } catch (IOException e) {
            log.error("Error getting reduce task reports, continuing", e);
        }
    } else {
        log.info("Skipping reduce task reports for job " + jobId);
    }
    return Lists.newArrayList();
}

---

From source file:org.smartfrog.services.hadoop.components.cluster.ClusterStatusCheckerImpl.java

License:Open Source License

/**
 * Check the cluster status//from  w  w w.j a  va  2  s  .  c  o m
 *
 * @throws SFHadoopException on any problem with the checks
 * @return a cluster status string
 */
private String checkClusterStatus() throws SmartFrogException {

    try {
        JobClient cluster = createClientOnDemand();
        ClusterStatus status = cluster.getClusterStatus();
        StringBuilder result = new StringBuilder();

        if (supportedFileSystem) {
            Path sysDir = cluster.getSystemDir();
            URI uri = sysDir.toUri();
            sfLog().info("Checking filesystem " + uri);
            ManagedConfiguration conf = (ManagedConfiguration) cluster.getConf();
            String impl = "fs." + uri.getScheme() + ".impl";
            String classname = conf.get(impl);
            if (classname == null) {
                maybeDumpConfiguration(conf);
                throw new SFHadoopException("File system " + uri + " will not load "
                        + " - no configuration mapping for " + impl + " in " + conf.dump(), this, conf);
            }
            try {
                conf.getClassByName(classname);
            } catch (ClassNotFoundException e) {
                throw new SFHadoopException("File system " + uri + " will not load "
                        + " - unable to locate class " + impl + " : " + e, e, this, conf);
            }
            try {
                result.append("Filesystem: ").append(uri).append(" ; ");
                FileSystem fs = cluster.getFs();
            } catch (IOException e) {
                throw new SFHadoopException("File system " + uri + " will not load " + e, e, this, conf);
            } catch (IllegalArgumentException e) {
                throw new SFHadoopException("Bad File system URI" + e, e, this, conf);
            }
        }
        if (jobtrackerLive) {
            sfLog().info("Checking jobTracker ");
            JobTracker.State state = status.getJobTrackerState();
            if (!state.equals(JobTracker.State.RUNNING)) {
                throw new SFHadoopException(
                        "Job Tracker at " + jobtracker + " is not running. It is in the state " + state, this);
            }
            result.append("Job tracker is in state ").append(status);
        }
        checkRange(minActiveMapTasks, maxActiveMapTasks, status.getMapTasks(), "map task");
        checkRange(minActiveReduceTasks, maxActiveReduceTasks, status.getReduceTasks(), "reduce task");
        checkMax(maxSupportedMapTasks, status.getMaxMapTasks(), "supported max map task");
        checkMax(maxSupportedReduceTasks, status.getMaxReduceTasks(), "supported max reduce task");
        result.append(" Map Tasks = ").append(status.getMapTasks());
        result.append(" Reduce Tasks = ").append(status.getReduceTasks());
        return result.toString();
    } catch (IOException e) {
        throw new SFHadoopException("Cannot connect to" + jobtracker, e, this);
    }
}

---