Example usage for org.apache.hadoop.mapred RunningJob reduceProgress

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred RunningJob reduceProgress.

Prototype

public float reduceProgress() throws IOException;

Source Link

Document

Get the progress of the job's reduce-tasks, as a float between 0.0 and 1.0.

Usage

From source file:org.godhuli.rhipe.FileUtils.java

License:Apache License

public byte[] getDetailedInfoForJob(String jd) throws Exception {
    org.apache.hadoop.mapred.JobID jj = org.apache.hadoop.mapred.JobID.forName(jd);
    if (jj == null)
        throw new IOException("Jobtracker could not find jobID: " + jd);
    org.apache.hadoop.mapred.RunningJob rj = jclient.getJob(jj);
    if (rj == null)
        throw new IOException(
                "No such job: " + jd + " available, wrong job? or try the History Viewer (see the Web UI) ");
    String jobfile = rj.getJobFile();
    String jobname = rj.getJobName();
    org.apache.hadoop.mapred.Counters cc = rj.getCounters();
    long startsec = getStart(jclient, jj);
    REXP allCounters = FileUtils.buildlistFromOldCounter(cc, 0);
    int jobs = rj.getJobState();
    String jobss = null;//w  w w.  j a  v  a 2s  .  c  o m
    if (jobs == JobStatus.FAILED)
        jobss = "FAILED";
    else if (jobs == JobStatus.KILLED)
        jobss = "KILLED";
    else if (jobs == JobStatus.PREP)
        jobss = "PREP";
    else if (jobs == JobStatus.RUNNING)
        jobss = "RUNNING";
    else if (jobs == JobStatus.SUCCEEDED)
        jobss = "SUCCEEDED";
    float mapprog = rj.mapProgress(), reduprog = rj.reduceProgress();

    REXP.Builder thevals = REXP.newBuilder();
    thevals.setRclass(REXP.RClass.LIST);
    thevals.addRexpValue(RObjects.makeStringVector(new String[] { jobss }));
    thevals.addRexpValue(RObjects.buildDoubleVector(new double[] { startsec }));
    thevals.addRexpValue(RObjects.buildDoubleVector(new double[] { (double) mapprog, (double) reduprog }));
    thevals.addRexpValue(allCounters);
    thevals.addRexpValue(RObjects.makeStringVector(rj.getTrackingURL()));
    thevals.addRexpValue(RObjects.makeStringVector(new String[] { jobname }));
    thevals.addRexpValue(RObjects.makeStringVector(new String[] { jobfile }));

    org.apache.hadoop.mapred.TaskReport[] maptr = jclient.getMapTaskReports(jj);
    REXP.Builder thevalsA = REXP.newBuilder();
    thevalsA.setRclass(REXP.RClass.LIST);
    for (TaskReport t : maptr) {
        thevalsA.addRexpValue(TaskReportToRexp(t));
    }
    thevals.addRexpValue(thevalsA.build());

    org.apache.hadoop.mapred.TaskReport[] redtr = jclient.getReduceTaskReports(jj);
    REXP.Builder thevalsB = REXP.newBuilder();
    thevalsB.setRclass(REXP.RClass.LIST);
    for (TaskReport t : redtr) {
        thevalsB.addRexpValue(TaskReportToRexp(t));
    }
    thevals.addRexpValue(thevalsB.build());

    return thevals.build().toByteArray();
}

---

From source file:org.pentaho.di.job.entries.hadoopjobexecutor.JobEntryHadoopJobExecutor.java

License:Apache License

public void printJobStatus(RunningJob runningJob) throws IOException {
    if (log.isBasic()) {
        float setupPercent = runningJob.setupProgress() * 100f;
        float mapPercent = runningJob.mapProgress() * 100f;
        float reducePercent = runningJob.reduceProgress() * 100f;
        logBasic(BaseMessages.getString(PKG, "JobEntryHadoopJobExecutor.RunningPercent", setupPercent,
                mapPercent, reducePercent));
    }//from w ww  . j  a v  a2s  .c o  m
}

---

From source file:setest.FormatStorageMR.java

License:Open Source License

public static void main(String[] args) throws Exception {

    if (args.length != 2) {
        System.out.println("FormatStorageMR <input> <output>");
        System.exit(-1);/*from   w w w  . j  a va2  s.  c  om*/
    }

    JobConf conf = new JobConf(FormatStorageMR.class);

    conf.setJobName("FormatStorageMR");

    conf.setNumMapTasks(1);

    conf.setOutputKeyClass(LongWritable.class);
    conf.setOutputValueClass(Unit.Record.class);

    conf.setMapperClass(FormatStorageTestMapper.class);
    conf.setReducerClass(FormatStorageTestReducer.class);

    conf.setInputFormat(FormatStorageInputFormat.class);
    conf.setOutputFormat(FormatStorageOutputFormat.class);
    conf.set("mapred.output.compress", "flase");

    Head head = new Head();
    initHead(head);

    head.toJobConf(conf);

    FileInputFormat.setInputPaths(conf, args[0]);
    Path outputPath = new Path(args[1]);
    FileOutputFormat.setOutputPath(conf, outputPath);

    FileSystem fs = outputPath.getFileSystem(conf);
    fs.delete(outputPath, true);

    JobClient jc = new JobClient(conf);
    RunningJob rj = null;
    rj = jc.submitJob(conf);

    String lastReport = "";
    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS");
    long reportTime = System.currentTimeMillis();
    long maxReportInterval = 3 * 1000;
    while (!rj.isComplete()) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        }

        int mapProgress = Math.round(rj.mapProgress() * 100);
        int reduceProgress = Math.round(rj.reduceProgress() * 100);

        String report = " map = " + mapProgress + "%,  reduce = " + reduceProgress + "%";

        if (!report.equals(lastReport) || System.currentTimeMillis() >= reportTime + maxReportInterval) {

            String output = dateFormat.format(Calendar.getInstance().getTime()) + report;
            System.out.println(output);
            lastReport = report;
            reportTime = System.currentTimeMillis();
        }
    }

    System.exit(0);

}

---

• «
• 1
• 2
• 3
• »