List of usage examples for org.apache.hadoop.mapred RunningJob mapProgress
public float mapProgress() throws IOException;
From source file:Text2FormatStorageMR.java
License:Open Source License
@SuppressWarnings("deprecation") public static void main(String[] args) throws Exception { if (args.length != 2) { System.out.println("FormatFileMR <input> <output>"); System.exit(-1);/* ww w . j ava 2 s .co m*/ } JobConf conf = new JobConf(FormatStorageMR.class); conf.setJobName("Text2FormatMR"); conf.setNumMapTasks(1); conf.setNumReduceTasks(4); conf.setOutputKeyClass(LongWritable.class); conf.setOutputValueClass(Unit.Record.class); conf.setMapperClass(TextFileTestMapper.class); conf.setReducerClass(FormatFileTestReducer.class); conf.setInputFormat(TextInputFormat.class); conf.setOutputFormat(FormatStorageOutputFormat.class); conf.set("mapred.output.compress", "flase"); Head head = new Head(); initHead(head); head.toJobConf(conf); FileInputFormat.setInputPaths(conf, args[0]); Path outputPath = new Path(args[1]); FileOutputFormat.setOutputPath(conf, outputPath); FileSystem fs = outputPath.getFileSystem(conf); fs.delete(outputPath, true); JobClient jc = new JobClient(conf); RunningJob rj = null; rj = jc.submitJob(conf); String lastReport = ""; SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS"); long reportTime = System.currentTimeMillis(); long maxReportInterval = 3 * 1000; while (!rj.isComplete()) { try { Thread.sleep(1000); } catch (InterruptedException e) { } int mapProgress = Math.round(rj.mapProgress() * 100); int reduceProgress = Math.round(rj.reduceProgress() * 100); String report = " map = " + mapProgress + "%, reduce = " + reduceProgress + "%"; if (!report.equals(lastReport) || System.currentTimeMillis() >= reportTime + maxReportInterval) { String output = dateFormat.format(Calendar.getInstance().getTime()) + report; System.out.println(output); lastReport = report; reportTime = System.currentTimeMillis(); } } System.exit(0); }
From source file:Text2ColumntStorageMR.java
License:Open Source License
@SuppressWarnings("deprecation") public static void main(String[] args) throws Exception { if (args.length != 3) { System.out.println("Text2ColumnStorageMR <input> <output> <columnStorageMode>"); System.exit(-1);//from w w w .j av a2 s . c o m } JobConf conf = new JobConf(Text2ColumntStorageMR.class); conf.setJobName("Text2ColumnStorageMR"); conf.setNumMapTasks(1); conf.setNumReduceTasks(4); conf.setOutputKeyClass(LongWritable.class); conf.setOutputValueClass(Unit.Record.class); conf.setMapperClass(TextFileMapper.class); conf.setReducerClass(ColumnStorageReducer.class); conf.setInputFormat(TextInputFormat.class); conf.setOutputFormat((Class<? extends OutputFormat>) ColumnStorageHiveOutputFormat.class); conf.set("mapred.output.compress", "flase"); Head head = new Head(); initHead(head); head.toJobConf(conf); int bt = Integer.valueOf(args[2]); FileInputFormat.setInputPaths(conf, args[0]); Path outputPath = new Path(args[1]); FileOutputFormat.setOutputPath(conf, outputPath); FileSystem fs = outputPath.getFileSystem(conf); fs.delete(outputPath, true); JobClient jc = new JobClient(conf); RunningJob rj = null; rj = jc.submitJob(conf); String lastReport = ""; SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS"); long reportTime = System.currentTimeMillis(); long maxReportInterval = 3 * 1000; while (!rj.isComplete()) { try { Thread.sleep(1000); } catch (InterruptedException e) { } int mapProgress = Math.round(rj.mapProgress() * 100); int reduceProgress = Math.round(rj.reduceProgress() * 100); String report = " map = " + mapProgress + "%, reduce = " + reduceProgress + "%"; if (!report.equals(lastReport) || System.currentTimeMillis() >= reportTime + maxReportInterval) { String output = dateFormat.format(Calendar.getInstance().getTime()) + report; System.out.println(output); lastReport = report; reportTime = System.currentTimeMillis(); } } System.exit(0); }
From source file:FormatStorage2ColumnStorageMR.java
License:Open Source License
@SuppressWarnings("deprecation") public static void main(String[] args) throws Exception { if (args.length != 2) { System.out.println("FormatStorage2ColumnStorageMR <input> <output>"); System.exit(-1);//from ww w . j a v a 2 s . c o m } JobConf conf = new JobConf(FormatStorageMR.class); conf.setJobName("FormatStorage2ColumnStorageMR"); conf.setNumMapTasks(1); conf.setNumReduceTasks(4); conf.setOutputKeyClass(LongWritable.class); conf.setOutputValueClass(Unit.Record.class); conf.setMapperClass(FormatStorageMapper.class); conf.setReducerClass(ColumnStorageReducer.class); conf.setInputFormat(FormatStorageInputFormat.class); conf.set("mapred.output.compress", "flase"); Head head = new Head(); initHead(head); head.toJobConf(conf); FileInputFormat.setInputPaths(conf, args[0]); Path outputPath = new Path(args[1]); FileOutputFormat.setOutputPath(conf, outputPath); FileSystem fs = outputPath.getFileSystem(conf); fs.delete(outputPath, true); JobClient jc = new JobClient(conf); RunningJob rj = null; rj = jc.submitJob(conf); String lastReport = ""; SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS"); long reportTime = System.currentTimeMillis(); long maxReportInterval = 3 * 1000; while (!rj.isComplete()) { try { Thread.sleep(1000); } catch (InterruptedException e) { } int mapProgress = Math.round(rj.mapProgress() * 100); int reduceProgress = Math.round(rj.reduceProgress() * 100); String report = " map = " + mapProgress + "%, reduce = " + reduceProgress + "%"; if (!report.equals(lastReport) || System.currentTimeMillis() >= reportTime + maxReportInterval) { String output = dateFormat.format(Calendar.getInstance().getTime()) + report; System.out.println(output); lastReport = report; reportTime = System.currentTimeMillis(); } } System.exit(0); }
From source file:azkaban.jobtype.MapReduceJobState.java
License:Apache License
public MapReduceJobState(RunningJob runningJob, TaskReport[] mapTaskReport, TaskReport[] reduceTaskReport) throws IOException { jobId = runningJob.getID().toString(); jobName = runningJob.getJobName();/* w ww. j a v a 2s. co m*/ trackingURL = runningJob.getTrackingURL(); isComplete = runningJob.isComplete(); isSuccessful = runningJob.isSuccessful(); mapProgress = runningJob.mapProgress(); reduceProgress = runningJob.reduceProgress(); failureInfo = runningJob.getFailureInfo(); totalMappers = mapTaskReport.length; totalReducers = reduceTaskReport.length; for (TaskReport report : mapTaskReport) { if (report.getStartTime() < jobStartTime || jobStartTime == 0L) { jobStartTime = report.getStartTime(); } TIPStatus status = report.getCurrentStatus(); if (status != TIPStatus.PENDING && status != TIPStatus.RUNNING) { finishedMappersCount++; } } for (TaskReport report : reduceTaskReport) { if (jobLastUpdateTime < report.getFinishTime()) { jobLastUpdateTime = report.getFinishTime(); } TIPStatus status = report.getCurrentStatus(); if (status != TIPStatus.PENDING && status != TIPStatus.RUNNING) { finishedReducersCount++; } } // If not all the reducers are finished. if (finishedReducersCount != reduceTaskReport.length || jobLastUpdateTime == 0) { jobLastUpdateTime = System.currentTimeMillis(); } counters = runningJob.getCounters(); }
From source file:cascading.flow.hadoop.HadoopStepStats.java
License:Open Source License
/** * Returns the underlying Map tasks progress. * <p/>// w ww. ja va2 s . c om * This method is experimental. * * @return float */ public float getMapProgress() { RunningJob runningJob = getRunningJob(); if (runningJob == null) return 0; try { return runningJob.mapProgress(); } catch (IOException exception) { throw new FlowException("unable to get progress"); } }
From source file:co.cask.cdap.app.mapreduce.MRJobClient.java
License:Apache License
/** * @param runId for which information will be returned. * @return a {@link MRJobInfo} containing information about a particular MapReduce program run. * @throws IOException if there is failure to communicate through the JobClient. * @throws NotFoundException if a Job with the given runId is not found. *///from w w w . j av a 2 s .c o m public MRJobInfo getMRJobInfo(Id.Run runId) throws IOException, NotFoundException { Preconditions.checkArgument(ProgramType.MAPREDUCE.equals(runId.getProgram().getType())); JobClient jobClient = new JobClient(hConf); JobStatus[] jobs = jobClient.getAllJobs(); JobStatus thisJob = findJobForRunId(jobs, runId); RunningJob runningJob = jobClient.getJob(thisJob.getJobID()); if (runningJob == null) { throw new IllegalStateException(String.format("JobClient returned null for RunId: '%s', JobId: '%s'", runId, thisJob.getJobID())); } Counters counters = runningJob.getCounters(); TaskReport[] mapTaskReports = jobClient.getMapTaskReports(thisJob.getJobID()); TaskReport[] reduceTaskReports = jobClient.getReduceTaskReports(thisJob.getJobID()); return new MRJobInfo(runningJob.mapProgress(), runningJob.reduceProgress(), groupToMap(counters.getGroup(TaskCounter.class.getName())), toMRTaskInfos(mapTaskReports), toMRTaskInfos(reduceTaskReports), true); }
From source file:com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.JobLayer.java
License:Apache License
public String getJobStatisticsByJobId(String jobTracker, String jobId) throws IOException { StringBuilder jobStatistics = new StringBuilder(); JobClient client = prepareJobClient(jobTracker); JobID id = convertToJobId(jobId);/* w ww . j a v a 2s. c om*/ RunningJob job = client.getJob(id); double mapProgress = job.mapProgress() * 100; double reduceProgress = job.reduceProgress() * 100; String mapPercentage = Double.toString(mapProgress) + "%"; String reducePercentage = Double.toString(reduceProgress) + "%"; jobStatistics.append("<job id='").append(jobId).append("'" + " name='").append(job.getJobName()) .append("'>\n"); jobStatistics.append(" <mapProgress>").append(mapPercentage).append("</mapProgress>\n"); jobStatistics.append(" <reduceProgress>").append(reducePercentage).append("</reduceProgress>\n"); jobStatistics.append(" <complete>").append(job.isComplete()).append("</complete>\n"); jobStatistics.append(" <successful>").append(job.isSuccessful()).append("</successful>\n"); jobStatistics.append(" <url>").append(job.getTrackingURL()).append("</url>\n"); jobStatistics.append("</job>"); return jobStatistics.toString(); }
From source file:com.jackbe.mapreduce.LocalJobManager.java
License:Open Source License
@Override public float getJobProgress(RunningJob job) { if (log.isTraceEnabled()) log.trace("called."); if (job == null) { return -1.0f; }/*from w w w .j a v a2 s . co m*/ try { if (job.mapProgress() == 0.0f) return 0.0f; // Return the progress as the combination of the map and reduce task progress. //TODO: Figure out a better way to convert reduce progress time to mapper time. return ((job.mapProgress() + job.reduceProgress()) / 2.0f) * 100.0f; } catch (IOException e) { log.error("Exception getting job progress: " + e, e); return 0.0f; } }
From source file:com.netflix.lipstick.pigtolipstick.BasicP2LClient.java
License:Apache License
/** * Build a P2jJobStatus object for the map/reduce job with id jobId. * * @param jobId the id of the map/reduce job * @return the newly created P2jJobStatus */// w w w .jav a 2 s . c o m @SuppressWarnings("deprecation") protected P2jJobStatus buildJobStatusMap(String jobId) { JobClient jobClient = PigStats.get().getJobClient(); P2jJobStatus js = jobIdToJobStatusMap.get(jobId); try { RunningJob rj = jobClient.getJob(jobId); if (rj == null) { LOG.warn("Couldn't find job status for jobId=" + jobId); return js; } JobID jobID = rj.getID(); Counters counters = rj.getCounters(); Map<String, P2jCounters> cMap = Maps.newHashMap(); for (Group g : counters) { P2jCounters countersObj = new P2jCounters(); cMap.put(g.getDisplayName(), countersObj); for (Counter c : g) { countersObj.getCounters().put(c.getDisplayName(), c.getValue()); } } js.setCounters(cMap); TaskReport[] mapTaskReport = jobClient.getMapTaskReports(jobID); TaskReport[] reduceTaskReport = jobClient.getReduceTaskReports(jobID); js.setJobName(rj.getJobName()); js.setTrackingUrl(rj.getTrackingURL()); js.setIsComplete(rj.isComplete()); js.setIsSuccessful(rj.isSuccessful()); js.setMapProgress(rj.mapProgress()); js.setReduceProgress(rj.reduceProgress()); js.setTotalMappers(mapTaskReport.length); js.setTotalReducers(reduceTaskReport.length); return js; } catch (IOException e) { LOG.error("Error getting job info.", e); } return null; }
From source file:edu.stolaf.cs.wmrserver.HadoopEngine.java
License:Apache License
public JobStatus getStatus(Submission submission) throws NotFoundException, InternalException { RunningJob job = getJob(submission); JobConf conf = loadJobConfiguration(job); JobStatus status = new JobStatus(); status.setInfo(getInfo(submission, job, conf)); try {//from w w w. ja v a 2s . co m JobClient client = new JobClient(new JobConf()); // Get job state // Thanks to the mentally handicapped switch statement, we have // to use a chain of ifs. Fuck Java. int jobState = job.getJobState(); if (jobState == org.apache.hadoop.mapred.JobStatus.FAILED) status.setState(State.FAILED); else if (jobState == org.apache.hadoop.mapred.JobStatus.SUCCEEDED) status.setState(State.SUCCESSFUL); else if (jobState == org.apache.hadoop.mapred.JobStatus.KILLED) status.setState(State.KILLED); else if (jobState == org.apache.hadoop.mapred.JobStatus.RUNNING) status.setState(State.RUNNING); else status.setState(State.PREP); // Get task counts TaskReport[] mapTaskReports = client.getMapTaskReports(job.getID()); TaskReport[] reduceTaskReports = client.getReduceTaskReports(job.getID()); // Get failed task logs TaskCompletionEvent[] events = job.getTaskCompletionEvents(0); Pair<ArrayList<TaskLog>, ArrayList<TaskLog>> failures; if (events != null) failures = getLogsFromCompletionEvents(events); else failures = getLogsFromHistory(job, new Configuration()); ArrayList<TaskLog> mapFailures = failures.first; ArrayList<TaskLog> reduceFailures = failures.second; // Get other mapper info PhaseStatus mapStatus = new PhaseStatus(); mapStatus.setProgress(job.mapProgress() * 100); if (!mapFailures.isEmpty()) mapStatus.setErrors(getMeaningfulTaskLog(mapFailures)); if (mapTaskReports != null) mapStatus.setTotalTasks(mapTaskReports.length); // TODO: Handle the state in a sane way mapStatus.setState(status.getState()); status.setMapStatus(mapStatus); // Get other reducer info PhaseStatus reduceStatus = new PhaseStatus(); reduceStatus.setProgress(job.reduceProgress() * 100); if (!reduceFailures.isEmpty()) reduceStatus.setErrors(getMeaningfulTaskLog(reduceFailures)); reduceStatus.setState(status.getState()); if (reduceTaskReports != null) reduceStatus.setTotalTasks(reduceTaskReports.length); if (conf != null) reduceStatus.setOutputPath(FileOutputFormat.getOutputPath(conf).toString()); status.setReduceStatus(reduceStatus); } catch (Exception ex) { throw JobServiceHandler.wrapException("Could not get job info.", ex); } return status; }