List of usage examples for org.apache.hadoop.mapred TaskAttemptContext getTaskAttemptID
public TaskAttemptID getTaskAttemptID();
From source file:com.asakusafw.runtime.stage.output.LegacyBridgeOutputCommitter.java
License:Apache License
private void logTask(TaskAttemptContext taskContext) { if (LOG.isInfoEnabled()) { LOG.info(MessageFormat.format("Old-style output committer is used in the task: {0} ({1})", taskContext.getJobName(), taskContext.getTaskAttemptID())); }/* w w w.j av a 2 s.c o m*/ }
From source file:com.ibm.bi.dml.runtime.matrix.data.MultipleOutputCommitter.java
License:Open Source License
@Override public void commitTask(TaskAttemptContext context) throws IOException { JobConf conf = context.getJobConf(); TaskAttemptID attemptId = context.getTaskAttemptID(); // get the mapping between index to output filename outputs = MRJobConfiguration.getOutputs(conf); //get temp task output path (compatible with hadoop1 and hadoop2) Path taskOutPath = FileOutputFormat.getWorkOutputPath(conf); FileSystem fs = taskOutPath.getFileSystem(conf); if (!fs.exists(taskOutPath)) throw new IOException("Task output path " + taskOutPath.toString() + "does not exist."); // Move the task outputs to their final places context.getProgressible().progress(); moveFinalTaskOutputs(context, fs, taskOutPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutPath, true)) LOG.debug(//from w w w . j a va 2 s . co m "Failed to delete the temporary output directory of task: " + attemptId + " - " + taskOutPath); }
From source file:com.ibm.bi.dml.runtime.matrix.data.MultipleOutputCommitter.java
License:Open Source License
/** * /*w w w. j a va 2 s. c o m*/ * @param context * @param fs * @param file * @throws IOException */ private void moveFileToDestination(TaskAttemptContext context, FileSystem fs, Path file) throws IOException { JobConf conf = context.getJobConf(); TaskAttemptID attemptId = context.getTaskAttemptID(); //get output index and final destination String taskType = (conf.getBoolean(JobContext.TASK_ISMAP, true)) ? "m" : "r"; String name = file.getName(); int charIx = name.indexOf("-" + taskType + "-"); int index = Integer.parseInt(name.substring(0, charIx)); Path finalPath = new Path(outputs[index], file.getName()); //move file from 'file' to 'finalPath' if (!fs.rename(file, finalPath)) { if (!fs.delete(finalPath, true)) throw new IOException("Failed to delete earlier output " + finalPath + " for rename of " + file + " in task " + attemptId); if (!fs.rename(file, finalPath)) throw new IOException( "Failed to save output " + finalPath + " for rename of " + file + " in task: " + attemptId); } }
From source file:org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil.java
License:Apache License
public static TaskAttemptContext createTaskAttemptContext( org.apache.hadoop.mapreduce.TaskAttemptContext context) { return createTaskAttemptContext(new JobConf(context.getConfiguration()), org.apache.hadoop.mapred.TaskAttemptID.forName(context.getTaskAttemptID().toString()), Reporter.NULL);/*from w w w . ja va 2s. co m*/ }
From source file:org.apache.sysml.runtime.matrix.data.MultipleOutputCommitter.java
License:Apache License
@Override public void commitTask(TaskAttemptContext context) throws IOException { JobConf conf = context.getJobConf(); TaskAttemptID attemptId = context.getTaskAttemptID(); // get the mapping between index to output filename outputs = MRJobConfiguration.getOutputs(conf); // get temp task output path (compatible with hadoop1 and hadoop2) Path taskOutPath = FileOutputFormat.getWorkOutputPath(conf); FileSystem fs = taskOutPath.getFileSystem(conf); if (!fs.exists(taskOutPath)) throw new IOException("Task output path " + taskOutPath.toString() + "does not exist."); // move the task outputs to their final places context.getProgressible().progress(); moveFinalTaskOutputs(context, fs, taskOutPath); // delete the temporary task-specific output directory if (!fs.delete(taskOutPath, true)) LOG.debug(//from w w w . jav a 2s . c om "Failed to delete the temporary output directory of task: " + attemptId + " - " + taskOutPath); }
From source file:org.apache.sysml.runtime.matrix.data.MultipleOutputCommitter.java
License:Apache License
private void moveFileToDestination(TaskAttemptContext context, FileSystem fs, Path file) throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); // get output index and final destination String name = file.getName(); //e.g., 0-r-00000 int index = Integer.parseInt(name.substring(0, name.indexOf("-"))); Path dest = new Path(outputs[index], name); //e.g., outX/0-r-00000 // move file from 'file' to 'finalPath' if (!fs.rename(file, dest)) { if (!fs.delete(dest, true)) throw new IOException("Failed to delete earlier output " + dest + " for rename of " + file + " in task " + attemptId); if (!fs.rename(file, dest)) throw new IOException( "Failed to save output " + dest + " for rename of " + file + " in task: " + attemptId); }//from www . j a va 2 s. c om }
From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java
License:Open Source License
Path getTempTaskOutputPath(TaskAttemptContext taskContext) { JobConf conf = taskContext.getJobConf(); Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path p = new Path(outputPath, (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + "_" + taskContext.getTaskAttemptID().toString())); try {/*from w w w .j av a2s.co m*/ FileSystem fs = p.getFileSystem(conf); return p.makeQualified(fs); } catch (IOException ie) { LOG.warn(StringUtils.stringifyException(ie)); return p; } } return null; }
From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java
License:Open Source License
@Override public void commitTask(TaskAttemptContext context) throws IOException { LOG.info("Commit Called on Task:" + context.getTaskAttemptID().toString()); Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf();/*from w w w . j av a2 s.c o m*/ if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); LOG.info("FileSystem for commit for Task:" + attemptId + " is:" + fs.getUri()); context.getProgressible().progress(); if (fs.exists(taskOutputPath)) { Path jobOutputPath = taskOutputPath.getParent().getParent(); // Move the task outputs to their final place moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutputPath, true)) { LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - " + taskOutputPath); } LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath); } } }
From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java
License:Open Source License
private void moveTaskOutputs(TaskAttemptContext context, FileSystem fs, Path jobOutputDir, Path taskOutput) throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); context.getProgressible().progress(); if (fs.isFile(taskOutput)) { Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTempTaskOutputPath(context)); LOG.info("Renaming:" + taskOutput + " to:" + finalOutputPath); if (!fs.rename(taskOutput, finalOutputPath)) { LOG.info("Rename Failed for:" + taskOutput + " to:" + finalOutputPath + " Trying Delete and then Rename"); if (!fs.delete(finalOutputPath, true)) { throw new IOException("Failed to delete earlier output of task: " + attemptId); }// w w w. ja va 2s .co m LOG.info("Renaming:" + taskOutput + " to: " + finalOutputPath); if (!fs.rename(taskOutput, finalOutputPath)) { throw new IOException("Failed to save output of task: " + attemptId); } } LOG.info("Moved " + taskOutput + " to " + finalOutputPath); } else if (fs.getFileStatus(taskOutput).isDir()) { FileStatus[] paths = fs.listStatus(taskOutput); Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTempTaskOutputPath(context)); LOG.info("Moving " + taskOutput + " to " + finalOutputPath); fs.mkdirs(finalOutputPath); if (paths != null) { for (FileStatus path : paths) { LOG.info("Moving " + path.getPath()); moveTaskOutputs(context, fs, jobOutputDir, path.getPath()); } } } }
From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java
License:Open Source License
@Override public boolean needsTaskCommit(TaskAttemptContext context) throws IOException { LOG.info("COMMITTER- Needs Commit Called on:" + context.getTaskAttemptID().toString()); try {/*from ww w.j a va2 s .c om*/ Path taskOutputPath = getTempTaskOutputPath(context); if (taskOutputPath != null) { context.getProgressible().progress(); FileSystem fs = FileSystem.get(context.getJobConf()); LOG.info("COMMITTER - Default FS is:" + fs.getUri()); // Get the file-system for the task output directory FileSystem fsFromPath = taskOutputPath.getFileSystem(context.getJobConf()); // since task output path is created on demand, // if it exists, task needs a commit LOG.info("COMMITTER - Checking if outputPath Exists:" + taskOutputPath + " for task:" + context.getTaskAttemptID().toString()); if (fs.exists(taskOutputPath)) { LOG.info("Needs Commit Returning TRUE"); return true; } } } catch (IOException ioe) { throw ioe; } LOG.info("COMMITTER Needs Commit Returning FALSE"); return false; }