List of usage examples for org.apache.hadoop.mapred TaskAttemptContext getJobConf
public JobConf getJobConf();
From source file:com.ibm.bi.dml.runtime.matrix.data.MultipleOutputCommitter.java
License:Open Source License
@Override public void commitTask(TaskAttemptContext context) throws IOException { JobConf conf = context.getJobConf(); TaskAttemptID attemptId = context.getTaskAttemptID(); // get the mapping between index to output filename outputs = MRJobConfiguration.getOutputs(conf); //get temp task output path (compatible with hadoop1 and hadoop2) Path taskOutPath = FileOutputFormat.getWorkOutputPath(conf); FileSystem fs = taskOutPath.getFileSystem(conf); if (!fs.exists(taskOutPath)) throw new IOException("Task output path " + taskOutPath.toString() + "does not exist."); // Move the task outputs to their final places context.getProgressible().progress(); moveFinalTaskOutputs(context, fs, taskOutPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutPath, true)) LOG.debug(//from ww w. j a v a 2s . c o m "Failed to delete the temporary output directory of task: " + attemptId + " - " + taskOutPath); }
From source file:com.ibm.bi.dml.runtime.matrix.data.MultipleOutputCommitter.java
License:Open Source License
/** * //from ww w.j a v a 2 s . co m * @param context * @param fs * @param file * @throws IOException */ private void moveFileToDestination(TaskAttemptContext context, FileSystem fs, Path file) throws IOException { JobConf conf = context.getJobConf(); TaskAttemptID attemptId = context.getTaskAttemptID(); //get output index and final destination String taskType = (conf.getBoolean(JobContext.TASK_ISMAP, true)) ? "m" : "r"; String name = file.getName(); int charIx = name.indexOf("-" + taskType + "-"); int index = Integer.parseInt(name.substring(0, charIx)); Path finalPath = new Path(outputs[index], file.getName()); //move file from 'file' to 'finalPath' if (!fs.rename(file, finalPath)) { if (!fs.delete(finalPath, true)) throw new IOException("Failed to delete earlier output " + finalPath + " for rename of " + file + " in task " + attemptId); if (!fs.rename(file, finalPath)) throw new IOException( "Failed to save output " + finalPath + " for rename of " + file + " in task: " + attemptId); } }
From source file:com.ibm.jaql.io.hadoop.DirectFileOutputCommiter.java
License:Apache License
@Override public void abortTask(TaskAttemptContext context) throws IOException { Path outputPath = FileOutputFormat.getOutputPath(context.getJobConf()); // TODO: LOG.warn("partial write of "+outputPath+" What should we do with partial output when writing directly?"); // Path taskOutputPath = getTempTaskOutputPath(context); // try {//from ww w.j a va2 s .com // if (taskOutputPath != null) { // FileSystem fs = taskOutputPath.getFileSystem(context.getJobConf()); // context.getProgressible().progress(); // fs.delete(taskOutputPath, true); // } // } catch (IOException ie) { // LOG.warn("Error discarding output" + StringUtils.stringifyException(ie)); // } }
From source file:org.apache.sysml.runtime.matrix.data.MultipleOutputCommitter.java
License:Apache License
@Override public void commitTask(TaskAttemptContext context) throws IOException { JobConf conf = context.getJobConf(); TaskAttemptID attemptId = context.getTaskAttemptID(); // get the mapping between index to output filename outputs = MRJobConfiguration.getOutputs(conf); // get temp task output path (compatible with hadoop1 and hadoop2) Path taskOutPath = FileOutputFormat.getWorkOutputPath(conf); FileSystem fs = taskOutPath.getFileSystem(conf); if (!fs.exists(taskOutPath)) throw new IOException("Task output path " + taskOutPath.toString() + "does not exist."); // move the task outputs to their final places context.getProgressible().progress(); moveFinalTaskOutputs(context, fs, taskOutPath); // delete the temporary task-specific output directory if (!fs.delete(taskOutPath, true)) LOG.debug(/*from w w w .ja v a2 s . co m*/ "Failed to delete the temporary output directory of task: " + attemptId + " - " + taskOutPath); }
From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java
License:Open Source License
Path getTempTaskOutputPath(TaskAttemptContext taskContext) { JobConf conf = taskContext.getJobConf(); Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path p = new Path(outputPath, (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + "_" + taskContext.getTaskAttemptID().toString())); try {/*from w w w .java2s.co m*/ FileSystem fs = p.getFileSystem(conf); return p.makeQualified(fs); } catch (IOException ie) { LOG.warn(StringUtils.stringifyException(ie)); return p; } } return null; }
From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java
License:Open Source License
@Override public void commitTask(TaskAttemptContext context) throws IOException { LOG.info("Commit Called on Task:" + context.getTaskAttemptID().toString()); Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); LOG.info("FileSystem for commit for Task:" + attemptId + " is:" + fs.getUri()); context.getProgressible().progress(); if (fs.exists(taskOutputPath)) { Path jobOutputPath = taskOutputPath.getParent().getParent(); // Move the task outputs to their final place moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutputPath, true)) { LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - " + taskOutputPath); }/* w ww . j a va 2s. c o m*/ LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath); } } }
From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java
License:Open Source License
@Override public boolean needsTaskCommit(TaskAttemptContext context) throws IOException { LOG.info("COMMITTER- Needs Commit Called on:" + context.getTaskAttemptID().toString()); try {/*from w w w . j a v a 2s. co m*/ Path taskOutputPath = getTempTaskOutputPath(context); if (taskOutputPath != null) { context.getProgressible().progress(); FileSystem fs = FileSystem.get(context.getJobConf()); LOG.info("COMMITTER - Default FS is:" + fs.getUri()); // Get the file-system for the task output directory FileSystem fsFromPath = taskOutputPath.getFileSystem(context.getJobConf()); // since task output path is created on demand, // if it exists, task needs a commit LOG.info("COMMITTER - Checking if outputPath Exists:" + taskOutputPath + " for task:" + context.getTaskAttemptID().toString()); if (fs.exists(taskOutputPath)) { LOG.info("Needs Commit Returning TRUE"); return true; } } } catch (IOException ioe) { throw ioe; } LOG.info("COMMITTER Needs Commit Returning FALSE"); return false; }
From source file:tachyon.client.keyvalue.hadoop.KeyValueOutputCommitter.java
License:Apache License
/** * {@inheritDoc}/*w w w . j a va2 s . c o m*/ * <p> * Merges the completed key-value store under the task's temporary output directory to the * key-value store created in {@link #setupJob(JobContext)}. */ @Override public void commitTask(TaskAttemptContext context) throws IOException { try { KEY_VALUE_STORES.merge(getTaskAttemptOutputURI(context), getOutputURI(context.getJobConf())); } catch (TachyonException e) { throw new IOException(e); } }