Example usage for org.apache.hadoop.mapred TaskAttemptContext getProgressible

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred TaskAttemptContext getProgressible.

Prototype

public Progressable getProgressible();

Source Link

Usage

From source file:com.ibm.bi.dml.runtime.matrix.data.MultipleOutputCommitter.java

License:Open Source License

@Override
public void commitTask(TaskAttemptContext context) throws IOException {
    JobConf conf = context.getJobConf();
    TaskAttemptID attemptId = context.getTaskAttemptID();

    // get the mapping between index to output filename
    outputs = MRJobConfiguration.getOutputs(conf);

    //get temp task output path (compatible with hadoop1 and hadoop2)
    Path taskOutPath = FileOutputFormat.getWorkOutputPath(conf);
    FileSystem fs = taskOutPath.getFileSystem(conf);
    if (!fs.exists(taskOutPath))
        throw new IOException("Task output path " + taskOutPath.toString() + "does not exist.");

    // Move the task outputs to their final places
    context.getProgressible().progress();
    moveFinalTaskOutputs(context, fs, taskOutPath);

    // Delete the temporary task-specific output directory
    if (!fs.delete(taskOutPath, true))
        LOG.debug(//from   ww w  . ja  v a2 s.co  m
                "Failed to delete the temporary output directory of task: " + attemptId + " - " + taskOutPath);
}

---

From source file:com.ibm.bi.dml.runtime.matrix.data.MultipleOutputCommitter.java

License:Open Source License

/**
 * /* w  w w.  j  a  va  2  s .com*/
 * @param context
 * @param fs
 * @param taskOutput
 * @throws IOException
 */
private void moveFinalTaskOutputs(TaskAttemptContext context, FileSystem fs, Path taskOutput)
        throws IOException {
    context.getProgressible().progress();

    if (fs.getFileStatus(taskOutput).isDirectory()) {
        FileStatus[] files = fs.listStatus(taskOutput);
        if (files != null)
            for (FileStatus file : files) //for all files
                if (!file.isDirectory()) //skip directories
                    moveFileToDestination(context, fs, file.getPath());
    }
}

---

From source file:org.apache.sysml.runtime.matrix.data.MultipleOutputCommitter.java

License:Apache License

@Override
public void commitTask(TaskAttemptContext context) throws IOException {
    JobConf conf = context.getJobConf();
    TaskAttemptID attemptId = context.getTaskAttemptID();

    // get the mapping between index to output filename
    outputs = MRJobConfiguration.getOutputs(conf);

    // get temp task output path (compatible with hadoop1 and hadoop2)
    Path taskOutPath = FileOutputFormat.getWorkOutputPath(conf);
    FileSystem fs = taskOutPath.getFileSystem(conf);
    if (!fs.exists(taskOutPath))
        throw new IOException("Task output path " + taskOutPath.toString() + "does not exist.");

    // move the task outputs to their final places
    context.getProgressible().progress();
    moveFinalTaskOutputs(context, fs, taskOutPath);

    // delete the temporary task-specific output directory
    if (!fs.delete(taskOutPath, true))
        LOG.debug(//from www .j ava  2 s  .co m
                "Failed to delete the temporary output directory of task: " + attemptId + " - " + taskOutPath);
}

---

From source file:org.apache.sysml.runtime.matrix.data.MultipleOutputCommitter.java

License:Apache License

private void moveFinalTaskOutputs(TaskAttemptContext context, FileSystem fs, Path taskOutput)
        throws IOException {
    context.getProgressible().progress();

    if (fs.getFileStatus(taskOutput).isDirectory()) {
        FileStatus[] files = fs.listStatus(taskOutput);
        if (files != null)
            for (FileStatus file : files) //for all files
                if (!file.isDirectory()) //skip directories
                    moveFileToDestination(context, fs, file.getPath());
    }//from  w  w  w  .  j a v a  2  s  .  co m
}

---

From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java

License:Open Source License

@Override
public void commitTask(TaskAttemptContext context) throws IOException {
    LOG.info("Commit Called on Task:" + context.getTaskAttemptID().toString());
    Path taskOutputPath = getTempTaskOutputPath(context);
    TaskAttemptID attemptId = context.getTaskAttemptID();
    JobConf job = context.getJobConf();/*w w w  .ja  va 2 s  . c  om*/
    if (taskOutputPath != null) {
        FileSystem fs = taskOutputPath.getFileSystem(job);
        LOG.info("FileSystem for commit for Task:" + attemptId + " is:" + fs.getUri());
        context.getProgressible().progress();
        if (fs.exists(taskOutputPath)) {
            Path jobOutputPath = taskOutputPath.getParent().getParent();
            // Move the task outputs to their final place
            moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath);
            // Delete the temporary task-specific output directory
            if (!fs.delete(taskOutputPath, true)) {
                LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - "
                        + taskOutputPath);
            }
            LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath);
        }
    }
}

---

From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java

License:Open Source License

private void moveTaskOutputs(TaskAttemptContext context, FileSystem fs, Path jobOutputDir, Path taskOutput)
        throws IOException {
    TaskAttemptID attemptId = context.getTaskAttemptID();
    context.getProgressible().progress();
    if (fs.isFile(taskOutput)) {
        Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTempTaskOutputPath(context));
        LOG.info("Renaming:" + taskOutput + " to:" + finalOutputPath);
        if (!fs.rename(taskOutput, finalOutputPath)) {
            LOG.info("Rename Failed for:" + taskOutput + " to:" + finalOutputPath
                    + " Trying Delete and then Rename");
            if (!fs.delete(finalOutputPath, true)) {
                throw new IOException("Failed to delete earlier output of task: " + attemptId);
            }//from  www  .ja va  2s . c om
            LOG.info("Renaming:" + taskOutput + " to: " + finalOutputPath);
            if (!fs.rename(taskOutput, finalOutputPath)) {
                throw new IOException("Failed to save output of task: " + attemptId);
            }
        }
        LOG.info("Moved " + taskOutput + " to " + finalOutputPath);
    } else if (fs.getFileStatus(taskOutput).isDir()) {
        FileStatus[] paths = fs.listStatus(taskOutput);
        Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTempTaskOutputPath(context));
        LOG.info("Moving " + taskOutput + " to " + finalOutputPath);
        fs.mkdirs(finalOutputPath);
        if (paths != null) {
            for (FileStatus path : paths) {
                LOG.info("Moving " + path.getPath());
                moveTaskOutputs(context, fs, jobOutputDir, path.getPath());
            }
        }
    }
}

---

From source file:org.commoncrawl.mapred.ec2.parser.OutputCommitter.java

License:Open Source License

@Override
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
    LOG.info("COMMITTER- Needs Commit Called on:" + context.getTaskAttemptID().toString());
    try {/*from   ww  w  . java  2s . com*/
        Path taskOutputPath = getTempTaskOutputPath(context);
        if (taskOutputPath != null) {
            context.getProgressible().progress();
            FileSystem fs = FileSystem.get(context.getJobConf());
            LOG.info("COMMITTER - Default FS is:" + fs.getUri());
            // Get the file-system for the task output directory
            FileSystem fsFromPath = taskOutputPath.getFileSystem(context.getJobConf());
            // since task output path is created on demand, 
            // if it exists, task needs a commit
            LOG.info("COMMITTER - Checking if outputPath Exists:" + taskOutputPath + " for task:"
                    + context.getTaskAttemptID().toString());
            if (fs.exists(taskOutputPath)) {
                LOG.info("Needs Commit Returning TRUE");
                return true;
            }
        }
    } catch (IOException ioe) {
        throw ioe;
    }
    LOG.info("COMMITTER Needs Commit Returning FALSE");
    return false;
}

---