Example usage for org.apache.hadoop.mapreduce TaskAttemptContext getTaskAttemptID

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext getTaskAttemptID.

Prototype

public TaskAttemptID getTaskAttemptID();

Source Link

Document

Get the unique name for this task attempt.

Usage

From source file:com.splicemachine.stream.output.SpliceOutputCommitter.java

License:Apache License

@Override
public void commitTask(TaskAttemptContext taskContext) throws IOException {
    if (LOG.isDebugEnabled())
        SpliceLogUtils.debug(LOG, "commitTask " + taskContext.getTaskAttemptID());
    TxnView txn = taskAttemptMap.remove(taskContext.getTaskAttemptID());
    if (txn == null)
        throw new IOException(
                "no transaction associated with task attempt Id " + taskContext.getTaskAttemptID());
    SIDriver.driver().lifecycleManager().commit(txn.getTxnId());
    if (LOG.isDebugEnabled())
        SpliceLogUtils.debug(LOG, "commitTxn=%s and destinationTable=%s", txn, destinationTable);
}

From source file:com.splicemachine.stream.output.SpliceOutputCommitter.java

License:Apache License

@Override
public void abortTask(TaskAttemptContext taskContext) throws IOException {
    if (LOG.isDebugEnabled())
        SpliceLogUtils.debug(LOG, "abortTask " + taskContext.getTaskAttemptID());
    TxnView txn = taskAttemptMap.remove(taskContext.getTaskAttemptID());
    if (txn == null)
        throw new IOException(
                "no transaction associated with task attempt Id " + taskContext.getTaskAttemptID());
    SIDriver.driver().lifecycleManager().rollback(txn.getTxnId());
}

From source file:com.zjy.mongo.output.MongoOutputCommitter.java

License:Apache License

/**
 * Get the Path to where temporary files should be stored for a
 * TaskAttempt, whose TaskAttemptContext is provided.
 *
 * @param context the TaskAttemptContext.
 * @return the Path to the temporary file for the TaskAttempt.
 *//* w  ww. j a  va2s  .c o  m*/
public static Path getTaskAttemptPath(final TaskAttemptContext context) {
    Configuration config = context.getConfiguration();
    // Try to use the following base temporary directories, in this order:
    // 1. New-style option for task tmp dir
    // 2. Old-style option for task tmp dir
    // 3. Hadoop system-wide tmp dir
    // 4. /tmp
    String basePath = config.get("mapreduce.task.tmp.dir",
            config.get("mapred.child.tmp", config.get("hadoop.tmp.dir", "/tmp")));
    // Hadoop Paths always use "/" as a directory separator.
    return new Path(
            String.format("%s/%s/%s/_out", basePath, context.getTaskAttemptID().toString(), TEMP_DIR_NAME));
}

From source file:cz.seznam.euphoria.hadoop.output.DataSinkOutputFormat.java

License:Apache License

private void instantiateWriter(TaskAttemptContext tac) throws IOException {
    if (writer == null) {
        instantiateSink(tac);//ww  w. j  a  va 2 s  .  co  m
        writer = sink.openWriter(tac.getTaskAttemptID().getTaskID().getId());
    }
}

From source file:cz.seznam.euphoria.hadoop.output.TestDataSinkOutputFormat.java

License:Apache License

private TaskAttemptContext mockContext(Configuration conf, int taskId) {
    TaskAttemptContext ret = mock(TaskAttemptContext.class);
    TaskAttemptID mockAttemptId = mock(TaskAttemptID.class);
    TaskID mockTaskId = mock(TaskID.class);
    when(ret.getConfiguration()).thenReturn(conf);
    when(ret.getTaskAttemptID()).thenReturn(mockAttemptId);
    when(mockAttemptId.getTaskID()).thenReturn(mockTaskId);
    when(mockTaskId.getId()).thenReturn(taskId);
    return ret;//from  ww w  . j a va2s  . c om
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputCommitter.java

License:Apache License

/**
 * Create a file output committer//from ww  w. j  a va 2 s  .  c  o m
 *
 * @param outputPath the job's output path
 * @param context the task's context
 * @throws IOException
 */
public HirodsFileOutputCommitter(Path outputPath, Path tempPath, TaskAttemptContext context)
        throws IOException {
    if (outputPath != null && tempPath != null) {
        this.outputPath = outputPath;
        this.outputFileSystem = outputPath.getFileSystem(context.getConfiguration());
        this.tempPath = tempPath;
        this.workFileSystem = tempPath.getFileSystem(context.getConfiguration());
        this.workPath = new Path(tempPath, (HirodsFileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + "_"
                + context.getTaskAttemptID().toString())).makeQualified(this.workFileSystem);
    }
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputCommitter.java

License:Apache License

/**
 * Move the files from the work directory to the job output directory
 *
 * @param context the task context/*w ww  .j a  va 2 s.  co  m*/
 */
public void commitTask(TaskAttemptContext context) throws IOException {
    TaskAttemptID attemptId = context.getTaskAttemptID();
    if (this.workPath != null) {
        context.progress();
        if (this.workFileSystem.exists(this.workPath)) {
            // Move the task outputs to their final place
            moveTaskOutputsToIRODS(context, this.outputFileSystem, this.outputPath, this.workFileSystem,
                    this.workPath);
            // Delete the temporary task-specific output directory
            if (!this.workFileSystem.delete(this.workPath, true)) {
                LOG.warn("Failed to delete the temporary output" + " directory of task: " + attemptId + " - "
                        + this.workPath);
            }
            LOG.info("Saved output of task '" + attemptId + "' to " + this.outputPath);
        }
    }
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputFormat.java

License:Apache License

/**
 * Generate a unique filename, based on the task id, name, and extension
 *
 * @param context the task that is calling this
 * @param name the base filename//  w ww  . ja  v a 2s  . co  m
 * @param extension the filename extension
 * @return a string like $name-[mr]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context, String name, String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    StringBuilder result = new StringBuilder();
    result.append(name);
    result.append('-');
    result.append(taskId.isMap() ? 'm' : 'r');
    result.append('-');
    result.append(NUMBER_FORMAT.format(partition));
    result.append(extension);
    return result.toString();
}

From source file:fi.tkk.ics.hadoop.bam.cli.Utils.java

License:Open Source License

/** Returns a name that mergeInto() will recognize as a file to be merged.
 *
 * The filename is the value of WORK_FILENAME_PROPERTY surrounded by
 * basePrefix and basePostfix, followed by the part number and with the
 * given extension./*w ww.j  av a  2 s.co  m*/
 */
public static Path getMergeableWorkFile(Path directory, String basePrefix, String basePostfix,
        TaskAttemptContext ctx, String extension) {
    return new Path(directory,
            basePrefix + ContextUtil.getConfiguration(ctx).get(WORK_FILENAME_PROPERTY) + basePostfix + "-"
                    + String.format("%06d", ctx.getTaskAttemptID().getTaskID().getId())
                    + (extension.isEmpty() ? extension : "." + extension));
}

From source file:gobblin.runtime.mapreduce.GobblinOutputCommitter.java

License:Apache License

@Override
public void commitTask(TaskAttemptContext arg0) throws IOException {
    String taskAttemptId = arg0.getTaskAttemptID().toString();
    LOG.info("Committing task attempt: " + taskAttemptId);
    this.attemptIdToMultiTaskAttempt.get(taskAttemptId).commit();
}