Example usage for org.apache.hadoop.mapreduce OutputCommitter needsTaskCommit

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce OutputCommitter needsTaskCommit.

Prototype

public abstract boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException;

Source Link

Document

Check whether task needs a commit.

Usage

From source file:org.apache.hive.hcatalog.mapreduce.DynamicPartitionFileRecordWriterContainer.java

License:Apache License

@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
    Reporter reporter = InternalUtil.createReporter(context);
    for (RecordWriter<? super WritableComparable<?>, ? super Writable> bwriter : baseDynamicWriters.values()) {
        // We are in RecordWriter.close() make sense that the context would be
        // TaskInputOutput.
        bwriter.close(reporter);/*from   ww  w.j  ava2 s.  c  o m*/
    }

    TaskCommitContextRegistry.getInstance().register(context,
            new TaskCommitContextRegistry.TaskCommitterProxy() {
                @Override
                public void abortTask(TaskAttemptContext context) throws IOException {
                    for (Map.Entry<String, OutputJobInfo> outputJobInfoEntry : dynamicOutputJobInfo
                            .entrySet()) {
                        String dynKey = outputJobInfoEntry.getKey();
                        OutputJobInfo outputJobInfo = outputJobInfoEntry.getValue();
                        LOG.info("Aborting task-attempt for " + outputJobInfo.getLocation());
                        baseDynamicCommitters.get(dynKey).abortTask(dynamicContexts.get(dynKey));
                    }
                }

                @Override
                public void commitTask(TaskAttemptContext context) throws IOException {
                    for (Map.Entry<String, OutputJobInfo> outputJobInfoEntry : dynamicOutputJobInfo
                            .entrySet()) {
                        String dynKey = outputJobInfoEntry.getKey();
                        OutputJobInfo outputJobInfo = outputJobInfoEntry.getValue();
                        LOG.info("Committing task-attempt for " + outputJobInfo.getLocation());
                        TaskAttemptContext dynContext = dynamicContexts.get(dynKey);
                        OutputCommitter dynCommitter = baseDynamicCommitters.get(dynKey);
                        if (dynCommitter.needsTaskCommit(dynContext)) {
                            dynCommitter.commitTask(dynContext);
                        } else {
                            LOG.info("Skipping commitTask() for " + outputJobInfo.getLocation());
                        }
                    }
                }
            });
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2Task.java

License:Apache License

/**
 * Commit task./*from  w  ww. jav  a  2s  .co m*/
 *
 * @param outputFormat Output format.
 * @throws IgniteCheckedException In case of Grid exception.
 * @throws IOException In case of IO exception.
 * @throws InterruptedException In case of interrupt.
 */
protected void commit(@Nullable OutputFormat outputFormat)
        throws IgniteCheckedException, IOException, InterruptedException {
    if (hadoopCtx.writer() != null) {
        assert outputFormat != null;

        OutputCommitter outputCommitter = outputFormat.getOutputCommitter(hadoopCtx);

        if (outputCommitter.needsTaskCommit(hadoopCtx))
            outputCommitter.commitTask(hadoopCtx);
    }
}

---

From source file:org.apache.pig.impl.io.PigFile.java

License:Apache License

public void store(DataBag data, FuncSpec storeFuncSpec, PigContext pigContext) throws IOException {
    Configuration conf = ConfigurationUtil.toConfiguration(pigContext.getProperties());
    // create a simulated JobContext
    JobContext jc = HadoopShims.createJobContext(conf, new JobID());
    StoreFuncInterface sfunc = (StoreFuncInterface) PigContext.instantiateFuncFromSpec(storeFuncSpec);
    OutputFormat<?, ?> of = sfunc.getOutputFormat();

    POStore store = new POStore(new OperatorKey());
    store.setSFile(new FileSpec(file, storeFuncSpec));
    PigOutputFormat.setLocation(jc, store);
    OutputCommitter oc;
    // create a simulated TaskAttemptContext

    TaskAttemptContext tac = HadoopShims.createTaskAttemptContext(conf, HadoopShims.getNewTaskAttemptID());
    PigOutputFormat.setLocation(tac, store);
    RecordWriter<?, ?> rw;//  ww w. ja  v a  2  s  .  c  o  m
    try {
        of.checkOutputSpecs(jc);
        oc = of.getOutputCommitter(tac);
        oc.setupJob(jc);
        oc.setupTask(tac);
        rw = of.getRecordWriter(tac);
        sfunc.prepareToWrite(rw);

        for (Iterator<Tuple> it = data.iterator(); it.hasNext();) {
            Tuple row = it.next();
            sfunc.putNext(row);
        }
        rw.close(tac);
    } catch (InterruptedException e) {
        throw new IOException(e);
    }
    if (oc.needsTaskCommit(tac)) {
        oc.commitTask(tac);
    }
    HadoopShims.commitOrCleanup(oc, jc);
}

---

From source file:org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2Task.java

License:Open Source License

/**
 * Commit task./*from  ww w. ja va 2  s  .  co  m*/
 *
 * @param outputFormat Output format.
 * @throws GridException In case of Grid exception.
 * @throws IOException In case of IO exception.
 * @throws InterruptedException In case of interrupt.
 */
protected void commit(@Nullable OutputFormat outputFormat)
        throws GridException, IOException, InterruptedException {
    if (hadoopCtx.writer() != null) {
        assert outputFormat != null;

        OutputCommitter outputCommitter = outputFormat.getOutputCommitter(hadoopCtx);

        if (outputCommitter.needsTaskCommit(hadoopCtx))
            outputCommitter.commitTask(hadoopCtx);
    }
}

---

From source file:org.mrgeo.hadoop.multipleoutputs.DirectoryMultipleOutputsCommitter.java

License:Apache License

@Override
public boolean needsTaskCommit(final TaskAttemptContext taskContext) throws IOException {
    for (final OutputCommitter that : committers) {
        if (that.needsTaskCommit(taskContext)) {
            return true;
        }//from  w  w w .j av  a 2  s .c o m
    }

    return false;
}

---

• «
• 1
• 2
• »