Example usage for org.apache.hadoop.mapred OutputCommitter setupJob

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred OutputCommitter setupJob.

Prototype

@Override
public final void setupJob(org.apache.hadoop.mapreduce.JobContext jobContext) throws IOException 

Source Link

Document

This method implements the new interface by calling the old method.

Usage

From source file:com.ibm.jaql.io.hadoop.DefaultHadoopOutputAdapter.java

License:Apache License

public void open() throws Exception {
    this.conf = new JobConf();
    this.reporter = Reporter.NULL;

    // Some OutputFormats (like FileOutputFormat) require that the job id/task id set.
    // So let's set it for all output formats, just in case they need it too.
    JobID jobid = new JobID("sequential", jobCounter.getAndIncrement());
    TaskAttemptID taskid = new TaskAttemptID(new TaskID(jobid, true, 0), 0);
    conf.set("mapred.task.id", taskid.toString());

    setSequential(conf);//from  w  ww . j a v a2  s . c  o  m

    // Create a task so we can use committers.
    sequentialJob = new ExposeJobContext(conf, jobid);
    sequentialTask = new ExposeTaskAttemptContext(conf, taskid);

    // Give the commiter a chance initialize.
    OutputCommitter committer = conf.getOutputCommitter();
    // FIXME: We skip job setup for now because  
    committer.setupJob(sequentialJob);
    committer.setupTask(sequentialTask);

    if (oFormat instanceof JobConfigurable)
        ((JobConfigurable) oFormat).configure(conf);
}

---

From source file:org.apache.flink.batch.connectors.hive.HiveTableOutputFormat.java

License:Apache License

private HivePartitionWriter writerForLocation(String location) throws IOException {
    JobConf clonedConf = new JobConf(jobConf);
    clonedConf.set(OUTDIR, location);//from  w ww.j  av a2 s . c  om
    OutputFormat outputFormat;
    try {
        StorageDescriptor sd = hiveTablePartition.getStorageDescriptor();
        Class outputFormatClz = Class.forName(sd.getOutputFormat(), true,
                Thread.currentThread().getContextClassLoader());
        outputFormatClz = HiveFileFormatUtils.getOutputFormatSubstitute(outputFormatClz);
        outputFormat = (OutputFormat) outputFormatClz.newInstance();
    } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
        throw new FlinkRuntimeException("Unable to instantiate the hadoop output format", e);
    }
    ReflectionUtils.setConf(outputFormat, clonedConf);
    OutputCommitter outputCommitter = clonedConf.getOutputCommitter();
    JobContext jobContext = new JobContextImpl(clonedConf, new JobID());
    outputCommitter.setupJob(jobContext);
    final boolean isCompressed = clonedConf.getBoolean(HiveConf.ConfVars.COMPRESSRESULT.varname, false);
    if (isCompressed) {
        String codecStr = clonedConf.get(HiveConf.ConfVars.COMPRESSINTERMEDIATECODEC.varname);
        if (!StringUtils.isNullOrWhitespaceOnly(codecStr)) {
            try {
                Class<? extends CompressionCodec> codec = (Class<? extends CompressionCodec>) Class
                        .forName(codecStr, true, Thread.currentThread().getContextClassLoader());
                FileOutputFormat.setOutputCompressorClass(clonedConf, codec);
            } catch (ClassNotFoundException e) {
                throw new RuntimeException(e);
            }
        }
        String typeStr = clonedConf.get(HiveConf.ConfVars.COMPRESSINTERMEDIATETYPE.varname);
        if (!StringUtils.isNullOrWhitespaceOnly(typeStr)) {
            SequenceFile.CompressionType style = SequenceFile.CompressionType.valueOf(typeStr);
            SequenceFileOutputFormat.setOutputCompressionType(clonedConf, style);
        }
    }
    String taskPartition = String.valueOf(clonedConf.getInt("mapreduce.task.partition", -1));
    Path taskPath = FileOutputFormat.getTaskOutputPath(clonedConf, taskPartition);
    FileSinkOperator.RecordWriter recordWriter;
    try {
        recordWriter = HiveFileFormatUtils.getRecordWriter(clonedConf, outputFormat, outputClass, isCompressed,
                tblProperties, taskPath, Reporter.NULL);
    } catch (HiveException e) {
        throw new IOException(e);
    }
    return new HivePartitionWriter(clonedConf, outputFormat, recordWriter, outputCommitter);
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1SetupTask.java

License:Apache License

/** {@inheritDoc} */
@Override//from  w w w.j  a  va 2 s. c  o  m
public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
    HadoopV2TaskContext ctx = (HadoopV2TaskContext) taskCtx;

    try {
        ctx.jobConf().getOutputFormat().checkOutputSpecs(null, ctx.jobConf());

        OutputCommitter committer = ctx.jobConf().getOutputCommitter();

        if (committer != null)
            committer.setupJob(ctx.jobContext());
    } catch (IOException e) {
        throw new IgniteCheckedException(e);
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.v1.GridHadoopV1SetupTask.java

License:Apache License

/** {@inheritDoc} */
@Override//from ww  w. ja v a  2s . c  o m
public void run(GridHadoopTaskContext taskCtx) throws IgniteCheckedException {
    GridHadoopV2TaskContext ctx = (GridHadoopV2TaskContext) taskCtx;

    try {
        ctx.jobConf().getOutputFormat().checkOutputSpecs(null, ctx.jobConf());

        OutputCommitter committer = ctx.jobConf().getOutputCommitter();

        if (committer != null)
            committer.setupJob(ctx.jobContext());
    } catch (IOException e) {
        throw new IgniteCheckedException(e);
    }
}

---

From source file:org.gridgain.grid.kernal.processors.hadoop.v1.GridHadoopV1SetupTask.java

License:Open Source License

/** {@inheritDoc} */
@Override/*from   w w w  . j  a v a2s.co m*/
public void run(GridHadoopTaskContext taskCtx) throws GridException {
    GridHadoopV2TaskContext ctx = (GridHadoopV2TaskContext) taskCtx;

    try {
        ctx.jobConf().getOutputFormat().checkOutputSpecs(null, ctx.jobConf());

        OutputCommitter committer = ctx.jobConf().getOutputCommitter();

        if (committer != null)
            committer.setupJob(ctx.jobContext());
    } catch (IOException e) {
        throw new GridException(e);
    }
}

---