Example usage for org.apache.hadoop.mapreduce OutputFormat checkOutputSpecs

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce OutputFormat checkOutputSpecs.

Prototype

public abstract void checkOutputSpecs(JobContext context) throws IOException, InterruptedException;

Source Link

Document

Check for validity of the output-specification for the job.

Usage

From source file:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat.java

License:Apache License

private void checkOutputSpecsHelper(List<POStore> stores, JobContext jobcontext)
        throws IOException, InterruptedException {
    for (POStore store : stores) {
        // make a copy of the original JobContext so that
        // each OutputFormat get a different copy 
        JobContext jobContextCopy = HadoopShims.createJobContext(jobcontext.getConfiguration(),
                jobcontext.getJobID());/*from w  w w.j a  va2 s  .c  om*/

        // set output location
        PigOutputFormat.setLocation(jobContextCopy, store);

        StoreFuncInterface sFunc = store.getStoreFunc();
        OutputFormat of = sFunc.getOutputFormat();

        // The above call should have update the conf in the JobContext
        // to have the output location - now call checkOutputSpecs()
        try {
            of.checkOutputSpecs(jobContextCopy);
        } catch (IOException ioe) {
            boolean shouldThrowException = true;
            if (sFunc instanceof OverwritableStoreFunc) {
                if (((OverwritableStoreFunc) sFunc).shouldOverwrite()) {
                    if (ioe instanceof FileAlreadyExistsException
                            || ioe instanceof org.apache.hadoop.fs.FileAlreadyExistsException) {
                        shouldThrowException = false;
                    }
                }
            }
            if (shouldThrowException)
                throw ioe;
        }
    }
}

---

From source file:org.apache.pig.impl.io.PigFile.java

License:Apache License

public void store(DataBag data, FuncSpec storeFuncSpec, PigContext pigContext) throws IOException {
    Configuration conf = ConfigurationUtil.toConfiguration(pigContext.getProperties());
    // create a simulated JobContext
    JobContext jc = HadoopShims.createJobContext(conf, new JobID());
    StoreFuncInterface sfunc = (StoreFuncInterface) PigContext.instantiateFuncFromSpec(storeFuncSpec);
    OutputFormat<?, ?> of = sfunc.getOutputFormat();

    POStore store = new POStore(new OperatorKey());
    store.setSFile(new FileSpec(file, storeFuncSpec));
    PigOutputFormat.setLocation(jc, store);
    OutputCommitter oc;/*  ww  w.jav a2  s .c  o  m*/
    // create a simulated TaskAttemptContext

    TaskAttemptContext tac = HadoopShims.createTaskAttemptContext(conf, HadoopShims.getNewTaskAttemptID());
    PigOutputFormat.setLocation(tac, store);
    RecordWriter<?, ?> rw;
    try {
        of.checkOutputSpecs(jc);
        oc = of.getOutputCommitter(tac);
        oc.setupJob(jc);
        oc.setupTask(tac);
        rw = of.getRecordWriter(tac);
        sfunc.prepareToWrite(rw);

        for (Iterator<Tuple> it = data.iterator(); it.hasNext();) {
            Tuple row = it.next();
            sfunc.putNext(row);
        }
        rw.close(tac);
    } catch (InterruptedException e) {
        throw new IOException(e);
    }
    if (oc.needsTaskCommit(tac)) {
        oc.commitTask(tac);
    }
    HadoopShims.commitOrCleanup(oc, jc);
}

---

From source file:org.elasticsearch.hadoop.mr.MultiOutputFormat.java

License:Apache License

@Override
public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
    List<OutputFormat> formats = getNewApiFormats(CompatHandler.jobContext(context).getConfiguration());
    for (OutputFormat format : formats) {
        format.checkOutputSpecs(context);
    }//ww w .j  a  va 2s . com
}

---

From source file:org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2SetupTask.java

License:Open Source License

/** {@inheritDoc} */
@SuppressWarnings("ConstantConditions")
@Override/*w w  w.j  a v a 2 s.co m*/
protected void run0(GridHadoopV2TaskContext taskCtx) throws GridException {
    try {
        JobContextImpl jobCtx = taskCtx.jobContext();

        OutputFormat outputFormat = getOutputFormat(jobCtx);

        outputFormat.checkOutputSpecs(jobCtx);

        OutputCommitter committer = outputFormat.getOutputCommitter(hadoopContext());

        if (committer != null)
            committer.setupJob(jobCtx);
    } catch (ClassNotFoundException | IOException e) {
        throw new GridException(e);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();

        throw new GridInterruptedException(e);
    }
}

---

• «
• 1
• 2
• »