Example usage for org.apache.hadoop.mapreduce TaskAttemptContext setStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskAttemptContext setStatus.

Prototype

public void setStatus(String msg);

Source Link

Document

Set the current status of the task to the given string.

Usage

From source file:org.apache.pig.builtin.TrevniStorage.java

License:Apache License

@Override
public InputFormat<NullWritable, GenericData.Record> getInputFormat() throws IOException {

    class TrevniStorageInputFormat extends PigFileInputFormat<NullWritable, GenericData.Record> {

        @Override//from   www.j a v  a  2  s.co  m
        protected boolean isSplitable(JobContext jc, Path p) {
            return false;
        }

        @Override
        protected List<FileStatus> listStatus(final JobContext job) throws IOException {
            List<FileStatus> results = Lists.newArrayList();
            job.getConfiguration().setBoolean(MRConfiguration.INPUT_DIR_RECURSIVE, true);
            for (FileStatus file : super.listStatus(job)) {
                if (Utils.VISIBLE_FILES.accept(file.getPath())) {
                    results.add(file);
                }
            }
            return results;
        }

        @Override
        public RecordReader<NullWritable, GenericData.Record> createRecordReader(final InputSplit is,
                final TaskAttemptContext tc) throws IOException, InterruptedException {
            RecordReader<NullWritable, GenericData.Record> rr = new RecordReader<NullWritable, GenericData.Record>() {

                private FileSplit fsplit;
                private AvroColumnReader.Params params;
                private AvroColumnReader<GenericData.Record> reader;
                private float rows;
                private long row = 0;
                private GenericData.Record currentRecord = null;

                @Override
                public void close() throws IOException {
                    reader.close();
                }

                @Override
                public NullWritable getCurrentKey() throws IOException, InterruptedException {
                    return NullWritable.get();
                }

                @Override
                public Record getCurrentValue() throws IOException, InterruptedException {
                    return currentRecord;
                }

                @Override
                public float getProgress() throws IOException, InterruptedException {
                    return row / rows;
                }

                @Override
                public void initialize(final InputSplit isplit, final TaskAttemptContext tac)
                        throws IOException, InterruptedException {
                    fsplit = (FileSplit) isplit;
                    params = new AvroColumnReader.Params(
                            new HadoopInput(fsplit.getPath(), tac.getConfiguration()));
                    Schema inputSchema = getInputAvroSchema();
                    params.setSchema(inputSchema);
                    reader = new AvroColumnReader<GenericData.Record>(params);
                    rows = reader.getRowCount();
                }

                @Override
                public boolean nextKeyValue() throws IOException, InterruptedException {
                    if (reader.hasNext()) {
                        currentRecord = reader.next();
                        row++;
                        return true;
                    } else {
                        return false;
                    }
                }
            };

            // rr.initialize(is, tc);
            tc.setStatus(is.toString());
            return rr;
        }

    }

    return new TrevniStorageInputFormat();

}

---

From source file:org.apache.solr.hadoop.BatchWriter.java

License:Apache License

public synchronized void close(TaskAttemptContext context)
        throws InterruptedException, SolrServerException, IOException {

    if (batchPool != null) {
        context.setStatus("Waiting for batches to complete");
        batchPool.shutdown();// www.  j  ava 2  s  .  c  o  m

        while (!batchPool.isTerminated()) {
            LOG.info(String.format("Waiting for %d items and %d threads to finish executing",
                    batchPool.getQueue().size(), batchPool.getActiveCount()));
            batchPool.awaitTermination(5, TimeUnit.SECONDS);
        }
    }
    context.setStatus("Committing Solr Phase 1");
    solr.commit(true, false);
    context.setStatus("Optimizing Solr");
    int maxSegments = context.getConfiguration().getInt(SolrOutputFormat.SOLR_RECORD_WRITER_MAX_SEGMENTS, 1);
    LOG.info("Optimizing Solr: forcing merge down to {} segments", maxSegments);
    long start = System.currentTimeMillis();
    solr.optimize(true, false, maxSegments);
    context.getCounter(SolrCounters.class.getName(), SolrCounters.PHYSICAL_REDUCER_MERGE_TIME.toString())
            .increment(System.currentTimeMillis() - start);
    float secs = (System.currentTimeMillis() - start) / 1000.0f;
    LOG.info("Optimizing Solr: done forcing merge down to {} segments in {} secs", maxSegments, secs);
    context.setStatus("Committing Solr Phase 2");
    solr.commit(true, false);
    context.setStatus("Shutting down Solr");
    solr.shutdown();
}

---

From source file:org.apache.solr.hadoop.SolrRecordWriter.java

License:Apache License

@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
    if (context != null) {
        heartBeater.setProgress(context);
    }//from  w  w  w . j av  a2s .  c  o m
    try {
        heartBeater.needHeartBeat();
        if (batch.size() > 0) {
            batchWriter.queueBatch(batch);
            numDocsWritten += batch.size();
            batch.clear();
        }
        LOG.info("docsWritten: {}", numDocsWritten);
        batchWriter.close(context);
        //      if (outputZipFile) {
        //        context.setStatus("Writing Zip");
        //        packZipFile(); // Written to the perm location
        //      } else {
        //        context.setStatus("Copying Index");
        //        fs.completeLocalOutput(perm, temp); // copy to dfs
        //      }
    } catch (Exception e) {
        if (e instanceof IOException) {
            throw (IOException) e;
        }
        throw new IOException(e);
    } finally {
        heartBeater.cancelHeartBeat();
        heartBeater.close();
        //      File tempFile = new File(temp.toString());
        //      if (tempFile.exists()) {
        //        FileUtils.forceDelete(new File(temp.toString()));
        //      }
    }

    context.setStatus("Done");
}

---

From source file:org.apache.trevni.avro.mapreduce.AvroTrevniRecordReaderBase.java

License:Apache License

/** {@inheritDoc} */
@Override/*from   w w  w . j  a va2s  .c  om*/
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    final FileSplit file = (FileSplit) inputSplit;
    context.setStatus(file.toString());

    final AvroColumnReader.Params params = new AvroColumnReader.Params(
            new HadoopInput(file.getPath(), context.getConfiguration()));
    params.setModel(ReflectData.get());

    if (AvroJob.getInputKeySchema(context.getConfiguration()) != null) {
        params.setSchema(AvroJob.getInputKeySchema(context.getConfiguration()));
    }

    reader = new AvroColumnReader<T>(params);
    rows = reader.getRowCount();
}

---

From source file:org.bdgenomics.adam.io.InterleavedFastqInputFormat.java

License:Apache License

public RecordReader<Void, Text> createRecordReader(InputSplit genericSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    context.setStatus(genericSplit.toString());

    // cast as per example in TextInputFormat
    return new InterleavedFastqRecordReader(context.getConfiguration(), (FileSplit) genericSplit);
}

---

From source file:org.bdgenomics.adam.io.SingleFastqInputFormat.java

License:Apache License

public RecordReader<Void, Text> createRecordReader(InputSplit genericSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    context.setStatus(genericSplit.toString());

    // cast as per example in TextInputFormat
    return new SingleFastqRecordReader(context.getConfiguration(), (FileSplit) genericSplit);
}

---

From source file:org.broadinstitute.sting.gatk.hadoop.NLineXInputFormat.java

License:Apache License

@Override
public RecordReader<LongWritable, Text> createRecordReader(InputSplit genericSplit, TaskAttemptContext context)
        throws IOException {
    context.setStatus(genericSplit.toString());
    return new NLineXRecordReader();
}

---

From source file:org.gbif.ocurrence.index.solr.BatchWriter.java

License:Apache License

public synchronized void close(TaskAttemptContext context, SolrCore core)
        throws InterruptedException, SolrServerException, IOException {

    context.setStatus("Waiting for batches to complete");
    batchPool.shutdown();//from w  ww  .  j  a  va  2  s.  c  o  m

    while (!batchPool.isTerminated()) {
        LOG.info(String.format("Waiting for %d items and %d threads to finish executing",
                batchPool.getQueue().size(), batchPool.getActiveCount()));
        batchPool.awaitTermination(5, TimeUnit.SECONDS);
    }
    //reporter.setStatus("Committing Solr");
    //solr.commit(true, false);
    context.setStatus("Optimizing Solr");
    solr.optimize(true, false, 1);
    context.setStatus("Closing Solr");
    core.close();
}

---

From source file:org.gbif.ocurrence.index.solr.SolrRecordWriter.java

License:Apache License

@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
    if (context != null) {
        heartBeater.setProgress(context);
    }/*from  w  w w .  j av  a2  s.  c  o  m*/
    try {
        heartBeater.needHeartBeat();
        batchWriter.close(context, core);
        if (outputZipFile) {
            context.setStatus("Writing Zip");
            packZipFile(); // Written to the perm location
        } else {
            context.setStatus("Copying Index");
            fs.completeLocalOutput(perm, temp); // copy to dfs
        }
    } catch (Exception e) {
        if (e instanceof IOException) {
            throw (IOException) e;
        }
        throw new IOException(e);
    } finally {
        heartBeater.cancelHeartBeat();
        File tempFile = new File(temp.toString());
        if (tempFile.exists()) {
            FileUtils.forceDelete(new File(temp.toString()));
        }
    }

    context.setStatus("Done");
}

---

From source file:org.godhuli.rhipe.RNLineInputFormat.java

License:Apache License

public RecordReader<RHNumeric, RHText> createRecordReader(InputSplit genericSplit, TaskAttemptContext context)
        throws IOException {
    context.setStatus(genericSplit.toString());
    return new RXLineRecordReader();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »