List of usage examples for org.apache.hadoop.mapreduce TaskAttemptContext setStatus
public void setStatus(String msg);
From source file:format.WikipediaPageInputFormat.java
License:Apache License
@Override public RecordReader<LongWritable, WikipediaPage> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { context.setStatus(split.toString()); return new WikipediaPageRecordReader(context.getConfiguration()); }
From source file:gov.llnl.ontology.text.hbase.XMLRecordReader.java
License:Open Source License
/** * Extract the {@link Path} for the file to be processed by this {@link * XMLRecordReader}./*from www . j av a 2 s . c o m*/ */ public void initialize(InputSplit isplit, TaskAttemptContext context) throws IOException, InterruptedException { Configuration config = context.getConfiguration(); // Get the file stream for the xml file. FileSplit split = (FileSplit) isplit; Path file = split.getPath(); FileSystem fs = file.getFileSystem(config); fsin = (useGzip) ? new GZIPInputStream(fs.open(split.getPath())) : fs.open(split.getPath()); fsin = new BufferedInputStream(fsin); // Setup the limits of the xml file. start = split.getStart(); end = start + split.getLength(); pos = 0; // Get the xml document delmiters for this xml file. if (!config.get(DELIMITER_TAG).equals("")) { startTag = ("<" + config.get(DELIMITER_TAG)).getBytes(); endTag = ("</" + config.get(DELIMITER_TAG) + ">").getBytes(); } else { String fileNameBase = file.getName().replace(".xml", ""); startTag = ("<" + fileNameBase).getBytes(); endTag = ("</" + fileNameBase).getBytes(); } context.setStatus(file.getName() + " " + pos + " " + end); }
From source file:io.druid.indexer.JobHelper.java
License:Apache License
public static ProgressIndicator progressIndicatorForContext(final TaskAttemptContext context) { return new ProgressIndicator() { @Override/*from w ww. j a va2 s . com*/ public void progress() { context.progress(); } @Override public void start() { context.progress(); context.setStatus("STARTED"); } @Override public void stop() { context.progress(); context.setStatus("STOPPED"); } @Override public void startSection(String section) { context.progress(); context.setStatus(String.format("STARTED [%s]", section)); } @Override public void progressSection(String section, String message) { log.info("Progress message for section [%s] : [%s]", section, message); context.progress(); context.setStatus(String.format("PROGRESS [%s]", section)); } @Override public void stopSection(String section) { context.progress(); context.setStatus(String.format("STOPPED [%s]", section)); } }; }
From source file:io.ssc.trackthetrackers.extraction.hadoop.io.ArcInputFormat.java
License:Open Source License
/** * Returns the <code>RecordReader</code> for reading the arc file. * *//*from w ww .j a v a 2 s. co m*/ public RecordReader<Text, ArcRecord> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException { context.setStatus(split.toString()); return new ArcRecordReader(); }
From source file:ir.ac.ut.snl.mrcd.MyFileInputFormat.java
@Override public RecordReader<Text, ShortestPathTuple> createRecordReader(InputSplit is, TaskAttemptContext tac) throws IOException, InterruptedException { tac.setStatus(is.toString()); return new MyRecordReader(tac.getConfiguration(), (FileSplit) is); }
From source file:org.apache.avro.mapreduce.lib.input.AvroInputFormat.java
License:Apache License
@Override public RecordReader<AvroWrapper<T>, NullWritable> createRecordReader(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { context.setStatus(inputSplit.toString()); return new AvroRecordReader<T>(context.getConfiguration(), (FileSplit) inputSplit); }
From source file:org.apache.crunch.types.avro.AvroInputFormat.java
License:Apache License
@Override public RecordReader<AvroWrapper<T>, NullWritable> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { context.setStatus(split.toString()); String jsonSchema = context.getConfiguration().get(AvroJob.INPUT_SCHEMA); Schema schema = new Schema.Parser().parse(jsonSchema); return new AvroRecordReader<T>(schema); }
From source file:org.apache.giraph.utils.LoggerUtils.java
License:Apache License
/** * Set Hadoop status message./*from w w w . j a v a 2 s . c o m*/ * * NOTE: In theory this function could get folded in to the callsites, but * the issue is that some Hadoop jars, e.g. 0.23 and 2.0.0, don't actually * throw IOException on setStatus while others do. This makes wrapping it in a * try/catch cause a compile error on those Hadoops. With this function every * caller sees a method that throws IOException. In case it doesn't actually, * there is no more compiler error because not throwing a decalred exception * is at best a warning. * * @param context Context to set the status with * @param message Message to set status with * @throws IOException If something goes wrong with setting status message */ private static void setStatus(TaskAttemptContext context, String message) throws IOException { context.setStatus(message); }
From source file:org.apache.hcatalog.rcfile.RCFileMapReduceInputFormat.java
License:Apache License
@Override public RecordReader<LongWritable, BytesRefArrayWritable> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { context.setStatus(split.toString()); return new RCFileMapReduceRecordReader<LongWritable, BytesRefArrayWritable>(); }
From source file:org.apache.pig.builtin.AvroStorage.java
License:Apache License
/** * @see org.apache.pig.LoadFunc#getInputFormat() *///from www . ja va 2s. com @Override public InputFormat<NullWritable, GenericData.Record> getInputFormat() throws IOException { return new org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigFileInputFormat<NullWritable, GenericData.Record>() { @Override public RecordReader<NullWritable, GenericData.Record> createRecordReader(final InputSplit is, final TaskAttemptContext tc) throws IOException, InterruptedException { Schema s = getInputAvroSchema(); RecordReader<NullWritable, GenericData.Record> rr = null; if (s.getType() == Type.ARRAY) { rr = new AvroArrayReader(s); } else { rr = new AvroRecordReader(s); } rr.initialize(is, tc); tc.setStatus(is.toString()); return rr; } }; }