Example usage for org.apache.hadoop.mapreduce.lib.input CombineFileRecordReader CombineFileRecordReader

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.input CombineFileRecordReader CombineFileRecordReader.

Prototype

public CombineFileRecordReader(CombineFileSplit split, TaskAttemptContext context,
        Class<? extends RecordReader<K, V>> rrClass) throws IOException

Source Link

Document

A generic RecordReader that can hand out different recordReaders for each chunk in the CombineFileSplit.

Usage

From source file:com.bonc.mr_roamRecognition_hjpt.comm.FirstJobCombineTextInputFormat.java

License:Apache License

public RecordReader<Text, Text> createRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException {
    return new CombineFileRecordReader<Text, Text>((CombineFileSplit) split, context,
            TextRecordReaderWrapper.class);
}

From source file:com.cloudera.science.avro2parquet.CombinedAvroKeyInputFormat.java

License:Apache License

@SuppressWarnings("unchecked")
@Override/*ww w  . ja  v  a2  s . c o  m*/
public RecordReader<AvroKey<T>, NullWritable> createRecordReader(InputSplit inputSplit,
        TaskAttemptContext context) throws IOException {
    Schema readerSchema = AvroJob.getInputKeySchema(context.getConfiguration());
    if (null == readerSchema) {
        LOG.warn("Reader schema was not set. Use AvroJob.setInputKeySchema() if desired.");
        LOG.info("Using a reader schema equal to the writer schema.");
    }

    Object c = CombinedAvroKeyRecordReader.class;
    return new CombineFileRecordReader<AvroKey<T>, NullWritable>((CombineFileSplit) inputSplit, context,
            (Class<? extends RecordReader<AvroKey<T>, NullWritable>>) c);
}

From source file:com.conversantmedia.mapreduce.input.CombineTextFileInputFormat.java

License:Apache License

@Override
public RecordReader<FileLineWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException {
    return new CombineFileRecordReader<>((CombineFileSplit) split, context, FileLineWritableRecordReader.class);
}

From source file:com.conversantmedia.mapreduce.io.avro.CombineAvroKeyFileInputFormat.java

License:Apache License

@Override
@SuppressWarnings({ "unchecked", "rawtypes" })
public RecordReader<AvroKey<T>, NullWritable> createRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException {
    return new CombineFileRecordReader((CombineFileSplit) split, context, DelegatingAvroRecordReader.class);
}

From source file:com.linkedin.whiteelephant.mapreduce.lib.input.CombineDocumentFileFormat.java

License:Apache License

@Override
public RecordReader<Text, BytesWritable> createRecordReader(InputSplit arg0, TaskAttemptContext arg1)
        throws IOException {
    return new CombineFileRecordReader<Text, BytesWritable>((CombineFileSplit) arg0, arg1,
            WholeFileRecordReader.class);
}

From source file:com.linkedin.whiteelephant.mapreduce.lib.input.CombinedTextInputFormat.java

License:Apache License

@Override
public RecordReader<LongWritable, Text> createRecordReader(org.apache.hadoop.mapreduce.InputSplit inputSplit,
        org.apache.hadoop.mapreduce.TaskAttemptContext taskAttemptContext) throws IOException {
    return new CombineFileRecordReader<LongWritable, Text>((CombineFileSplit) inputSplit, taskAttemptContext,
            MyLineRecordReader.class);
}

From source file:com.marcolotz.lung.io.inputFormat.MultipleFilesInputFormat.java

License:Creative Commons License

/**
 * Creates a CombineFileRecordReader to read each file assigned to this
 * InputSplit. Note, that unlike ordinary InputSplits, split must be a
 * CombineFileSplit, and therefore is expected to specify multiple files.
 * //  ww  w.  j  av a2 s .  c  o m
 * @param split
 *            The InputSplit to read. Throws an IllegalArgumentException if
 *            this is not a CombineFileSplit.
 * @param context
 *            The context for this task.
 * @return a CombineFileRecordReader to process each file in split. It will
 *         read each file with a WholeFileRecordReader.
 * @throws IOException
 *             if there is an error.
 */
@Override
public RecordReader<NullWritable, BytesWritable> createRecordReader(InputSplit split,
        TaskAttemptContext context) throws IOException {

    if (!(split instanceof CombineFileSplit)) {
        throw new IllegalArgumentException("split must be a CombineFileSplit");
    }
    return new CombineFileRecordReader<NullWritable, BytesWritable>((CombineFileSplit) split, context,
            MultipleFilesRecordReader.class);
}

From source file:com.moz.fiji.mapreduce.input.impl.WholeFileInputFormat.java

License:Apache License

/**
 * Creates a CombineFileRecordReader to read each file assigned to this InputSplit.
 * Note, that unlike ordinary InputSplits, split must be a CombineFileSplit, and therefore
 * is expected to specify multiple files.
 *
 * @param split The InputSplit to read.  Throws an IllegalArgumentException if this is
 *        not a CombineFileSplit./* w ww .ja v a 2s  . c o  m*/
 * @param context The context for this task.
 * @return a CombineFileRecordReader to process each file in split.
 *         It will read each file with a WholeFileRecordReader.
 * @throws IOException if there is an error.
 */
@Override
public RecordReader<Text, Text> createRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException {
    if (!(split instanceof CombineFileSplit)) {
        throw new IllegalArgumentException("split must be a CombineFileSplit");
    }
    return new CombineFileRecordReader<Text, Text>((CombineFileSplit) split, context,
            WholeFileRecordReader.class);
}

From source file:com.rw.legion.input.CombineCsvInputFormat.java

License:Apache License

public RecordReader<NullWritable, LegionRecord> createRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException {

    return new CombineFileRecordReader<NullWritable, LegionRecord>((CombineFileSplit) split, context,
            LegionRecordReaderWrapper.class);
}

From source file:edu.gslis.streamcorpus.ThriftFileInputFormat.java

License:Apache License

@Override
public RecordReader<Text, StreamItemWritable> createRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException {
    return new CombineFileRecordReader<Text, StreamItemWritable>((CombineFileSplit) split, context,
            ThriftRecordReader.class);
}