Example usage for org.apache.hadoop.mapreduce.lib.input SequenceFileRecordReader SequenceFileRecordReader

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.input SequenceFileRecordReader SequenceFileRecordReader.

Prototype

SequenceFileRecordReader

Source Link

Usage

From source file:com.alexholmes.hadooputils.combine.seqfile.mapreduce.CombineSequenceFileInputFormat.java

License:Apache License

@Override
public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException {
    return new CommonCombineFileRecordReader<K, V>(
            new CommonCombineFileRecordReader.RecordReaderEngineerer<K, V>() {
                @Override/*from  w  w  w .  j a v a  2 s .  co m*/
                public RecordReader<K, V> createRecordReader() {
                    return new SequenceFileRecordReader<K, V>();
                }
            });
}

---

From source file:com.conductor.hadoop.WritableValueInputFormat.java

License:Apache License

@Override
public RecordReader<NullWritable, V> createRecordReader(final InputSplit split,
        final TaskAttemptContext context) throws IOException, InterruptedException {
    final SequenceFileRecordReader<NullWritable, V> reader = new SequenceFileRecordReader<NullWritable, V>();
    reader.initialize(split, context);//  ww  w . ja v  a  2  s . co  m
    return reader;
}

---

From source file:com.conductor.s3.S3SequenceFileInputFormat.java

License:Apache License

@Override
public RecordReader<K, V> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException {
    return new SequenceFileRecordReader<K, V>();
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.lib.DynamicInputChunk.java

License:Apache License

private void openForRead(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
    reader = new SequenceFileRecordReader<K, V>();
    reader.initialize(/* w w  w  .j av  a 2  s  .  c om*/
            new FileSplit(chunkFilePath, 0, DistCpUtils.getFileSize(chunkFilePath, chunkSet.getConf()), null),
            taskAttemptContext);
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.UniformSizeInputFormat.java

License:Apache License

/**
 * Implementation of InputFormat::createRecordReader().
 * @param split: The split for which the RecordReader is sought.
 * @param context: The context of the current task-attempt.
 * @return A SequenceFileRecordReader instance, (since the copy-listing is a
 * simple sequence-file.)/*  w  w w.  ja  v a 2 s  . c  o  m*/
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public RecordReader<Text, FileStatus> createRecordReader(InputSplit split, TaskAttemptContext context)
        throws IOException, InterruptedException {
    return new SequenceFileRecordReader<Text, FileStatus>();
}

---

From source file:jp.ac.u.tokyo.m.dpc.pig.udf.load.path.MultiFileInputFormat.java

License:Apache License

/**
 * return reader which accepted extension. <br>
 * <br>/*from  w  w w . j  a  v  a 2  s .c  om*/
 * ???? reader ??? <br>
 * SequenceFile : .seq? <br>
 *  : .txt??????????? <br>
 */
@Override
public RecordReader<LongWritable, Text> createRecordReader(InputSplit aSplit, TaskAttemptContext aContext) {
    // FileSplit ??????
    if (aSplit instanceof FileSplit) {
        FileSplit tFileSplit = (FileSplit) aSplit;
        Path tPath = tFileSplit.getPath();
        String tFileName = tPath.getName();
        String tExtension = tFileName.substring(tFileName.lastIndexOf("."));
        // ???? RecordReader ?
        if (tExtension != null) {
            if (tExtension.equals(PathConstants.EXTENSION_TEXT)) {
                return new LineRecordReader();
            } else if (tExtension.equals(PathConstants.EXTENSION_SEQUENCE)) {
                return new SequenceFileRecordReader<LongWritable, Text>();
            }
        }
    }
    // ? Text ? RecordReader
    return new LineRecordReader();
}

---

From source file:org.apache.beam.sdk.io.hadoop.format.HadoopFormatIOSequenceFileTest.java

License:Apache License

private Stream<KV<Text, LongWritable>> extractResultsFromFile(String fileName) {
    try (SequenceFileRecordReader<Text, LongWritable> reader = new SequenceFileRecordReader<>()) {
        Path path = new Path(fileName);
        TaskAttemptContext taskContext = HadoopFormats.createTaskAttemptContext(new Configuration(),
                new JobID("readJob", 0), 0);
        reader.initialize(new FileSplit(path, 0L, Long.MAX_VALUE, new String[] { "localhost" }), taskContext);
        List<KV<Text, LongWritable>> result = new ArrayList<>();

        while (reader.nextKeyValue()) {
            result.add(KV.of(new Text(reader.getCurrentKey().toString()),
                    new LongWritable(reader.getCurrentValue().get())));
        }//from  w w  w  .  ja v a 2s.co m

        return result.stream();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

---

From source file:org.godhuli.rhipe.SequenceFileAsRHTextRecordReader.java

License:Apache License

public SequenceFileAsRHTextRecordReader() throws IOException {
    sequenceFileRecordReader = new SequenceFileRecordReader<WritableComparable<?>, Writable>();
}

---