Example usage for org.apache.hadoop.mapred.lib CombineFileSplit getStartOffsets

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred.lib CombineFileSplit getStartOffsets.

Prototype

public long[] getStartOffsets() 

Source Link

Document

Returns an array containing the start offsets of the files in the split

Usage

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapred.AbstractGFRecordReader.java

License:Apache License

/**
 * Initializes instance of record reader using file split and job
 * configuration//ww w .  ja  va  2  s  .c  o  m
 * 
 * @param split
 * @param conf
 * @throws IOException
 */
public void initialize(CombineFileSplit split, JobConf conf) throws IOException {
    CombineFileSplit cSplit = (CombineFileSplit) split;
    Path[] path = cSplit.getPaths();
    long[] start = cSplit.getStartOffsets();
    long[] len = cSplit.getLengths();

    FileSystem fs = cSplit.getPath(0).getFileSystem(conf);
    this.splitIterator = HDFSSplitIterator.newInstance(fs, path, start, len, 0l, 0l);
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapred.GFInputFormat.java

License:Apache License

/**
 * Creates an input split for every block occupied by hoplogs of the input
 * regions// w w w  .jav a 2  s.c om
 * 
 * @param job 
 * @param hoplogs
 * @return array of input splits of type file input split
 * @throws IOException
 */
private InputSplit[] createSplits(JobConf job, Collection<FileStatus> hoplogs) throws IOException {
    if (hoplogs == null || hoplogs.isEmpty()) {
        return new InputSplit[0];
    }

    HoplogOptimizedSplitter splitter = new HoplogOptimizedSplitter(hoplogs);
    List<org.apache.hadoop.mapreduce.InputSplit> mr2Splits = splitter.getOptimizedSplits(conf);
    InputSplit[] splits = new InputSplit[mr2Splits.size()];
    int i = 0;
    for (org.apache.hadoop.mapreduce.InputSplit inputSplit : mr2Splits) {
        org.apache.hadoop.mapreduce.lib.input.CombineFileSplit mr2Spit;
        mr2Spit = (org.apache.hadoop.mapreduce.lib.input.CombineFileSplit) inputSplit;

        CombineFileSplit split = new CombineFileSplit(job, mr2Spit.getPaths(), mr2Spit.getStartOffsets(),
                mr2Spit.getLengths(), mr2Spit.getLocations());
        splits[i] = split;
        i++;
    }

    return splits;
}

---

From source file:com.ricemap.spateDB.mapred.SpatialRecordReader.java

License:Apache License

/**
 * Initialize from an input split/*ww w .ja  v a  2  s . c om*/
 * @param split
 * @param conf
 * @param reporter
 * @param index
 * @throws IOException
 */
public SpatialRecordReader(CombineFileSplit split, Configuration conf, Reporter reporter, Integer index)
        throws IOException {
    this(conf, split.getStartOffsets()[index], split.getLength(index), split.getPath(index));
}

---