Example usage for org.apache.hadoop.mapred.lib CombineFileSplit getLengths

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred.lib CombineFileSplit getLengths.

Prototype

public long[] getLengths() 

Source Link

Document

Returns an array containing the lengths of the files in the split

Usage

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapred.AbstractGFRecordReader.java

License:Apache License

/**
 * Initializes instance of record reader using file split and job
 * configuration// w  w w . j a va2 s .  com
 * 
 * @param split
 * @param conf
 * @throws IOException
 */
public void initialize(CombineFileSplit split, JobConf conf) throws IOException {
    CombineFileSplit cSplit = (CombineFileSplit) split;
    Path[] path = cSplit.getPaths();
    long[] start = cSplit.getStartOffsets();
    long[] len = cSplit.getLengths();

    FileSystem fs = cSplit.getPath(0).getFileSystem(conf);
    this.splitIterator = HDFSSplitIterator.newInstance(fs, path, start, len, 0l, 0l);
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapred.GFInputFormat.java

License:Apache License

/**
 * Creates an input split for every block occupied by hoplogs of the input
 * regions//from   www .jav  a2s  .c om
 * 
 * @param job 
 * @param hoplogs
 * @return array of input splits of type file input split
 * @throws IOException
 */
private InputSplit[] createSplits(JobConf job, Collection<FileStatus> hoplogs) throws IOException {
    if (hoplogs == null || hoplogs.isEmpty()) {
        return new InputSplit[0];
    }

    HoplogOptimizedSplitter splitter = new HoplogOptimizedSplitter(hoplogs);
    List<org.apache.hadoop.mapreduce.InputSplit> mr2Splits = splitter.getOptimizedSplits(conf);
    InputSplit[] splits = new InputSplit[mr2Splits.size()];
    int i = 0;
    for (org.apache.hadoop.mapreduce.InputSplit inputSplit : mr2Splits) {
        org.apache.hadoop.mapreduce.lib.input.CombineFileSplit mr2Spit;
        mr2Spit = (org.apache.hadoop.mapreduce.lib.input.CombineFileSplit) inputSplit;

        CombineFileSplit split = new CombineFileSplit(job, mr2Spit.getPaths(), mr2Spit.getStartOffsets(),
                mr2Spit.getLengths(), mr2Spit.getLocations());
        splits[i] = split;
        i++;
    }

    return splits;
}

---