Example usage for org.apache.hadoop.mapred.lib CombineFileSplit getLocations

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred.lib CombineFileSplit getLocations.

Prototype

public String[] getLocations() throws IOException 

Source Link

Document

Returns all the Paths where this input-split resides

Usage

From source file:cascading.tap.hadoop.io.CombineFileRecordReaderWrapper.java

License:Open Source License

public CombineFileRecordReaderWrapper(CombineFileSplit split, Configuration conf, Reporter reporter,
        Integer idx) throws Exception {
    FileSplit fileSplit = new FileSplit(split.getPath(idx), split.getOffset(idx), split.getLength(idx),
            split.getLocations());

    Class<?> clz = conf.getClass(INDIVIDUAL_INPUT_FORMAT, null);
    FileInputFormat<K, V> inputFormat = (FileInputFormat<K, V>) clz.newInstance();

    if (inputFormat instanceof Configurable)
        ((Configurable) inputFormat).setConf(conf);

    delegate = inputFormat.getRecordReader(fileSplit, (JobConf) conf, reporter);
}

---

From source file:com.alexholmes.hadooputils.combine.common.mapred.SplitMetricsCombineInputFormat.java

License:Apache License

public static String extractLocation(CombineFileSplit split) throws IOException {
    if (split.getLocations() == null || split.getLocations().length == 0) {
        return null;
    }//from  www . ja v a2 s  . com
    return split.getLocations()[0];
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapred.GFInputFormat.java

License:Apache License

/**
 * Creates an input split for every block occupied by hoplogs of the input
 * regions/* w  w w .  j ava 2s  . co m*/
 * 
 * @param job 
 * @param hoplogs
 * @return array of input splits of type file input split
 * @throws IOException
 */
private InputSplit[] createSplits(JobConf job, Collection<FileStatus> hoplogs) throws IOException {
    if (hoplogs == null || hoplogs.isEmpty()) {
        return new InputSplit[0];
    }

    HoplogOptimizedSplitter splitter = new HoplogOptimizedSplitter(hoplogs);
    List<org.apache.hadoop.mapreduce.InputSplit> mr2Splits = splitter.getOptimizedSplits(conf);
    InputSplit[] splits = new InputSplit[mr2Splits.size()];
    int i = 0;
    for (org.apache.hadoop.mapreduce.InputSplit inputSplit : mr2Splits) {
        org.apache.hadoop.mapreduce.lib.input.CombineFileSplit mr2Spit;
        mr2Spit = (org.apache.hadoop.mapreduce.lib.input.CombineFileSplit) inputSplit;

        CombineFileSplit split = new CombineFileSplit(job, mr2Spit.getPaths(), mr2Spit.getStartOffsets(),
                mr2Spit.getLengths(), mr2Spit.getLocations());
        splits[i] = split;
        i++;
    }

    return splits;
}

---

From source file:org.icgc.dcc.release.core.hadoop.CombineFileRecordReaderWrapper.java

License:Open Source License

protected CombineFileRecordReaderWrapper(FileInputFormat<K, V> inputFormat, CombineFileSplit split,
        Configuration conf, Reporter reporter, Integer index) throws IOException {
    val fileSplit = new FileSplit(split.getPath(index), split.getOffset(index), split.getLength(index),
            split.getLocations());

    delegate = inputFormat.getRecordReader(fileSplit, (JobConf) conf, reporter);
}

---