Example usage for org.apache.hadoop.mapreduce.lib.join CompositeInputSplit CompositeInputSplit

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.join CompositeInputSplit CompositeInputSplit.

Prototype

public CompositeInputSplit(int capacity) 

Source Link

Usage

From source file:ca.uwaterloo.iss4e.hadoop.io.CartesianInputFormat.java

License:Open Source License

@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {

    try {/*from   ww  w .  j  a  va2 s.co  m*/
        // Get the input splits from both the left and right data sets
        Configuration conf = job.getConfiguration();
        List<InputSplit> leftSplits = getInputSplits(job, conf.get(LEFT_INPUT_FORMAT),
                new Path(conf.get(LEFT_INPUT_PATH)));
        List<InputSplit> rightSplits = getInputSplits(job, conf.get(RIGHT_INPUT_FORMAT),
                new Path(conf.get(RIGHT_INPUT_PATH)));

        // Create our CompositeInputSplits, size equal to left.length *
        // right.length
        List<InputSplit> compoisteInputSplits = new ArrayList<InputSplit>();

        // For each of the left input splits
        for (InputSplit left : leftSplits) {
            // For each of the right input splits
            for (InputSplit right : rightSplits) {
                // Create a new composite input split composing of the
                // two
                CompositeInputSplit returnSplits = new CompositeInputSplit(2);
                returnSplits.add(left);
                returnSplits.add(right);
                compoisteInputSplits.add(returnSplits);
            }
        }

        // Return the composite splits
        LOG.info("Total CompositeSplits to process: " + compoisteInputSplits.size());
        return compoisteInputSplits;
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
        throw new IOException(e);
    } catch (InterruptedException e) {
        e.printStackTrace();
        throw new IOException(e);
    }
}

---