Example usage for org.apache.hadoop.mapreduce.lib.join CompositeInputSplit add

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.join CompositeInputSplit add.

Prototype

public void add(InputSplit s) throws IOException, InterruptedException 

Source Link

Document

Add an InputSplit to this collection.

Usage

From source file:ca.uwaterloo.iss4e.hadoop.io.CartesianInputFormat.java

License:Open Source License

@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {

    try {/*from ww  w  .  jav  a2s  .  c o m*/
        // Get the input splits from both the left and right data sets
        Configuration conf = job.getConfiguration();
        List<InputSplit> leftSplits = getInputSplits(job, conf.get(LEFT_INPUT_FORMAT),
                new Path(conf.get(LEFT_INPUT_PATH)));
        List<InputSplit> rightSplits = getInputSplits(job, conf.get(RIGHT_INPUT_FORMAT),
                new Path(conf.get(RIGHT_INPUT_PATH)));

        // Create our CompositeInputSplits, size equal to left.length *
        // right.length
        List<InputSplit> compoisteInputSplits = new ArrayList<InputSplit>();

        // For each of the left input splits
        for (InputSplit left : leftSplits) {
            // For each of the right input splits
            for (InputSplit right : rightSplits) {
                // Create a new composite input split composing of the
                // two
                CompositeInputSplit returnSplits = new CompositeInputSplit(2);
                returnSplits.add(left);
                returnSplits.add(right);
                compoisteInputSplits.add(returnSplits);
            }
        }

        // Return the composite splits
        LOG.info("Total CompositeSplits to process: " + compoisteInputSplits.size());
        return compoisteInputSplits;
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
        throw new IOException(e);
    } catch (InterruptedException e) {
        e.printStackTrace();
        throw new IOException(e);
    }
}

---