Example usage for org.apache.hadoop.mapred.join CompositeInputSplit CompositeInputSplit

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred.join CompositeInputSplit CompositeInputSplit.

Prototype

public CompositeInputSplit(int capacity) 

Source Link

Usage

From source file:org.pooledtimeseries.cartesian.CartesianInputFormat.java

License:Apache License

@Override
public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException {

    try {/*  ww w.jav a  2s .  c  o  m*/
        // Get the input splits from both the left and right data sets
        InputSplit[] leftSplits = getInputSplits(conf, conf.get(LEFT_INPUT_FORMAT), conf.get(LEFT_INPUT_PATH),
                numSplits);
        InputSplit[] rightSplits = getInputSplits(conf, conf.get(RIGHT_INPUT_FORMAT),
                conf.get(RIGHT_INPUT_PATH), numSplits);

        // Create our CartesianInputSplits, size equal to left.length *
        // right.length
        CompositeInputSplit[] returnSplits = new CompositeInputSplit[((leftSplits.length
                * (rightSplits.length - 1)) / 2) + leftSplits.length];

        int i = 0;
        // For each of the left input splits
        for (int leftLoop = 0; leftLoop < leftSplits.length; leftLoop++) {
            InputSplit left = leftSplits[leftLoop];
            // For each of the right input splits

            for (int rightLoop = leftLoop; rightLoop < rightSplits.length; rightLoop++) {
                InputSplit right = rightSplits[rightLoop];
                // Create a new composite input split composing of the two

                returnSplits[i] = new CompositeInputSplit(2);
                returnSplits[i].add(left);
                returnSplits[i].add(right);
                ++i;
            }
        }

        // Return the composite splits
        LOG.info("Total splits to process: " + returnSplits.length);
        return returnSplits;
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
        throw new IOException(e);
    }
}

---