Example usage for org.apache.hadoop.io.erasurecode.rawcoder NativeRSRawErasureCoderFactory NativeRSRawErasureCoderFactory

Introduction

In this page you can find the example usage for org.apache.hadoop.io.erasurecode.rawcoder NativeRSRawErasureCoderFactory NativeRSRawErasureCoderFactory.

Prototype

NativeRSRawErasureCoderFactory

Source Link

Usage

From source file:io.hops.erasure_coding.NativeReedSolomonCode.java

License:Apache License

private void init(int stripeSize, int paritySize) {
    this.stripeSize = stripeSize;
    this.paritySize = paritySize;
    factory = new NativeRSRawErasureCoderFactory();
    encoder = (NativeRSRawEncoder) factory
            .createEncoder(new ErasureCoderOptions(this.stripeSize, this.paritySize));
    decoder = (NativeRSRawDecoder) factory
            .createDecoder(new ErasureCoderOptions(this.stripeSize, this.paritySize));
}

---

From source file:io.hops.erasure_coding.TestNativeErasureCodes.java

License:Apache License

@Test
public void testNativeEncodeDecode() {

    long overallEncode = 0L;
    long overallDecode = 0L;

    for (int n = 0; n < TEST_CODES; n++) {
        int stripeSize = 10;//RAND.nextInt(99) + 1; // 1, 2, 3, ... 100
        int paritySize = 4;//RAND.nextInt(9) + 1; //1, 2, 3, 4, ... 10
        NativeRSRawErasureCoderFactory factory = new NativeRSRawErasureCoderFactory();
        NativeRSRawEncoder enc = (NativeRSRawEncoder) factory
                .createEncoder(new ErasureCoderOptions(stripeSize, paritySize));
        NativeRSRawDecoder dec = (NativeRSRawDecoder) factory
                .createDecoder(new ErasureCoderOptions(stripeSize, paritySize));

        for (int m = 0; m < TEST_TIMES; m++) {

            int symbolMax = (int) Math.pow(2, (int) Math.round(Math.log(GF.getFieldSize()) / Math.log(2)));

            int[] message = new int[stripeSize];
            for (int i = 0; i < stripeSize; i++) {
                message[i] = RAND.nextInt(symbolMax) + 2;
            }//w ww . j  av a2  s.c o  m

            int[] parity = new int[paritySize];

            /* Native Encode Starts */
            ByteBuffer[] encodeData = new ByteBuffer[message.length];
            ByteBuffer[] parityData = new ByteBuffer[parity.length];

            int[] inputOffsets = new int[encodeData.length];
            int[] outputOffsets = new int[parityData.length];

            for (int i = 0; i < message.length; i++) {
                encodeData[i] = ByteBuffer.allocateDirect(1024);
                encodeData[i].putInt(message[i]);
                for (int j = 0; j < 255; j++) {
                    encodeData[i].putInt(RAND.nextInt(symbolMax) + 2);
                }
                encodeData[i].flip();
            }

            for (int i = 0; i < parity.length; i++) {
                parityData[i] = ByteBuffer.allocateDirect(1024);
            }

            long startTime = System.currentTimeMillis();
            enc.performEncodeImpl(encodeData, inputOffsets, 1024, parityData, outputOffsets);
            long stopTime = System.currentTimeMillis();
            overallEncode += (stopTime - startTime);

            /* Native Encode Ends here*/

            /* Native Decode Starts here*/
            int[] data = new int[stripeSize + paritySize];
            int[] copy = new int[data.length];
            for (int i = 0; i < stripeSize; i++) {
                data[i] = message[i];
                copy[i] = message[i];
            }
            for (int i = 0; i < paritySize; i++) {
                data[i + stripeSize] = parity[i];
                copy[i + stripeSize] = parity[i];
            }
            int erasedLen = 4;//paritySize == 1 ? 1 : RAND.nextInt(paritySize - 1) + 1;
            int[] erasedLocations = randomErasedLocation(erasedLen, message.length);
            for (int i = 0; i < erasedLocations.length; i++) {
                data[erasedLocations[i]] = 0;
            }
            int[] erasedValues = new int[erasedLen];

            //Native Decode
            ByteBuffer[] decodeData = new ByteBuffer[stripeSize + paritySize];
            ByteBuffer[] recoverData = new ByteBuffer[erasedValues.length];

            inputOffsets = new int[decodeData.length];
            outputOffsets = new int[recoverData.length];

            for (int i = 0; i < stripeSize; i++) {
                if (data[i] == 0) {
                    decodeData[i] = null;
                    continue;
                }
                decodeData[i] = encodeData[i];
                decodeData[i].flip();
            }
            for (int i = stripeSize; i < stripeSize + paritySize; i++) {
                decodeData[i] = parityData[i - stripeSize];
                decodeData[i].flip();
            }
            for (int i = 0; i < erasedValues.length; i++) {
                recoverData[i] = ByteBuffer.allocateDirect(1024);
            }

            startTime = System.currentTimeMillis();
            dec.performDecodeImpl(decodeData, inputOffsets, 1024, erasedLocations, recoverData, outputOffsets);
            stopTime = System.currentTimeMillis();
            overallDecode += (stopTime - startTime);

            for (int i = 0; i < recoverData.length; i++) {
                erasedValues[i] = recoverData[i].getInt();
            }

            /* Native Decode Ends here */

            for (int i = 0; i < erasedLen; i++) {

                StringBuffer sb = new StringBuffer();
                sb.append("\nC ");
                for (int j = 0; j < data.length; j++) {
                    sb.append(" " + copy[j]);
                }

                sb.append("\nD ");

                for (int j = 0; j < data.length; j++) {
                    sb.append(" " + data[j]);
                }

                assertEquals("Decode failed " + sb, copy[erasedLocations[i]], erasedValues[i]);

            }
        }
        enc.release();
        dec.release();

    }

    System.out.println("Encoding " + (overallEncode) + " milliseconds");
    System.out.println("Decoding " + (overallDecode) + " milliseconds");
}

---