List of usage examples for org.apache.hadoop.io.erasurecode.rawcoder NativeRSRawEncoder release
@Override public void release()
From source file:io.hops.erasure_coding.TestNativeErasureCodes.java
License:Apache License
@Test public void testNativeEncodeDecode() { long overallEncode = 0L; long overallDecode = 0L; for (int n = 0; n < TEST_CODES; n++) { int stripeSize = 10;//RAND.nextInt(99) + 1; // 1, 2, 3, ... 100 int paritySize = 4;//RAND.nextInt(9) + 1; //1, 2, 3, 4, ... 10 NativeRSRawErasureCoderFactory factory = new NativeRSRawErasureCoderFactory(); NativeRSRawEncoder enc = (NativeRSRawEncoder) factory .createEncoder(new ErasureCoderOptions(stripeSize, paritySize)); NativeRSRawDecoder dec = (NativeRSRawDecoder) factory .createDecoder(new ErasureCoderOptions(stripeSize, paritySize)); for (int m = 0; m < TEST_TIMES; m++) { int symbolMax = (int) Math.pow(2, (int) Math.round(Math.log(GF.getFieldSize()) / Math.log(2))); int[] message = new int[stripeSize]; for (int i = 0; i < stripeSize; i++) { message[i] = RAND.nextInt(symbolMax) + 2; }//from ww w . java2 s.c o m int[] parity = new int[paritySize]; /* Native Encode Starts */ ByteBuffer[] encodeData = new ByteBuffer[message.length]; ByteBuffer[] parityData = new ByteBuffer[parity.length]; int[] inputOffsets = new int[encodeData.length]; int[] outputOffsets = new int[parityData.length]; for (int i = 0; i < message.length; i++) { encodeData[i] = ByteBuffer.allocateDirect(1024); encodeData[i].putInt(message[i]); for (int j = 0; j < 255; j++) { encodeData[i].putInt(RAND.nextInt(symbolMax) + 2); } encodeData[i].flip(); } for (int i = 0; i < parity.length; i++) { parityData[i] = ByteBuffer.allocateDirect(1024); } long startTime = System.currentTimeMillis(); enc.performEncodeImpl(encodeData, inputOffsets, 1024, parityData, outputOffsets); long stopTime = System.currentTimeMillis(); overallEncode += (stopTime - startTime); /* Native Encode Ends here*/ /* Native Decode Starts here*/ int[] data = new int[stripeSize + paritySize]; int[] copy = new int[data.length]; for (int i = 0; i < stripeSize; i++) { data[i] = message[i]; copy[i] = message[i]; } for (int i = 0; i < paritySize; i++) { data[i + stripeSize] = parity[i]; copy[i + stripeSize] = parity[i]; } int erasedLen = 4;//paritySize == 1 ? 1 : RAND.nextInt(paritySize - 1) + 1; int[] erasedLocations = randomErasedLocation(erasedLen, message.length); for (int i = 0; i < erasedLocations.length; i++) { data[erasedLocations[i]] = 0; } int[] erasedValues = new int[erasedLen]; //Native Decode ByteBuffer[] decodeData = new ByteBuffer[stripeSize + paritySize]; ByteBuffer[] recoverData = new ByteBuffer[erasedValues.length]; inputOffsets = new int[decodeData.length]; outputOffsets = new int[recoverData.length]; for (int i = 0; i < stripeSize; i++) { if (data[i] == 0) { decodeData[i] = null; continue; } decodeData[i] = encodeData[i]; decodeData[i].flip(); } for (int i = stripeSize; i < stripeSize + paritySize; i++) { decodeData[i] = parityData[i - stripeSize]; decodeData[i].flip(); } for (int i = 0; i < erasedValues.length; i++) { recoverData[i] = ByteBuffer.allocateDirect(1024); } startTime = System.currentTimeMillis(); dec.performDecodeImpl(decodeData, inputOffsets, 1024, erasedLocations, recoverData, outputOffsets); stopTime = System.currentTimeMillis(); overallDecode += (stopTime - startTime); for (int i = 0; i < recoverData.length; i++) { erasedValues[i] = recoverData[i].getInt(); } /* Native Decode Ends here */ for (int i = 0; i < erasedLen; i++) { StringBuffer sb = new StringBuffer(); sb.append("\nC "); for (int j = 0; j < data.length; j++) { sb.append(" " + copy[j]); } sb.append("\nD "); for (int j = 0; j < data.length; j++) { sb.append(" " + data[j]); } assertEquals("Decode failed " + sb, copy[erasedLocations[i]], erasedValues[i]); } } enc.release(); dec.release(); } System.out.println("Encoding " + (overallEncode) + " milliseconds"); System.out.println("Decoding " + (overallDecode) + " milliseconds"); }