Example usage for org.apache.hadoop.io.compress BlockDecompressorStream BlockDecompressorStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress BlockDecompressorStream BlockDecompressorStream.

Prototype

public BlockDecompressorStream(InputStream in, Decompressor decompressor, int bufferSize) throws IOException 

Source Link

Document

Create a BlockDecompressorStream .

Usage

From source file:com.hadoop.compression.fourmc.Lz4Codec.java

License:BSD License

public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException {
    if (!isNativeLoaded(conf)) {
        throw new RuntimeException("native hadoop-4mc library not available");
    }/*from w  w w .j  av a 2 s . c om*/
    return new BlockDecompressorStream(in, decompressor, LZ4_BUFFER_SIZE);
}

---

From source file:com.hadoop.compression.fourmc.ZstdCodec.java

License:BSD License

public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException {
    if (!isNativeLoaded(conf)) {
        throw new RuntimeException("native hadoop-4mc library not available");
    }//from w w  w  .j av a2  s . co  m
    return new BlockDecompressorStream(in, decompressor, ZSTD_BUFFER_SIZE);
}

---

From source file:com.hadoop.compression.lzo.LzoCodec.java

License:Open Source License

public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException {
    // Ensure native-lzo library is loaded & initialized
    if (!isNativeLzoLoaded(conf)) {
        throw new RuntimeException("native-lzo library not available");
    }/*  w  w w . j a  v a  2s .c  o  m*/
    return new BlockDecompressorStream(in, decompressor,
            conf.getInt("io.compression.codec.lzo.buffersize", 64 * 1024));
}

---

From source file:io.github.dlmarion.clowncar.hdfs.TestBloscCompressorDecompressor.java

License:Apache License

@Test
public void testCompressorDecompressorEmptyStreamLogic() {
    ByteArrayInputStream bytesIn = null;
    ByteArrayOutputStream bytesOut = null;
    byte[] buf = null;
    BlockDecompressorStream blockDecompressorStream = null;
    try {/*from  w  w w . ja va  2  s.  co m*/
        Configuration conf = new Configuration(false);
        conf.set(BloscCompressor.COMPRESSOR_NAME_KEY, compressor);
        conf.set(BloscCompressor.COMPRESSION_LEVEL_KEY, Integer.toString(level));
        conf.set(BloscCompressor.BYTES_FOR_TYPE_KEY, Integer.toString(Integer.BYTES));
        conf.set(BloscCompressor.SHUFFLE_TYPE_KEY, Integer.toString(shuffle));
        conf.set(BloscCompressor.NUM_THREADS_KEY, Integer.toString(threads));
        // compress empty stream
        bytesOut = new ByteArrayOutputStream();
        BlockCompressorStream blockCompressorStream = new BlockCompressorStream(bytesOut,
                new BloscCompressor(conf), 1024, 0);
        // close without write
        blockCompressorStream.close();
        // check compressed output
        buf = bytesOut.toByteArray();
        assertEquals("empty stream compressed output size != 4", 4, buf.length);
        // use compressed output as input for decompression
        bytesIn = new ByteArrayInputStream(buf);
        // create decompression stream
        blockDecompressorStream = new BlockDecompressorStream(bytesIn, new BloscDecompressor(), 1024);
        // no byte is available because stream was closed
        assertEquals("return value is not -1", -1, blockDecompressorStream.read());
    } catch (Exception e) {
        e.printStackTrace();
        fail("testCompressorDecompressorEmptyStreamLogic ex error !!!" + e.getMessage());
    } finally {
        if (blockDecompressorStream != null)
            try {
                bytesIn.close();
                bytesOut.close();
                blockDecompressorStream.close();
            } catch (IOException e) {
            }
    }
}

---

From source file:io.github.dlmarion.clowncar.hdfs.TestBloscCompressorDecompressor.java

License:Apache License

@Test
public void testCompressorDecompressorLogicWithCompressionStreams() {
    DataOutputStream deflateOut = null;
    DataInputStream inflateIn = null;
    int BYTE_SIZE = 1024 * 100;
    byte[] bytes = generate(BYTE_SIZE);
    int bufferSize = 262144;
    int compressionOverhead = (bufferSize / 6) + 32;
    try {/* w w  w .  j  a va2s. c o  m*/
        Configuration conf = new Configuration(false);
        conf.set(BloscCompressor.COMPRESSOR_NAME_KEY, compressor);
        conf.set(BloscCompressor.COMPRESSION_LEVEL_KEY, Integer.toString(level));
        conf.set(BloscCompressor.BYTES_FOR_TYPE_KEY, Integer.toString(Integer.BYTES));
        conf.set(BloscCompressor.SHUFFLE_TYPE_KEY, Integer.toString(shuffle));
        conf.set(BloscCompressor.NUM_THREADS_KEY, Integer.toString(threads));
        DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
        CompressionOutputStream deflateFilter = new BlockCompressorStream(compressedDataBuffer,
                new BloscCompressor(bufferSize, conf), bufferSize, compressionOverhead);
        deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
        deflateOut.write(bytes, 0, bytes.length);
        deflateOut.flush();
        deflateFilter.finish();

        DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
        deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0, compressedDataBuffer.getLength());

        CompressionInputStream inflateFilter = new BlockDecompressorStream(deCompressedDataBuffer,
                new BloscDecompressor(bufferSize), bufferSize);

        inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter));

        byte[] result = new byte[BYTE_SIZE];
        inflateIn.read(result);

        assertArrayEquals("original array not equals compress/decompressed array", result, bytes);
    } catch (IOException e) {
        e.printStackTrace();
        fail("testBloscCompressorDecopressorLogicWithCompressionStreams ex error !!!");
    } finally {
        try {
            if (deflateOut != null)
                deflateOut.close();
            if (inflateIn != null)
                inflateIn.close();
        } catch (Exception e) {
        }
    }
}

---

From source file:org.anarres.lzo.hadoop.codec.LzoCodec.java

License:Open Source License

@Override
public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException {
    Configuration conf = getConf();
    return new BlockDecompressorStream(in, decompressor, getBufferSize(conf));
}

---