Example usage for org.apache.hadoop.io.compress CompressionCodec createOutputStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress CompressionCodec createOutputStream.

Prototype

CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException;

Source Link

Document

Create a CompressionOutputStream that will write to the given OutputStream with the given Compressor .

Usage

From source file:org.mrgeo.vector.mrsvector.VectorTileWritable.java

License:Apache License

public static VectorTileWritable toWritable(final VectorTile vector, final CompressionCodec codec,
        final Compressor compressor) throws IOException {
    compressor.reset();//from   ww w  . j  a v  a  2  s  . co m

    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final CompressionOutputStream cos = codec.createOutputStream(baos, compressor);

    vector.toProtobuf(cos);

    return new VectorTileWritable(baos.toByteArray());
}

---

From source file:org.springframework.data.hadoop.serialization.ResourceSerializationFormat.java

License:Apache License

/**
 * Writes the content of Spring {@link Resource}s to a single HDFS location.
 *//*w w  w. j a  v a 2  s  .c  om*/
@Override
protected SerializationWriterSupport createWriter(final OutputStream output) {
    // Extend and customize Serialization Writer template
    return new SerializationWriterSupport() {

        private OutputStream outputStream = output;

        private InputStream resourceSeparatorInputStream;

        @Override
        protected Closeable doOpen() throws IOException {

            resourceSeparatorInputStream = null;

            CompressionCodec codec = CompressionUtils.getHadoopCompression(getConfiguration(),
                    getCompressionAlias());

            // If a compression is not specified and if passed stream does have compression capabilities...
            if (codec == null || CompressionOutputStream.class.isInstance(outputStream)) {
                // ...just return original stream untouched
                return outputStream;
            }

            // Eventually re-use Compressor from underlying CodecPool
            final Compressor compressor = CodecPool.getCompressor(codec);

            // Create compression stream wrapping passed stream
            outputStream = codec.createOutputStream(outputStream, compressor);

            return new Closeable() {

                @Override
                public void close() throws IOException {
                    resourceSeparatorInputStream = null;
                    IOUtils.closeStream(outputStream);
                    CodecPool.returnCompressor(compressor);
                }
            };
        }

        @Override
        protected void doWrite(Resource source) throws IOException {
            InputStream inputStream = null;
            try {
                writeSeparator();

                inputStream = source.getInputStream();

                // Write source to HDFS destination
                copyBytes(inputStream, outputStream, getConfiguration(), /* close */false);

            } finally {
                closeStream(inputStream);
            }
        }

        protected void writeSeparator() throws IOException {
            if (getResourceSeparator() == null) {
                return;
            }

            if (resourceSeparatorInputStream == null) {

                // First call inits 'resourceSeparatorInputStream' and does not write anything

                resourceSeparatorInputStream = toInputStream(getResourceSeparator(), "UTF-8");

                return;
            }

            resourceSeparatorInputStream.reset();

            // Write resource separator to HDFS destination
            copyBytes(resourceSeparatorInputStream, outputStream, getConfiguration(), /* close */false);
        }
    };
}

---

• «
• 1
• 2
• »