Example usage for org.apache.hadoop.io.compress CompressionOutputStream finish

List of usage examples for org.apache.hadoop.io.compress CompressionOutputStream finish

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.compress.*
  5. CompressionOutputStream
  6. finish

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress CompressionOutputStream finish.

Prototype

public abstract void finish() throws IOException;

Source Link

Document

Finishes writing compressed data to the output stream without closing the underlying stream.

Usage

From source file:TestCodec.java

License:Open Source License

public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    DefaultCodec codec = new DefaultCodec();
    codec.setConf(conf);//ww  w  .  j av a  2 s. c o  m
    DataOutputBuffer chunksWriteBuffer = new DataOutputBuffer();
    CompressionOutputStream compressionOutputStream = codec.createOutputStream(chunksWriteBuffer);

    DataInputBuffer chunkReadBuffer = new DataInputBuffer();
    CompressionInputStream compressionInputStream = codec.createInputStream(chunkReadBuffer);
    String str = "laksjldfkjalskdjfl;aksjdflkajsldkfjalksjdflkajlsdkfjlaksjdflka";
    compressionOutputStream.write(str.getBytes());
    compressionOutputStream.finish();
    byte[] data = chunksWriteBuffer.getData();
    System.out.println(str.length());
    System.out.println(chunksWriteBuffer.getLength());

    chunkReadBuffer.reset(data, chunksWriteBuffer.getLength());

    DataOutputBuffer dob = new DataOutputBuffer();
    IOUtils.copyBytes(compressionInputStream, dob, conf);
    System.out.println(dob.getData());

}

From source file:cn.lhfei.hadoop.ch04.PooledStreamCompressor.java

License:Apache License

/**
 * use case: //ww w . j av  a 2 s .  c o  m
 * 
 * @param args
 */
public static void main(String[] args) {
    String codecClassname = args[0];
    Class<?> codecClass = null;
    CompressionOutputStream out = null;
    Compressor compressor = null;
    try {
        codecClass = Class.forName(codecClassname);
        Configuration conf = new Configuration();
        CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        compressor = CodecPool.getCompressor(codec);

        out = codec.createOutputStream(System.out, compressor);

        IOUtils.copyBytes(System.in, out, 4096, false);

        out.finish();

    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        CodecPool.returnCompressor(compressor);
    }
}

From source file:cn.lhfei.hadoop.ch04.StreamCompressor.java

License:Apache License

/**
 * Use case: % echo "Text" | hadoop StreamCompressor org.apache.hadoop.io.compress.GzipCodec \ | gunzip - Text 
 * @param args//ww w  . j  a  v a 2  s .com
 */
public static void main(String[] args) {
    String codecClassname = args[0];

    try {
        Class<?> codecClass = Class.forName(codecClassname);

        Configuration conf = new Configuration();

        CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);

        CompressionOutputStream out = codec.createOutputStream(System.out);

        IOUtils.copyBytes(System.in, out, 4096, false);

        out.finish();

    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }

}

From source file:com.jeffy.hdfs.compression.FileCompressor.java

License:Apache License

/**
 * @param args//www .  j a v a 2 s .com
 * ??????
 * ????
 * @throws IOException 
 */
public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    //??
    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    // For example for the 'GzipCodec' codec class name the alias are 'gzip' and 'gzipcodec'.
    CompressionCodec codec = factory.getCodecByName(args[0]);
    if (codec == null) {//???
        System.err.println("Comperssion codec not found for " + args[0]);
        System.exit(1);
    }
    String ext = codec.getDefaultExtension();
    Compressor compressor = null;
    try {
        //?CodecPool?Compressor
        compressor = CodecPool.getCompressor(codec);
        for (int i = 1; i < args.length; i++) {
            String filename = args[i] + ext;
            System.out.println("Compression the file " + filename);
            try (FileSystem outFs = FileSystem.get(URI.create(filename), conf);
                    FileSystem inFs = FileSystem.get(URI.create(args[i]), conf);
                    InputStream in = inFs.open(new Path(args[i]))) {//
                //Compressor?
                CompressionOutputStream out = codec.createOutputStream(outFs.create(new Path(filename)),
                        compressor);
                //?????
                IOUtils.copy(in, out);
                out.finish();//?finish()?flush()???
                compressor.reset(); //???????java.io.IOException: write beyond end of stream
            }
        }
    } finally {//?Compressor??
        CodecPool.returnCompressor(compressor);
    }
}

From source file:com.jeffy.hdfs.compression.StreamCompressor.java

License:Apache License

public static void main(String[] args) throws ClassNotFoundException, IOException {
    ///*from  www.j a v  a2 s . c o m*/
    String codecClassname = args[0];

    Class<?> codecClass = Class.forName(codecClassname);

    Configuration conf = new Configuration();

    CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);

    CompressionOutputStream out = codec.createOutputStream(System.out);

    IOUtils.copy(System.in, out);

    out.finish();

}

From source file:Compress.TestLZO.java

License:Open Source License

public static void main(String[] argv) throws IOException {
    System.out.println(System.getProperty("java.library.path"));

    Configuration conf = new Configuration();

    conf.setInt("io.compression.codec.lzo.buffersize", 64 * 1024);

    LzoCodec codec = new LzoCodec();
    codec.setConf(conf);/* ww w .  ja  va2s  .c o  m*/

    OutputStream out = new DataOutputBuffer();
    CompressionOutputStream out2 = codec.createOutputStream(out);

    byte[] str2 = new byte[20];

    int num = 10000;
    int before = 0;
    String part = "hello konten hello konten";
    for (long i = 0; i < num; i++) {
        Util.long2bytes(str2, i);
        out2.write(str2, 0, 8);

    }
    out2.finish();

    byte[] buffer = ((DataOutputBuffer) out).getData();

    System.out.println("org len:" + num * 8 + ", compressed len:" + ((DataOutputBuffer) out).getLength());

    InputStream in = new DataInputBuffer();
    ((DataInputBuffer) in).reset(((DataOutputBuffer) out).getData(), 0, ((DataOutputBuffer) out).getLength());

    CompressionInputStream in2 = codec.createInputStream(in);

    byte[] buf = new byte[100];
    for (long i = 0; i < num; i++) {
        int count = 0;
        count = in2.read(buf, 0, 8);
        if (count > 0) {
            long value = Util.bytes2long(buf, 0, 8);
            if (value != i) {
                System.out.println(i + ",count:" + count + ",value:" + value);
            } else if (i > (num - 20)) {
                System.out.println(i + ",value:" + value);
            }

        } else {
            System.out.println("count:" + count + ", string " + i);
            break;
        }
    }

    in2.close();

    System.out.println("test compress array...");

    OutputStream out3 = new DataOutputBuffer();
    CompressionOutputStream out4 = codec.createOutputStream(out3);

    DataOutputBuffer tout3 = new DataOutputBuffer();

    for (long i = 0; i < num; i++) {
        Util.long2bytes(str2, i);
        out4.write(str2, 0, 8);
    }
    out4.finish();

    buffer = ((DataOutputBuffer) out3).getData();

    System.out.println("org len:" + num * 8 + ", compressed len:" + ((DataOutputBuffer) out3).getLength());

    InputStream in3 = new DataInputBuffer();
    ((DataInputBuffer) in3).reset(((DataOutputBuffer) out3).getData(), 0,
            ((DataOutputBuffer) out3).getLength());

    CompressionInputStream in4 = codec.createInputStream(in3);

    for (long i = 0; i < num; i++) {
        int count = 0;
        count = in4.read(buf, 0, 8);
        if (count > 0) {
            long value = Util.bytes2long(buf, 0, 8);
            if (value != i) {
                System.out.println(i + ",count:" + count + ",value:" + value);
            }

            if (i > (num - 20)) {
                System.out.println(i + ",value:" + value);
            }

        } else {
            System.out.println("count:" + count + ", string " + i);
            break;
        }
    }

    in2.close();

}

From source file:crunch.MaxTemperature.java

License:Apache License

public static void main(String[] args) throws Exception {
        String codecClassname = args[0];
        Class<?> codecClass = Class.forName(codecClassname);
        Configuration conf = new Configuration();
        CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        /*[*/Compressor compressor = null;
        try {/*www  .ja  va  2  s .  com*/
            compressor = CodecPool.getCompressor(codec);/*]*/
            CompressionOutputStream out = codec.createOutputStream(System.out, /*[*/compressor/*]*/);
            IOUtils.copyBytes(System.in, out, 4096, false);
            out.finish();
            /*[*/} finally {
            CodecPool.returnCompressor(compressor);
        } /*]*/
    }

From source file:crunch.MaxTemperature.java

License:Apache License

public static void main(String[] args) throws Exception {
        String codecClassname = args[0];
        Class<?> codecClass = Class.forName(codecClassname);
        Configuration conf = new Configuration();
        CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);

        CompressionOutputStream out = codec.createOutputStream(System.out);
        IOUtils.copyBytes(System.in, out, 4096, false);
        out.finish();
    }//w  w w .  j  a  v  a2  s . com

From source file:io.aos.hdfs.PooledStreamCompressor.java

License:Apache License

public static void main(String... args) throws Exception {
    String codecClassname = args[0];
    Class<?> codecClass = Class.forName(codecClassname);
    Configuration conf = new Configuration();
    CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
    /*[*/Compressor compressor = null;
    try {/*w  ww  .j av a 2 s  .  c  om*/
        compressor = CodecPool.getCompressor(codec);/*]*/
        CompressionOutputStream out = codec.createOutputStream(System.out, /*[*/compressor/*]*/);
        IOUtils.copyBytes(System.in, out, 4096, false);
        out.finish();
        /*[*/} finally {
        CodecPool.returnCompressor(compressor);
    } /*]*/
}

From source file:io.aos.hdfs.StreamCompressor.java

License:Apache License

public static void main(String... args) throws Exception {
    String codecClassname = args[0];
    Class<?> codecClass = Class.forName(codecClassname);
    Configuration conf = new Configuration();
    CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);

    CompressionOutputStream out = codec.createOutputStream(System.out);
    IOUtils.copyBytes(System.in, out, 4096, false);
    out.finish();
}