Example usage for org.apache.hadoop.io.compress GzipCodec createOutputStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress GzipCodec createOutputStream.

Prototype

@Override
    public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException 

Source Link

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

private static void touchFile(String path) throws Exception {
    FileSystem fs;/*from w  w w  .  j  a  v a  2  s  .c  o  m*/
    DataOutputStream outputStream = null;
    GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, getConfiguration());
    Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec);
    OutputStream compressedOut = null;
    try {
        fs = cluster.getFileSystem();
        final Path qualifiedPath = new Path(path).makeQualified(fs);
        final long blockSize = fs.getDefaultBlockSize() * 2;
        outputStream = fs.create(qualifiedPath, true, 0, (short) (fs.getDefaultReplication() * 2), blockSize);
        compressedOut = gzipCodec.createOutputStream(outputStream, gzipCompressor);
        Message msg = new Message("generating test data".getBytes());
        AuditUtil.attachHeaders(msg, currentTimestamp);
        byte[] encodeMsg = Base64.encodeBase64(msg.getData().array());
        compressedOut.write(encodeMsg);
        compressedOut.write("\n".getBytes());
        compressedOut.write(encodeMsg);
        compressedOut.write("\n".getBytes());
        // Genearate a msg with different timestamp.  Default window period is 60sec
        AuditUtil.attachHeaders(msg, nextMinuteTimeStamp);
        encodeMsg = Base64.encodeBase64(msg.getData().array());
        compressedOut.write(encodeMsg);
        compressedOut.write("\n".getBytes());
        compressedOut.flush();
        compressedOut.close();
        pathList.add(qualifiedPath);
        ++nFiles;

        FileStatus fileStatus = fs.getFileStatus(qualifiedPath);
        System.out.println(fileStatus.getBlockSize());
        System.out.println(fileStatus.getReplication());
    } finally {
        compressedOut.close();
        IOUtils.cleanup(null, outputStream);
        CodecPool.returnCompressor(gzipCompressor);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.TestDistCp.java

License:Apache License

private static void touchFile(String path) throws Exception {
    FileSystem fs;/*from   w  w w .  j  av a2 s  . c o m*/
    DataOutputStream outputStream = null;
    GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, getConfigurationForCluster());
    Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec);
    OutputStream compressedOut = null;
    try {
        fs = cluster.getFileSystem();
        final Path qualifiedPath = new Path(path).makeQualified(fs);
        final long blockSize = fs.getDefaultBlockSize() * 2;
        outputStream = fs.create(qualifiedPath, true, 0, (short) (fs.getDefaultReplication() * 2), blockSize);
        compressedOut = gzipCodec.createOutputStream(outputStream, gzipCompressor);
        compressedOut.write(new byte[FILE_SIZE]);
        compressedOut.write("\n".getBytes());
        compressedOut.flush();
        //outputStream.write(new byte[FILE_SIZE]);
        pathList.add(qualifiedPath);
    } finally {
        compressedOut.close();
        IOUtils.cleanup(null, outputStream);
        CodecPool.returnCompressor(gzipCompressor);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.TestIntegration.java

License:Apache License

private void createFileForAudit(String... entries) throws IOException {
    for (String entry : entries) {
        OutputStream out = fs.create(new Path(root + "/" + entry));
        GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, new Configuration());
        Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec);
        OutputStream compressedOut = null;
        try {// w  ww  .j a  va2s  .  c o m
            compressedOut = gzipCodec.createOutputStream(out, gzipCompressor);
            Message msg = new Message((root + "/" + entry).getBytes());
            long currentTimestamp = new Date().getTime();
            AuditUtil.attachHeaders(msg, currentTimestamp);
            byte[] encodeMsg = Base64.encodeBase64(msg.getData().array());
            compressedOut.write(encodeMsg);
            compressedOut.write("\n".getBytes());
            compressedOut.write(encodeMsg);
            compressedOut.write("\n".getBytes());
            /* add a different timestamp to the msg header.
             * The generation time stamp falls in diff window
             * Each file will be having two counters
             */
            AuditUtil.attachHeaders(msg, currentTimestamp + 60001);
            encodeMsg = Base64.encodeBase64(msg.getData().array());
            compressedOut.write(encodeMsg);
            compressedOut.write("\n".getBytes());
            compressedOut.flush();
        } finally {
            compressedOut.close();
            out.close();
            CodecPool.returnCompressor(gzipCompressor);
        }
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.TestIntegration.java

License:Apache License

private void createFiles(String... entries) throws IOException {
    for (String entry : entries) {
        OutputStream out = fs.create(new Path(root + "/" + entry));
        GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, new Configuration());
        Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec);
        OutputStream compressedOut = null;
        try {/*w ww.j a v a  2 s  .com*/
            compressedOut = gzipCodec.createOutputStream(out, gzipCompressor);
            compressedOut.write((root + "/" + entry).getBytes());
            compressedOut.write("\n".getBytes());
        } finally {
            compressedOut.close();
            out.close();
            CodecPool.returnCompressor(gzipCompressor);
        }
    }
}

---

From source file:com.inmobi.messaging.consumer.util.FileUtil.java

License:Apache License

public static void gzip(Path src, Path target, Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    FSDataOutputStream out = fs.create(target);
    GzipCodec gzipCodec = (GzipCodec) ReflectionUtils.newInstance(GzipCodec.class, conf);
    Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec);
    OutputStream compressedOut = gzipCodec.createOutputStream(out, gzipCompressor);
    FSDataInputStream in = fs.open(src);
    try {//from  w  ww  .ja v a 2s  .com
        IOUtils.copyBytes(in, compressedOut, conf);
    } catch (Exception e) {
        LOG.error("Error in compressing ", e);
    } finally {
        in.close();
        CodecPool.returnCompressor(gzipCompressor);
        compressedOut.close();
        out.close();
    }
}

---

From source file:org.trafodion.sql.HBaseAccess.SequenceFileWriter.java

License:Apache License

boolean hdfsCreate(String fname, boolean compress) throws IOException {
    if (logger.isDebugEnabled())
        logger.debug("SequenceFileWriter.hdfsCreate() - started");
    Path filePath = null;/*from  w  w w. j  a  va 2s  .co  m*/
    if (!compress || (compress && fname.endsWith(".gz")))
        filePath = new Path(fname);
    else
        filePath = new Path(fname + ".gz");

    fs = FileSystem.get(filePath.toUri(), conf);
    fsOut = fs.create(filePath, true);

    outStream = fsOut;

    if (logger.isDebugEnabled())
        logger.debug("SequenceFileWriter.hdfsCreate() - file created");
    if (compress) {
        GzipCodec gzipCodec = (GzipCodec) ReflectionUtils.newInstance(GzipCodec.class, conf);
        Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec);
        try {
            outStream = gzipCodec.createOutputStream(fsOut, gzipCompressor);
        } catch (IOException e) {
            if (logger.isDebugEnabled())
                logger.debug("SequenceFileWriter.hdfsCreate() --exception :" + e);
            throw e;
        }
    }

    if (logger.isDebugEnabled())
        logger.debug("SequenceFileWriter.hdfsCreate() - compressed output stream created");
    return true;
}

---