com.jeffy.hdfs.compression.FileCompressor.java Source code

Introduction

Here is the source code for com.jeffy.hdfs.compression.FileCompressor.java

Source

/*
 * Copyright AsiaInfo Authors.
 * 
 * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
 */
package com.jeffy.hdfs.compression;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;

import org.apache.commons.compress.utils.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CodecPool;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.io.compress.Compressor;

/**
 * CodecPool?Compressors??????
 * CodecPool???
 * 
 * @author Jeffy<renwu58@gmail.com>
 *
 */
public class FileCompressor {

    /**
     * @param args
     * ??????
     * ????
     * @throws IOException 
     */
    public static void main(String[] args) throws IOException {
        Configuration conf = new Configuration();
        //??
        CompressionCodecFactory factory = new CompressionCodecFactory(conf);
        // For example for the 'GzipCodec' codec class name the alias are 'gzip' and 'gzipcodec'.
        CompressionCodec codec = factory.getCodecByName(args[0]);
        if (codec == null) {//???
            System.err.println("Comperssion codec not found for " + args[0]);
            System.exit(1);
        }
        String ext = codec.getDefaultExtension();
        Compressor compressor = null;
        try {
            //?CodecPool?Compressor
            compressor = CodecPool.getCompressor(codec);
            for (int i = 1; i < args.length; i++) {
                String filename = args[i] + ext;
                System.out.println("Compression the file " + filename);
                try (FileSystem outFs = FileSystem.get(URI.create(filename), conf);
                        FileSystem inFs = FileSystem.get(URI.create(args[i]), conf);
                        InputStream in = inFs.open(new Path(args[i]))) {//
                    //Compressor?
                    CompressionOutputStream out = codec.createOutputStream(outFs.create(new Path(filename)),
                            compressor);
                    //?????
                    IOUtils.copy(in, out);
                    out.finish();//?finish()?flush()???
                    compressor.reset(); //???????java.io.IOException: write beyond end of stream
                }
            }
        } finally {//?Compressor??
            CodecPool.returnCompressor(compressor);
        }
    }
}