Example usage for org.apache.hadoop.io.compress CompressionCodec createCompressor

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress CompressionCodec createCompressor.

Prototype

Compressor createCompressor();

Source Link

Document

Create a new Compressor for use by this CompressionCodec .

Usage

From source file:com.cloudera.flume.handlers.hdfs.CustomDfsSink.java

License:Apache License

/**
 * Hadoop Compression Codecs that use Native libs require an instance of a
 * Configuration Object. They require this due to some check against knowing
 * weather or not the native libs have been loaded. GzipCodec, LzoCodec,
 * LzopCodec are all codecs that require Native libs. GZipCodec has a slight
 * exception that if native libs are not accessible it will use Pure Java.
 * This results in no errors just notices. BZip2Codec is an example codec that
 * doesn't use native libs./*  w w w. j a  v a 2s .  c  om*/
 */
@Override
public void open() throws IOException {
    FlumeConfiguration conf = FlumeConfiguration.get();
    FileSystem hdfs;
    String codecName = conf.getCollectorDfsCompressCodec();
    CompressionCodec codec = getCodec(conf, codecName);

    if (codec == null) {
        dstPath = new Path(path);
        hdfs = dstPath.getFileSystem(conf);
        pathManager = new PathManager(hdfs, dstPath.getParent(), dstPath.getName());
        writer = pathManager.open();
        LOG.info("Creating HDFS file: " + pathManager.getOpenPath());
        return;
    }

    Compressor cmp = codec.createCompressor();
    dstPath = new Path(path + codec.getDefaultExtension());
    hdfs = dstPath.getFileSystem(conf);
    pathManager = new PathManager(hdfs, dstPath.getParent(), dstPath.getName());
    writer = pathManager.open();
    try {
        writer = codec.createOutputStream(writer, cmp);
    } catch (NullPointerException npe) {
        // tries to find "native" version of codec, if that fails, then tries to
        // find java version. If there is no java version, the createOutputStream
        // exits via NPE. We capture this and convert it into a IOE with a more
        // useful error message.
        LOG.error("Unable to load compression codec " + codec);
        throw new IOException("Unable to load compression codec " + codec);
    }
    LOG.info("Creating " + codec + " compressed HDFS file: " + pathManager.getOpenPath());
}

---

From source file:com.flipkart.fdp.migration.distcp.core.MirrorUtils.java

License:Apache License

public static OutputStream getCodecOutputStream(Configuration conf, String codecName, OutputStream out)
        throws IOException {
    CompressionCodecFactory compressionCodecs = new CompressionCodecFactory(conf);
    String codecClassName = codecName;
    CodecType codecType = CodecType.getCodecType(codecName);
    if (codecType != null) {
        codecClassName = codecType.getIOCompressionCodecs();
    }//from  www  .j  ava2  s  . c o  m
    System.out.println("codec class : " + codecClassName);
    CompressionCodec codec = compressionCodecs.getCodecByName(codecClassName);

    if (codec == null) {
        return out;
    }

    System.out.println("Getting OutputStream : " + codec.getDefaultExtension());
    System.out.println("Getting OutputStream : " + codec);
    Compressor compressor = codec.createCompressor();
    return codec.createOutputStream(out, compressor);
}

---

From source file:data.intelligence.platform.yarn.etl.io.CodecPool.java

License:Apache License

/**
 * Get a {@link Compressor} for the given {@link CompressionCodec} from the
 * pool or a new one./*from ww  w. j a  va2s .c  om*/
 * 
 * @param codec
 *            the <code>CompressionCodec</code> for which to get the
 *            <code>Compressor</code>
 * @return <code>Compressor</code> for the given
 *         <code>CompressionCodec</code> from the pool or a new one
 */
public static Compressor getCompressor(CompressionCodec codec) {
    Compressor compressor = borrow(COMPRESSOR_POOL, codec.getCompressorType());
    if (compressor == null) {
        compressor = codec.createCompressor();
        LOG.info("Got brand-new compressor");
    } else {
        LOG.debug("Got recycled compressor");
    }
    return compressor;
}

---

From source file:org.apache.sqoop.connector.hdfs.hdfsWriter.HdfsTextWriter.java

License:Apache License

@Override
public void initialize(Path filepath, Configuration conf, CompressionCodec codec) throws IOException {
    FileSystem fs = filepath.getFileSystem(conf);

    DataOutputStream filestream = fs.create(filepath, false);
    if (codec != null) {
        filewriter = new BufferedWriter(new OutputStreamWriter(
                codec.createOutputStream(filestream, codec.createCompressor()), Charsets.UTF_8));
    } else {/*from w ww. ja v  a  2 s. c o  m*/
        filewriter = new BufferedWriter(new OutputStreamWriter(filestream, Charsets.UTF_8));
    }
}

---

From source file:org.apache.sqoop.connector.hdfs.TestHdfsBase.java

License:Apache License

protected void createTextInput(String indir, Class<? extends CompressionCodec> clz, int numberOfFiles,
        int numberOfRows) throws IOException, InstantiationException, IllegalAccessException {
    Configuration conf = new Configuration();

    CompressionCodec codec = null;
    String extension = "";
    if (clz != null) {
        codec = clz.newInstance();/*from   ww w . ja  v  a2  s  .c  o m*/
        if (codec instanceof Configurable) {
            ((Configurable) codec).setConf(conf);
        }
        extension = codec.getDefaultExtension();
    }

    int index = 1;
    for (int fi = 0; fi < numberOfFiles; fi++) {
        String fileName = indir + "/" + UUID.randomUUID() + extension;
        OutputStream filestream = FileUtils.create(fileName);
        BufferedWriter filewriter;
        if (codec != null) {
            filewriter = new BufferedWriter(new OutputStreamWriter(
                    codec.createOutputStream(filestream, codec.createCompressor()), "UTF-8"));
        } else {
            filewriter = new BufferedWriter(new OutputStreamWriter(filestream, "UTF-8"));
        }

        for (int ri = 0; ri < numberOfRows; ri++) {
            String row = index + "," + (double) index + ",'" + index + "'";
            filewriter.write(row + HdfsConstants.DEFAULT_RECORD_DELIMITER);
            index++;
        }

        filewriter.close();
    }
}

---

From source file:org.apache.tajo.storage.compress.CodecPool.java

License:Apache License

/**
 * Get a {@link Compressor} for the given {@link CompressionCodec} from the
 * pool or a new one./*from   w ww  . java 2s . c o  m*/
 *
 * @param codec
 *          the <code>CompressionCodec</code> for which to get the
 *          <code>Compressor</code>
 * @param conf the <code>Configuration</code> object which contains confs for creating or reinit the compressor
 * @return <code>Compressor</code> for the given <code>CompressionCodec</code>
 *         from the pool or a new one
 */
public static Compressor getCompressor(CompressionCodec codec, Configuration conf) {
    Compressor compressor = borrow(COMPRESSOR_POOL, codec.getCompressorType());
    if (compressor == null) {
        compressor = codec.createCompressor();
        LOG.info("Got brand-new compressor [" + codec.getDefaultExtension() + "]");
    } else {
        compressor.reinit(conf);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Got recycled compressor");
        }
    }
    return compressor;
}

---