Example usage for org.apache.hadoop.io.compress.zlib ZlibFactory isNativeZlibLoaded

List of usage examples for org.apache.hadoop.io.compress.zlib ZlibFactory isNativeZlibLoaded

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.compress.zlib.*
  5. ZlibFactory
  6. isNativeZlibLoaded

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress.zlib ZlibFactory isNativeZlibLoaded.

Prototype

public static boolean isNativeZlibLoaded(Configuration conf)

Source Link

Document

Check if native-zlib code is loaded & initialized correctly and can be loaded for this job.

Usage

From source file:com.facebook.presto.hadoop.TestHadoopNative.java

License:Apache License

@Test
public void testNative() {
    HadoopNative.requireHadoopNative();/*  w  ww.  j a v a 2s . c om*/

    assertTrue(NativeCodeLoader.isNativeCodeLoaded());
    assertTrue(NativeCodeLoader.buildSupportsSnappy());
    assertTrue(ZlibFactory.isNativeZlibLoaded(new Configuration()));
    assertTrue(Bzip2Factory.isNativeBzip2Loaded(new Configuration()));
}

From source file:dima.kmeansseq.SequenceFile.java

License:Apache License

/**
 * Construct the preferred type of SequenceFile Writer.
 * //from   ww  w  .j a v  a  2s. co m
 * @param fs
 *            The configured filesystem.
 * @param conf
 *            The configuration.
 * @param name
 *            The name of the file.
 * @param keyClass
 *            The 'key' type.
 * @param valClass
 *            The 'value' type.
 * @param bufferSize
 *            buffer size for the underlaying outputstream.
 * @param replication
 *            replication factor for the file.
 * @param blockSize
 *            block size for the file.
 * @param compressionType
 *            The compression type.
 * @param codec
 *            The compression codec.
 * @param progress
 *            The Progressable object to track progress.
 * @param metadata
 *            The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass,
        int bufferSize, short replication, long blockSize, CompressionType compressionType,
        CompressionCodec codec, Progressable progress, Metadata metadata) throws IOException {
    if ((codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "SequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (compressionType == CompressionType.NONE) {
        writer = new Writer(fs, conf, name, keyClass, valClass, bufferSize, replication, blockSize, progress,
                metadata);
    } else if (compressionType == CompressionType.RECORD) {
        writer = new RecordCompressWriter(fs, conf, name, keyClass, valClass, bufferSize, replication,
                blockSize, codec, progress, metadata);
    } else if (compressionType == CompressionType.BLOCK) {
        writer = new BlockCompressWriter(fs, conf, name, keyClass, valClass, bufferSize, replication, blockSize,
                codec, progress, metadata);
    }

    return writer;
}

From source file:dima.kmeansseq.SequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * /*from  w w  w  . j av a 2s  .c  o  m*/
 * @param out
 *            The stream on top which the writer is to be constructed.
 * @param keyClass
 *            The 'key' type.
 * @param valClass
 *            The 'value' type.
 * @param compress
 *            Compress data?
 * @param blockCompress
 *            Compress blocks?
 * @param metadata
 *            The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
private static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass,
        boolean compress, boolean blockCompress, CompressionCodec codec, Metadata metadata) throws IOException {
    if (codec != null && (codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "SequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (!compress) {
        writer = new Writer(conf, out, keyClass, valClass, metadata);
    } else if (compress && !blockCompress) {
        writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    } else {
        writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    }

    return writer;
}

From source file:dima.kmeansseq.SequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * //w  w  w . j  a  v a 2 s  . c  o m
 * @param fs
 *            The configured filesystem.
 * @param conf
 *            The configuration.
 * @param file
 *            The name of the file.
 * @param keyClass
 *            The 'key' type.
 * @param valClass
 *            The 'value' type.
 * @param compress
 *            Compress data?
 * @param blockCompress
 *            Compress blocks?
 * @param codec
 *            The compression codec.
 * @param progress
 * @param metadata
 *            The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
private static Writer createWriter(FileSystem fs, Configuration conf, Path file, Class keyClass, Class valClass,
        boolean compress, boolean blockCompress, CompressionCodec codec, Progressable progress,
        Metadata metadata) throws IOException {
    if (codec != null && (codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "SequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (!compress) {
        writer = new Writer(fs, conf, file, keyClass, valClass, progress, metadata);
    } else if (compress && !blockCompress) {
        writer = new RecordCompressWriter(fs, conf, file, keyClass, valClass, codec, progress, metadata);
    } else {
        writer = new BlockCompressWriter(fs, conf, file, keyClass, valClass, codec, progress, metadata);
    }

    return writer;
}

From source file:dima.kmeansseq.SequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * /*from   w  w  w . j  av  a  2 s  . c o  m*/
 * @param conf
 *            The configuration.
 * @param out
 *            The stream on top which the writer is to be constructed.
 * @param keyClass
 *            The 'key' type.
 * @param valClass
 *            The 'value' type.
 * @param compressionType
 *            The compression type.
 * @param codec
 *            The compression codec.
 * @param metadata
 *            The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass,
        CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException {
    if ((codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "SequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (compressionType == CompressionType.NONE) {
        writer = new Writer(conf, out, keyClass, valClass, metadata);
    } else if (compressionType == CompressionType.RECORD) {
        writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    } else if (compressionType == CompressionType.BLOCK) {
        writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    }

    return writer;
}

From source file:io.airlift.compress.HadoopNative.java

License:Apache License

private static void requireNativeZlib() {
    Configuration conf = new Configuration();
    if (!ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new RuntimeException("native zlib is not loaded");
    }/*from w w  w. ja va2  s .c om*/

    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    CompressionCodec codec = factory.getCodecByClassName(GzipCodec.class.getName());
    if (codec == null) {
        throw new RuntimeException("failed to load GzipCodec");
    }
    org.apache.hadoop.io.compress.Decompressor decompressor = codec.createDecompressor();
    if (!(decompressor instanceof ZlibDecompressor)) {
        throw new RuntimeException("wrong gzip decompressor: " + decompressor.getClass().getName());
    }
}

From source file:org.apache.accumulo.core.file.map.MySequenceFile.java

License:Apache License

/**
 * Construct the preferred type of MySequenceFile Writer.
 * //  w  w w  . ja  va2s.co  m
 * @param fs
 *          The configured filesystem.
 * @param conf
 *          The configuration.
 * @param name
 *          The name of the file.
 * @param keyClass
 *          The 'key' type.
 * @param valClass
 *          The 'value' type.
 * @param bufferSize
 *          buffer size for the underlaying outputstream.
 * @param replication
 *          replication factor for the file.
 * @param blockSize
 *          block size for the file.
 * @param compressionType
 *          The compression type.
 * @param codec
 *          The compression codec.
 * @param progress
 *          The Progressable object to track progress.
 * @param metadata
 *          The metadata of the file.
 * @return Returns the handle to the constructed MySequenceFile Writer.
 */
public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass,
        int bufferSize, short replication, long blockSize, CompressionType compressionType,
        CompressionCodec codec, Progressable progress, Metadata metadata) throws IOException {
    if ((codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "MySequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (compressionType == CompressionType.NONE) {
        writer = new Writer(fs, conf, name, keyClass, valClass, bufferSize, replication, blockSize, progress,
                metadata);
    } else if (compressionType == CompressionType.RECORD) {
        writer = new RecordCompressWriter(fs, conf, name, keyClass, valClass, bufferSize, replication,
                blockSize, codec, progress, metadata);
    } else if (compressionType == CompressionType.BLOCK) {
        writer = new BlockCompressWriter(fs, conf, name, keyClass, valClass, bufferSize, replication, blockSize,
                codec, progress, metadata);
    }

    return writer;
}

From source file:org.apache.accumulo.core.file.map.MySequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' MySequenceFile Writer.
 * //from w w w.  j  a  v a  2s  .  c  o  m
 * @param out
 *          The stream on top which the writer is to be constructed.
 * @param keyClass
 *          The 'key' type.
 * @param valClass
 *          The 'value' type.
 * @param compress
 *          Compress data?
 * @param blockCompress
 *          Compress blocks?
 * @param metadata
 *          The metadata of the file.
 * @return Returns the handle to the constructed MySequenceFile Writer.
 * @throws IOException
 */
private static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass,
        boolean compress, boolean blockCompress, CompressionCodec codec, Metadata metadata) throws IOException {
    if (codec != null && (codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "MySequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (!compress) {
        writer = new Writer(conf, out, keyClass, valClass, metadata);
    } else if (compress && !blockCompress) {
        writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    } else {
        writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    }

    return writer;
}

From source file:org.apache.accumulo.core.file.map.MySequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' MySequenceFile Writer.
 * /*from  ww w. ja va 2 s .  c om*/
 * @param fs
 *          The configured filesystem.
 * @param conf
 *          The configuration.
 * @param file
 *          The name of the file.
 * @param keyClass
 *          The 'key' type.
 * @param valClass
 *          The 'value' type.
 * @param compress
 *          Compress data?
 * @param blockCompress
 *          Compress blocks?
 * @param codec
 *          The compression codec.
 * @param progress
 * @param metadata
 *          The metadata of the file.
 * @return Returns the handle to the constructed MySequenceFile Writer.
 * @throws IOException
 */
private static Writer createWriter(FileSystem fs, Configuration conf, Path file, Class keyClass, Class valClass,
        boolean compress, boolean blockCompress, CompressionCodec codec, Progressable progress,
        Metadata metadata) throws IOException {
    if (codec != null && (codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "MySequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (!compress) {
        writer = new Writer(fs, conf, file, keyClass, valClass, progress, metadata);
    } else if (compress && !blockCompress) {
        writer = new RecordCompressWriter(fs, conf, file, keyClass, valClass, codec, progress, metadata);
    } else {
        writer = new BlockCompressWriter(fs, conf, file, keyClass, valClass, codec, progress, metadata);
    }

    return writer;
}

From source file:org.apache.accumulo.core.file.map.MySequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' MySequenceFile Writer.
 * //  w  ww.j a  v  a  2  s. co  m
 * @param conf
 *          The configuration.
 * @param out
 *          The stream on top which the writer is to be constructed.
 * @param keyClass
 *          The 'key' type.
 * @param valClass
 *          The 'value' type.
 * @param compressionType
 *          The compression type.
 * @param codec
 *          The compression codec.
 * @param metadata
 *          The metadata of the file.
 * @return Returns the handle to the constructed MySequenceFile Writer.
 */
public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass,
        CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException {
    if ((codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "MySequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (compressionType == CompressionType.NONE) {
        writer = new Writer(conf, out, keyClass, valClass, metadata);
    } else if (compressionType == CompressionType.RECORD) {
        writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    } else if (compressionType == CompressionType.BLOCK) {
        writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    }

    return writer;
}