Example usage for org.apache.hadoop.util NativeCodeLoader isNativeCodeLoaded

List of usage examples for org.apache.hadoop.util NativeCodeLoader isNativeCodeLoaded

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.util.*
  5. NativeCodeLoader
  6. isNativeCodeLoaded

Introduction

In this page you can find the example usage for org.apache.hadoop.util NativeCodeLoader isNativeCodeLoaded.

Prototype

public static boolean isNativeCodeLoaded()

Source Link

Document

Check if native-hadoop code is loaded for this platform.

Usage

From source file:com.facebook.presto.hadoop.TestHadoopNative.java

License:Apache License

@Test
public void testNative() {
    HadoopNative.requireHadoopNative();/*from  w  w w.ja  va2 s.com*/

    assertTrue(NativeCodeLoader.isNativeCodeLoaded());
    assertTrue(NativeCodeLoader.buildSupportsSnappy());
    assertTrue(ZlibFactory.isNativeZlibLoaded(new Configuration()));
    assertTrue(Bzip2Factory.isNativeBzip2Loaded(new Configuration()));
}

From source file:dima.kmeansseq.SequenceFile.java

License:Apache License

/**
 * Construct the preferred type of SequenceFile Writer.
 * /*from w  ww. j av  a 2  s .  c  om*/
 * @param fs
 *            The configured filesystem.
 * @param conf
 *            The configuration.
 * @param name
 *            The name of the file.
 * @param keyClass
 *            The 'key' type.
 * @param valClass
 *            The 'value' type.
 * @param bufferSize
 *            buffer size for the underlaying outputstream.
 * @param replication
 *            replication factor for the file.
 * @param blockSize
 *            block size for the file.
 * @param compressionType
 *            The compression type.
 * @param codec
 *            The compression codec.
 * @param progress
 *            The Progressable object to track progress.
 * @param metadata
 *            The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass,
        int bufferSize, short replication, long blockSize, CompressionType compressionType,
        CompressionCodec codec, Progressable progress, Metadata metadata) throws IOException {
    if ((codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "SequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (compressionType == CompressionType.NONE) {
        writer = new Writer(fs, conf, name, keyClass, valClass, bufferSize, replication, blockSize, progress,
                metadata);
    } else if (compressionType == CompressionType.RECORD) {
        writer = new RecordCompressWriter(fs, conf, name, keyClass, valClass, bufferSize, replication,
                blockSize, codec, progress, metadata);
    } else if (compressionType == CompressionType.BLOCK) {
        writer = new BlockCompressWriter(fs, conf, name, keyClass, valClass, bufferSize, replication, blockSize,
                codec, progress, metadata);
    }

    return writer;
}

From source file:dima.kmeansseq.SequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * /*from w  w w  .j a v a 2 s  . co m*/
 * @param out
 *            The stream on top which the writer is to be constructed.
 * @param keyClass
 *            The 'key' type.
 * @param valClass
 *            The 'value' type.
 * @param compress
 *            Compress data?
 * @param blockCompress
 *            Compress blocks?
 * @param metadata
 *            The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
private static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass,
        boolean compress, boolean blockCompress, CompressionCodec codec, Metadata metadata) throws IOException {
    if (codec != null && (codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "SequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (!compress) {
        writer = new Writer(conf, out, keyClass, valClass, metadata);
    } else if (compress && !blockCompress) {
        writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    } else {
        writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    }

    return writer;
}

From source file:dima.kmeansseq.SequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * // w ww. jav  a  2  s  .c  om
 * @param fs
 *            The configured filesystem.
 * @param conf
 *            The configuration.
 * @param file
 *            The name of the file.
 * @param keyClass
 *            The 'key' type.
 * @param valClass
 *            The 'value' type.
 * @param compress
 *            Compress data?
 * @param blockCompress
 *            Compress blocks?
 * @param codec
 *            The compression codec.
 * @param progress
 * @param metadata
 *            The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
private static Writer createWriter(FileSystem fs, Configuration conf, Path file, Class keyClass, Class valClass,
        boolean compress, boolean blockCompress, CompressionCodec codec, Progressable progress,
        Metadata metadata) throws IOException {
    if (codec != null && (codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "SequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (!compress) {
        writer = new Writer(fs, conf, file, keyClass, valClass, progress, metadata);
    } else if (compress && !blockCompress) {
        writer = new RecordCompressWriter(fs, conf, file, keyClass, valClass, codec, progress, metadata);
    } else {
        writer = new BlockCompressWriter(fs, conf, file, keyClass, valClass, codec, progress, metadata);
    }

    return writer;
}

From source file:dima.kmeansseq.SequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' SequenceFile Writer.
 * /*from w ww .  j  a v a 2 s.  co m*/
 * @param conf
 *            The configuration.
 * @param out
 *            The stream on top which the writer is to be constructed.
 * @param keyClass
 *            The 'key' type.
 * @param valClass
 *            The 'value' type.
 * @param compressionType
 *            The compression type.
 * @param codec
 *            The compression codec.
 * @param metadata
 *            The metadata of the file.
 * @return Returns the handle to the constructed SequenceFile Writer.
 * @throws IOException
 */
public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass,
        CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException {
    if ((codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "SequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (compressionType == CompressionType.NONE) {
        writer = new Writer(conf, out, keyClass, valClass, metadata);
    } else if (compressionType == CompressionType.RECORD) {
        writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    } else if (compressionType == CompressionType.BLOCK) {
        writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    }

    return writer;
}

From source file:org.apache.accumulo.core.file.map.MySequenceFile.java

License:Apache License

/**
 * Construct the preferred type of MySequenceFile Writer.
 * //ww  w. j  a  va  2  s  .c  om
 * @param fs
 *          The configured filesystem.
 * @param conf
 *          The configuration.
 * @param name
 *          The name of the file.
 * @param keyClass
 *          The 'key' type.
 * @param valClass
 *          The 'value' type.
 * @param bufferSize
 *          buffer size for the underlaying outputstream.
 * @param replication
 *          replication factor for the file.
 * @param blockSize
 *          block size for the file.
 * @param compressionType
 *          The compression type.
 * @param codec
 *          The compression codec.
 * @param progress
 *          The Progressable object to track progress.
 * @param metadata
 *          The metadata of the file.
 * @return Returns the handle to the constructed MySequenceFile Writer.
 */
public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass,
        int bufferSize, short replication, long blockSize, CompressionType compressionType,
        CompressionCodec codec, Progressable progress, Metadata metadata) throws IOException {
    if ((codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "MySequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (compressionType == CompressionType.NONE) {
        writer = new Writer(fs, conf, name, keyClass, valClass, bufferSize, replication, blockSize, progress,
                metadata);
    } else if (compressionType == CompressionType.RECORD) {
        writer = new RecordCompressWriter(fs, conf, name, keyClass, valClass, bufferSize, replication,
                blockSize, codec, progress, metadata);
    } else if (compressionType == CompressionType.BLOCK) {
        writer = new BlockCompressWriter(fs, conf, name, keyClass, valClass, bufferSize, replication, blockSize,
                codec, progress, metadata);
    }

    return writer;
}

From source file:org.apache.accumulo.core.file.map.MySequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' MySequenceFile Writer.
 * /*from w ww . j  a  v a 2 s  . c  o m*/
 * @param out
 *          The stream on top which the writer is to be constructed.
 * @param keyClass
 *          The 'key' type.
 * @param valClass
 *          The 'value' type.
 * @param compress
 *          Compress data?
 * @param blockCompress
 *          Compress blocks?
 * @param metadata
 *          The metadata of the file.
 * @return Returns the handle to the constructed MySequenceFile Writer.
 * @throws IOException
 */
private static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass,
        boolean compress, boolean blockCompress, CompressionCodec codec, Metadata metadata) throws IOException {
    if (codec != null && (codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "MySequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (!compress) {
        writer = new Writer(conf, out, keyClass, valClass, metadata);
    } else if (compress && !blockCompress) {
        writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    } else {
        writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    }

    return writer;
}

From source file:org.apache.accumulo.core.file.map.MySequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' MySequenceFile Writer.
 * /* ww  w .j  a  v a 2s .  c o  m*/
 * @param fs
 *          The configured filesystem.
 * @param conf
 *          The configuration.
 * @param file
 *          The name of the file.
 * @param keyClass
 *          The 'key' type.
 * @param valClass
 *          The 'value' type.
 * @param compress
 *          Compress data?
 * @param blockCompress
 *          Compress blocks?
 * @param codec
 *          The compression codec.
 * @param progress
 * @param metadata
 *          The metadata of the file.
 * @return Returns the handle to the constructed MySequenceFile Writer.
 * @throws IOException
 */
private static Writer createWriter(FileSystem fs, Configuration conf, Path file, Class keyClass, Class valClass,
        boolean compress, boolean blockCompress, CompressionCodec codec, Progressable progress,
        Metadata metadata) throws IOException {
    if (codec != null && (codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "MySequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (!compress) {
        writer = new Writer(fs, conf, file, keyClass, valClass, progress, metadata);
    } else if (compress && !blockCompress) {
        writer = new RecordCompressWriter(fs, conf, file, keyClass, valClass, codec, progress, metadata);
    } else {
        writer = new BlockCompressWriter(fs, conf, file, keyClass, valClass, codec, progress, metadata);
    }

    return writer;
}

From source file:org.apache.accumulo.core.file.map.MySequenceFile.java

License:Apache License

/**
 * Construct the preferred type of 'raw' MySequenceFile Writer.
 * /*from   ww  w.  ja  v  a2 s  . c o m*/
 * @param conf
 *          The configuration.
 * @param out
 *          The stream on top which the writer is to be constructed.
 * @param keyClass
 *          The 'key' type.
 * @param valClass
 *          The 'value' type.
 * @param compressionType
 *          The compression type.
 * @param codec
 *          The compression codec.
 * @param metadata
 *          The metadata of the file.
 * @return Returns the handle to the constructed MySequenceFile Writer.
 */
public static Writer createWriter(Configuration conf, FSDataOutputStream out, Class keyClass, Class valClass,
        CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException {
    if ((codec instanceof GzipCodec) && !NativeCodeLoader.isNativeCodeLoaded()
            && !ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new IllegalArgumentException(
                "MySequenceFile doesn't work with " + "GzipCodec without native-hadoop code!");
    }

    Writer writer = null;

    if (compressionType == CompressionType.NONE) {
        writer = new Writer(conf, out, keyClass, valClass, metadata);
    } else if (compressionType == CompressionType.RECORD) {
        writer = new RecordCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    } else if (compressionType == CompressionType.BLOCK) {
        writer = new BlockCompressWriter(conf, out, keyClass, valClass, codec, metadata);
    }

    return writer;
}

From source file:org.apache.ignite.internal.processors.hadoop.HadoopSnappyTest.java

License:Apache License

/**
 * Internal check routine.//from   w w  w  .j  a va2s .c o m
 *
 * @throws Throwable If failed.
 */
public static void checkSnappy() throws Throwable {
    try {
        byte[] expBytes = new byte[BYTE_SIZE];
        byte[] actualBytes = new byte[BYTE_SIZE];

        for (int i = 0; i < expBytes.length; i++)
            expBytes[i] = (byte) ThreadLocalRandom.current().nextInt(16);

        SnappyCodec codec = new SnappyCodec();

        codec.setConf(new Configuration());

        ByteArrayOutputStream baos = new ByteArrayOutputStream();

        try (CompressionOutputStream cos = codec.createOutputStream(baos)) {
            cos.write(expBytes);
            cos.flush();
        }

        try (CompressionInputStream cis = codec
                .createInputStream(new ByteArrayInputStream(baos.toByteArray()))) {
            int read = cis.read(actualBytes, 0, actualBytes.length);

            assert read == actualBytes.length;
        }

        assert Arrays.equals(expBytes, actualBytes);
    } catch (Throwable e) {
        System.out.println("Snappy check failed:");
        System.out
                .println("### NativeCodeLoader.isNativeCodeLoaded:  " + NativeCodeLoader.isNativeCodeLoaded());
        System.out
                .println("### SnappyCompressor.isNativeCodeLoaded:  " + SnappyCompressor.isNativeCodeLoaded());

        throw e;
    }
}