Example usage for org.apache.hadoop.io.compress CompressionCodecFactory getCodecByClassName

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress CompressionCodecFactory getCodecByClassName.

Prototype

public CompressionCodec getCodecByClassName(String classname) 

Source Link

Document

Find the relevant compression codec for the codec's canonical class name.

Usage

From source file:be.ugent.intec.halvade.uploader.HalvadeUploader.java

License:Open Source License

public void parseArguments(String[] args) throws ParseException {
    createOptions();//from  w  w  w.ja v a  2  s .  c  o  m
    CommandLineParser parser = new GnuParser();
    CommandLine line = parser.parse(options, args);
    manifest = line.getOptionValue("1");
    if (!manifest.endsWith(".manifest")) {
        file1 = manifest;
        manifest = null;
    }
    outputDir = line.getOptionValue("O");
    if (!outputDir.endsWith("/"))
        outputDir += "/";

    if (line.hasOption("2"))
        file2 = line.getOptionValue("2");
    if (line.hasOption("profile"))
        profile = line.getOptionValue("profile");
    if (line.hasOption("t"))
        mthreads = Integer.parseInt(line.getOptionValue("t"));
    if (line.hasOption("i"))
        isInterleaved = true;
    if (line.hasOption("sse"))
        SSE = true;
    if (line.hasOption("snappy")) {
        CompressionCodecFactory codecFactory = new CompressionCodecFactory(getConf());
        codec = codecFactory.getCodecByClassName("org.apache.hadoop.io.compress.SnappyCodec");
    }
    if (line.hasOption("lz4")) {
        CompressionCodecFactory codecFactory = new CompressionCodecFactory(getConf());
        codec = codecFactory.getCodecByClassName("org.apache.hadoop.io.compress.Lz4Codec");
    }
    if (codec != null)
        Logger.DEBUG("Hadoop encryption: " + codec.getDefaultExtension().substring(1));
    if (line.hasOption("size"))
        bestFileSize = Integer.parseInt(line.getOptionValue("size")) * 1024 * 1024;
}

---

From source file:com.facebook.presto.hadoop.HadoopNative.java

License:Apache License

private static void loadAllCodecs() {
    Configuration conf = new Configuration();
    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    for (Class<? extends CompressionCodec> clazz : getCodecClasses(conf)) {
        CompressionCodec codec = factory.getCodecByClassName(clazz.getName());
        if (codec == null) {
            throw new RuntimeException("failed to load codec: " + clazz.getName());
        }/*www .ja  v  a2  s  .co m*/
        codec.getDecompressorType();
    }
}

---

From source file:com.facebook.presto.hadoop.TestHadoopNative.java

License:Apache License

@Test
public void testCodecRoundTrip() throws Exception {
    HadoopNative.requireHadoopNative();/* w  w  w  . j  av  a 2s . c  om*/

    Configuration conf = new Configuration();
    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    for (Class<? extends CompressionCodec> clazz : getCodecClasses(conf)) {
        CompressionCodec codec = factory.getCodecByClassName(clazz.getName());
        assertNotNull(codec, clazz.getName());

        byte[] expected = "Hello world! Goodbye world!".getBytes(UTF_8);
        byte[] actual = decompress(codec, compress(codec, expected));
        assertEquals(actual, expected);
    }
}

---

From source file:com.kylinolap.job.hadoop.cube.CopySeq.java

License:Apache License

static CompressionCodec getLZOCodec(Configuration hconf) {
    CompressionCodecFactory factory = new CompressionCodecFactory(hconf);
    return factory.getCodecByClassName("org.apache.hadoop.io.compress.LzoCodec");
}

---

From source file:com.pinterest.secor.util.CompressionUtil.java

License:Apache License

public static CompressionCodec createCompressionCodec(String className) throws Exception {
    Configuration configuration = new Configuration();
    CompressionCodecFactory.setCodecClasses(configuration,
            new LinkedList<Class>(Collections.singletonList(Class.forName(className))));
    CompressionCodecFactory ccf = new CompressionCodecFactory(configuration);
    return ccf.getCodecByClassName(className);
}

---

From source file:io.airlift.compress.HadoopNative.java

License:Apache License

private static void requireNativeZlib() {
    Configuration conf = new Configuration();
    if (!ZlibFactory.isNativeZlibLoaded(conf)) {
        throw new RuntimeException("native zlib is not loaded");
    }//from w  w w .  ja  va2s  . c o m

    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    CompressionCodec codec = factory.getCodecByClassName(GzipCodec.class.getName());
    if (codec == null) {
        throw new RuntimeException("failed to load GzipCodec");
    }
    org.apache.hadoop.io.compress.Decompressor decompressor = codec.createDecompressor();
    if (!(decompressor instanceof ZlibDecompressor)) {
        throw new RuntimeException("wrong gzip decompressor: " + decompressor.getClass().getName());
    }
}

---

From source file:org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.java

License:Apache License

/**
 * Returns the configured CompressionCodec, or null if none is configured.
 *
 * @param context//ww  w  .  j  a v  a  2s  .  c  om
 *            the ProcessContext
 * @param configuration
 *            the Hadoop Configuration
 * @return CompressionCodec or null
 */
protected org.apache.hadoop.io.compress.CompressionCodec getCompressionCodec(ProcessContext context,
        Configuration configuration) {
    org.apache.hadoop.io.compress.CompressionCodec codec = null;
    if (context.getProperty(COMPRESSION_CODEC).isSet()) {
        String compressionClassname = CompressionType.valueOf(context.getProperty(COMPRESSION_CODEC).getValue())
                .toString();
        CompressionCodecFactory ccf = new CompressionCodecFactory(configuration);
        codec = ccf.getCodecByClassName(compressionClassname);
    }

    return codec;
}

---

From source file:org.apache.tez.runtime.library.common.sort.impl.TestIFile.java

License:Apache License

@Before
public void setUp() throws Exception {
    CompressionCodecFactory codecFactory = new CompressionCodecFactory(new Configuration());
    codec = codecFactory.getCodecByClassName("org.apache.hadoop.io.compress.DefaultCodec");
    outputPath = new Path(workDir, outputFileName);
}

---

From source file:org.archive.hadoop.mapreduce.LineDereferencingRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    forceCompressed = conf.getBoolean(FORCE_COMPRESSED_FLAG, false);

    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
    codec = ccf.getCodecByClassName(GzipCodec.class.getName());

    FileSplit fileSplit = (FileSplit) split;
    fileSystem = fileSplit.getPath().getFileSystem(conf);
    internal.initialize(split, context);
}

---