Example usage for org.apache.hadoop.io.compress CompressionCodec getDefaultExtension

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress CompressionCodec getDefaultExtension.

Prototype

String getDefaultExtension();

Source Link

Document

Get the default filename extension for this kind of compression.

Usage

From source file:org.seqdoop.hadoop_bam.QseqOutputFormat.java

License:Open Source License

public RecordWriter<Text, SequencedFragment> getRecordWriter(TaskAttemptContext task) throws IOException {
    Configuration conf = ContextUtil.getConfiguration(task);
    boolean isCompressed = getCompressOutput(task);

    CompressionCodec codec = null;
    String extension = "";

    if (isCompressed) {
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(task, GzipCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        extension = codec.getDefaultExtension();
    }//  w w  w. j  a v  a 2 s  . co m

    Path file = getDefaultWorkFile(task, extension);
    FileSystem fs = file.getFileSystem(conf);

    OutputStream output;

    if (isCompressed) {
        FSDataOutputStream fileOut = fs.create(file, false);
        output = new DataOutputStream(codec.createOutputStream(fileOut));
    } else
        output = fs.create(file, false);

    return new QseqRecordWriter(conf, output);
}

From source file:org.springframework.data.hadoop.serialization.ResourceSerializationFormat.java

License:Apache License

/**
 * @return compression default {@link CompressionCodec#getDefaultExtension() extension} if compression alias is
 * specified; <code>empty</code> string otherwise.
 *///from  w  ww  . j a v a2  s .co m
@Override
protected String getDefaultExtension() {
    CompressionCodec codec = CompressionUtils.getHadoopCompression(getConfiguration(), getCompressionAlias());

    return codec != null ? codec.getDefaultExtension() : "";
}

From source file:org.v1.ioformat.NoSeperatorTextOutputFormate.java

License:Apache License

public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    boolean isCompressed = getCompressOutput(job);
    String keyValueSeparator = conf.get("mapred.textoutputformat.separator", "");
    CompressionCodec codec = null;
    String extension = "";
    if (isCompressed) {
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
        extension = codec.getDefaultExtension();
    }//from   w w w .  j a  va 2s . c o  m
    Path file = getDefaultWorkFile(job, extension);
    FileSystem fs = file.getFileSystem(conf);
    if (!isCompressed) {
        FSDataOutputStream fileOut = fs.create(file, false);
        return new LineRecordWriter<K, V>(fileOut, keyValueSeparator);
    } else {
        FSDataOutputStream fileOut = fs.create(file, false);
        return new LineRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)),
                keyValueSeparator);
    }
}

From source file:org.zuinnote.hadoop.office.format.common.HadoopUtil.java

License:Apache License

public static DataOutputStream getDataOutputStream(Configuration conf, Path file, Progressable progress,
        boolean compressed, Class<? extends CompressionCodec> compressorClass) throws IOException {
    if (!compressed) { // uncompressed
        FileSystem fs = file.getFileSystem(conf);
        return fs.create(file, progress);
    } else { // compressed (note partially adapted from TextOutputFormat)
        Class<? extends CompressionCodec> codecClass = compressorClass;
        // create the named codec
        CompressionCodec codec = ReflectionUtils.newInstance(codecClass, conf);
        // provide proper file extension
        Path compressedFile = file.suffix(codec.getDefaultExtension());
        // build the filename including the extension
        FileSystem fs = compressedFile.getFileSystem(conf);
        return new DataOutputStream(codec.createOutputStream(fs.create(compressedFile, progress)));
    }//ww  w  . ja va 2s  . c  om
}

From source file:uk.bl.wa.hadoop.KeylessTextOutputFormat.java

License:Open Source License

@Override
public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress)
        throws IOException {
    boolean isCompressed = getCompressOutput(job);
    if (!isCompressed) {
        Path file = FileOutputFormat.getTaskOutputPath(job, name);
        FileSystem fs = file.getFileSystem(job);
        FSDataOutputStream fileOut = fs.create(file, progress);
        return new LineRecordWriter<K, V>(fileOut);
    } else {/*from w  ww  .j  a  v a  2 s. c  o m*/
        Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class);
        CompressionCodec codec = ReflectionUtils.newInstance(codecClass, job);
        Path file = FileOutputFormat.getTaskOutputPath(job, name + codec.getDefaultExtension());
        FileSystem fs = file.getFileSystem(job);
        FSDataOutputStream fileOut = fs.create(file, progress);
        return new LineRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)));
    }
}