Example usage for org.apache.hadoop.mapred FileOutputFormat getOutputCompressorClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred FileOutputFormat getOutputCompressorClass.

Prototype

public static Class<? extends CompressionCodec> getOutputCompressorClass(JobConf conf,
        Class<? extends CompressionCodec> defaultValue) 

Source Link

Document

Get the CompressionCodec for compressing the job outputs.

Usage

From source file:com.datascience.hadoop.CsvOutputFormat.java

License:Apache License

@Override
public RecordWriter<LongWritable, ListWritable<Text>> getRecordWriter(FileSystem fileSystem, JobConf conf,
        String name, Progressable progress) throws IOException {
    String charsetName = conf.get(CHARSET);
    Charset charset = charsetName != null ? Charset.forName(charsetName) : StandardCharsets.UTF_8;

    Path path;/* ww  w. j av a  2 s . c  o m*/
    if (FileOutputFormat.getCompressOutput(conf)) {
        Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(conf,
                GzipCodec.class);
        CompressionCodec codec = ReflectionUtils.newInstance(codecClass, conf);
        path = FileOutputFormat.getTaskOutputPath(conf, name + codec.getDefaultExtension());
    } else {
        path = FileOutputFormat.getTaskOutputPath(conf, name);
    }
    return new CsvRecordWriter(new OutputStreamWriter(path.getFileSystem(conf).create(path, progress), charset),
            createFormat(conf));
}

---

From source file:com.ricemap.spateDB.core.GridRecordWriter.java

License:Apache License

/**
 * Creates an output stream that will be used to write the final cell file
 * @param cellFilePath//from w  ww  . ja v a  2 s . c o m
 * @return
 * @throws IOException 
 */
protected OutputStream createFinalCellStream(Path cellFilePath) throws IOException {
    OutputStream cellStream;
    boolean isCompressed = jobConf != null && FileOutputFormat.getCompressOutput(jobConf);

    if (!isCompressed) {
        // Create new file
        cellStream = fileSystem.create(cellFilePath, true,
                fileSystem.getConf().getInt("io.file.buffer.size", 4096),
                fileSystem.getDefaultReplication(outDir), this.blockSize);
    } else {
        Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jobConf,
                GzipCodec.class);
        // create the named codec
        CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jobConf);

        // Open a stream to the output file
        cellStream = fileSystem.create(cellFilePath, true,
                fileSystem.getConf().getInt("io.file.buffer.size", 4096),
                fileSystem.getDefaultReplication(outDir), this.blockSize);

        // Encode the output stream using the codec
        cellStream = new DataOutputStream(codec.createOutputStream(cellStream));
    }

    return cellStream;
}

---

From source file:com.ricemap.spateDB.core.GridRecordWriter.java

License:Apache License

/**
 * Returns path to a file in which the final cell will be written.
 * @param column//from w  ww. java 2 s . c om
 * @param row
 * @return
 * @throws IOException 
 */
protected Path getFinalCellPath(int cellIndex) throws IOException {
    Path path = null;
    do {
        String filename = counter == 0 ? String.format("data_%05d", cellIndex)
                : String.format("data_%05d_%d", cellIndex, counter);
        boolean isCompressed = jobConf != null && FileOutputFormat.getCompressOutput(jobConf);
        if (isCompressed) {
            Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jobConf,
                    GzipCodec.class);
            // create the named codec
            CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jobConf);
            filename += codec.getDefaultExtension();
        }

        path = getFilePath(filename);
        counter++;
    } while (fileSystem.exists(path));
    return path;
}

---

From source file:edu.umn.cs.spatialHadoop.core.GridRecordWriter.java

License:Open Source License

/**
 * Creates an output stream that will be used to write the final cell file
 * @param cellFilePath/* w ww  . j a  v a  2  s .co m*/
 * @return
 * @throws IOException 
 */
protected OutputStream createFinalCellStream(Path cellFilePath) throws IOException {
    OutputStream cellStream;
    boolean isCompressed = jobConf != null && FileOutputFormat.getCompressOutput(jobConf);

    if (!isCompressed) {
        // Create new file
        cellStream = fileSystem.create(cellFilePath, true,
                fileSystem.getConf().getInt("io.file.buffer.size", 4096),
                fileSystem.getDefaultReplication(cellFilePath), this.blockSize);
    } else {
        Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jobConf,
                GzipCodec.class);
        // create the named codec
        CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jobConf);

        // Open a stream to the output file
        cellStream = fileSystem.create(cellFilePath, true,
                fileSystem.getConf().getInt("io.file.buffer.size", 4096),
                fileSystem.getDefaultReplication(cellFilePath), this.blockSize);

        // Encode the output stream using the codec
        cellStream = new DataOutputStream(codec.createOutputStream(cellStream));
    }

    return cellStream;
}

---

From source file:edu.umn.cs.spatialHadoop.core.GridRecordWriter.java

License:Open Source License

/**
 * Returns path to a file in which the final cell will be written.
 * @param cellIndex The index of the cell to retrieve its output path.
 * @return//www  . j  av a 2s  .  co  m
 * @throws IOException
 */
protected Path getFinalCellPath(int cellIndex) throws IOException {
    Path path;
    do {
        String filename = counter == 0 ? String.format("data_%05d", cellIndex)
                : String.format("data_%05d_%d", cellIndex, counter);
        boolean isCompressed = jobConf != null && FileOutputFormat.getCompressOutput(jobConf);
        if (isCompressed) {
            Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jobConf,
                    GzipCodec.class);
            // create the named codec
            CompressionCodec codec = ReflectionUtils.newInstance(codecClass, jobConf);
            filename += codec.getDefaultExtension();
        }

        path = getFilePath(filename);
        counter++;
    } while (fileSystem.exists(path));
    return path;
}

---