Example usage for org.apache.hadoop.io.compress CompressionCodec createInputStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress CompressionCodec createInputStream.

Prototype

CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException;

Source Link

Document

Create a CompressionInputStream that will read from the given InputStream with the given Decompressor .

Usage

From source file:org.springframework.data.hadoop.store.input.AbstractDataStreamReader.java

License:Apache License

/**
 * Gets the input./*w ww . j a v  a 2 s . c  o m*/
 *
 * @param inputPath the input path
 * @return the input
 * @throws IOException Signals that an I/O exception has occurred.
 */
protected StreamsHolder<InputStream> getInput(Path inputPath) throws IOException {
    log.info("Creating new InputStream");
    StreamsHolder<InputStream> holder = new StreamsHolder<InputStream>();
    final FileSystem fs = getPath().getFileSystem(getConfiguration());
    Path p = inputPath.isAbsolute() ? inputPath : new Path(getPath(), inputPath);
    if (!fs.exists(p)) {
        throw new StoreException("Path " + p + " does not exist");
    }
    if (!isCompressed()) {
        InputStream input = fs.open(p);
        holder.setStream(input);
    } else {
        // TODO: will isCompressed() really guard for npe against getCodec()
        Class<?> clazz = ClassUtils.resolveClassName(getCodec().getCodecClass(), getClass().getClassLoader());
        CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(clazz,
                getConfiguration());
        Decompressor decompressor = CodecPool.getDecompressor(compressionCodec);
        FSDataInputStream winput = fs.open(p);
        InputStream input = compressionCodec.createInputStream(winput, decompressor);
        holder.setWrappedStream(winput);
        holder.setStream(input);
    }
    return holder;
}

From source file:org.zuinnote.hadoop.office.example.MapReduceExcelInputIntegrationTest.java

License:Apache License

private InputStream openFile(Path path) throws IOException {
    CompressionCodec codec = new CompressionCodecFactory(miniCluster.getConfig()).getCodec(path);
    FSDataInputStream fileIn = dfsCluster.getFileSystem().open(path);
    // check if compressed
    if (codec == null) { // uncompressed
        return fileIn;
    } else { // compressed
        Decompressor decompressor = CodecPool.getDecompressor(codec);
        this.openDecompressors.add(decompressor); // to be returned later using close
        if (codec instanceof SplittableCompressionCodec) {
            long end = dfsCluster.getFileSystem().getFileStatus(path).getLen();
            final SplitCompressionInputStream cIn = ((SplittableCompressionCodec) codec).createInputStream(
                    fileIn, decompressor, 0, end, SplittableCompressionCodec.READ_MODE.CONTINUOUS);
            return cIn;
        } else {/* w w  w  .  j  ava 2  s.  co  m*/
            return codec.createInputStream(fileIn, decompressor);
        }
    }
}

From source file:org.zuinnote.hadoop.office.format.common.HadoopFileReader.java

License:Apache License

public InputStream openFile(Path path) throws IOException {
    CompressionCodec codec = compressionCodecs.getCodec(path);
    FSDataInputStream fileIn = fs.open(path);
    // check if compressed
    if (codec == null) { // uncompressed
        LOG.debug("Reading from an uncompressed file \"" + path + "\"");
        return fileIn;
    } else { // compressed
        Decompressor decompressor = CodecPool.getDecompressor(codec);
        this.openDecompressors.add(decompressor); // to be returned later using close
        if (codec instanceof SplittableCompressionCodec) {
            LOG.debug("Reading from a compressed file \"" + path + "\" with splittable compression codec");
            long end = fs.getFileStatus(path).getLen();
            return ((SplittableCompressionCodec) codec).createInputStream(fileIn, decompressor, 0, end,
                    SplittableCompressionCodec.READ_MODE.CONTINUOUS);
        } else {//w w w. j  a v a 2s .  c o m
            LOG.debug("Reading from a compressed file \"" + path + "\" with non-splittable compression codec");
            return codec.createInputStream(fileIn, decompressor);
        }
    }
}