Example usage for org.apache.hadoop.io.compress GzipCodec createInputStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress GzipCodec createInputStream.

Prototype

@Override
    public CompressionInputStream createInputStream(InputStream in) throws IOException 

Source Link

Usage

From source file:com.pinterest.secor.io.FileReaderWriterFactoryTest.java

License:Apache License

private void mockDelimitedTextFileWriter(boolean isCompressed) throws Exception {
    PowerMockito.mockStatic(FileSystem.class);
    FileSystem fs = Mockito.mock(FileSystem.class);
    Mockito.when(FileSystem.get(Mockito.any(URI.class), Mockito.any(Configuration.class))).thenReturn(fs);

    Path fsPath = (!isCompressed) ? new Path(PATH) : new Path(PATH_GZ);

    GzipCodec codec = PowerMockito.mock(GzipCodec.class);
    PowerMockito.whenNew(GzipCodec.class).withNoArguments().thenReturn(codec);

    FSDataInputStream fileInputStream = Mockito.mock(FSDataInputStream.class);
    FSDataOutputStream fileOutputStream = Mockito.mock(FSDataOutputStream.class);

    Mockito.when(fs.open(fsPath)).thenReturn(fileInputStream);
    Mockito.when(fs.create(fsPath)).thenReturn(fileOutputStream);

    CompressionInputStream inputStream = Mockito.mock(CompressionInputStream.class);
    CompressionOutputStream outputStream = Mockito.mock(CompressionOutputStream.class);
    Mockito.when(codec.createInputStream(Mockito.any(InputStream.class))).thenReturn(inputStream);

    Mockito.when(codec.createOutputStream(Mockito.any(OutputStream.class))).thenReturn(outputStream);
}

---

From source file:com.pinterest.secor.io.FileReaderWriterTest.java

License:Apache License

private void mockDelimitedTextFileReaderWriter(boolean isCompressed) throws Exception {
    PowerMockito.mockStatic(FileSystem.class);
    FileSystem fs = Mockito.mock(FileSystem.class);
    Mockito.when(FileSystem.get(Mockito.any(URI.class), Mockito.any(Configuration.class))).thenReturn(fs);

    Path fsPath = (!isCompressed) ? new Path(PATH) : new Path(PATH_GZ);

    GzipCodec codec = PowerMockito.mock(GzipCodec.class);
    PowerMockito.whenNew(GzipCodec.class).withNoArguments().thenReturn(codec);

    FSDataInputStream fileInputStream = Mockito.mock(FSDataInputStream.class);
    FSDataOutputStream fileOutputStream = Mockito.mock(FSDataOutputStream.class);

    Mockito.when(fs.open(fsPath)).thenReturn(fileInputStream);
    Mockito.when(fs.create(fsPath)).thenReturn(fileOutputStream);

    CompressionInputStream inputStream = Mockito.mock(CompressionInputStream.class);
    CompressionOutputStream outputStream = Mockito.mock(CompressionOutputStream.class);
    Mockito.when(codec.createInputStream(Mockito.any(InputStream.class))).thenReturn(inputStream);

    Mockito.when(codec.createOutputStream(Mockito.any(OutputStream.class))).thenReturn(outputStream);
}

---

From source file:org.apache.carbondata.core.datastore.impl.FileFactory.java

License:Apache License

public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize)
        throws IOException {
    path = path.replace("\\", "/");
    boolean gzip = path.endsWith(".gz");
    boolean bzip2 = path.endsWith(".bz2");
    InputStream stream;/* ww w  .  ja v  a  2  s.co  m*/
    switch (fileType) {
    case LOCAL:
        path = getUpdatedFilePath(path, fileType);
        if (gzip) {
            stream = new GZIPInputStream(new FileInputStream(path));
        } else if (bzip2) {
            stream = new BZip2CompressorInputStream(new FileInputStream(path));
        } else {
            stream = new FileInputStream(path);
        }
        break;
    case HDFS:
    case ALLUXIO:
    case VIEWFS:
        Path pt = new Path(path);
        FileSystem fs = pt.getFileSystem(configuration);
        if (bufferSize == -1) {
            stream = fs.open(pt);
        } else {
            stream = fs.open(pt, bufferSize);
        }
        if (gzip) {
            GzipCodec codec = new GzipCodec();
            stream = codec.createInputStream(stream);
        } else if (bzip2) {
            BZip2Codec codec = new BZip2Codec();
            stream = codec.createInputStream(stream);
        }
        break;
    default:
        throw new UnsupportedOperationException("unsupported file system");
    }
    return new DataInputStream(new BufferedInputStream(stream));
}

---