Example usage for org.apache.hadoop.io.compress GzipCodec GzipCodec

List of usage examples for org.apache.hadoop.io.compress GzipCodec GzipCodec

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.compress.*
  5. GzipCodec
  6. GzipCodec

Introduction

In this page you can find the example usage for org.apache.hadoop.io.compress GzipCodec GzipCodec.

Prototype

GzipCodec

Source Link

Usage

From source file:com.pinterest.secor.io.FileReaderWriterFactoryTest.java

License:Apache License

public void testSequenceFileReader() throws Exception {
    setupSequenceFileReaderConfig();//from  w  ww  .j ava  2 s.co m
    mockSequenceFileWriter(false);
    ReflectionUtil.createFileReader(mConfig.getFileReaderWriterFactory(), mLogFilePath, null, mConfig);

    // Verify that the method has been called exactly once (the default).
    PowerMockito.verifyStatic();
    FileSystem.get(Mockito.any(URI.class), Mockito.any(Configuration.class));

    mockSequenceFileWriter(true);
    ReflectionUtil.createFileWriter(mConfig.getFileReaderWriterFactory(), mLogFilePathGz, new GzipCodec(),
            mConfig);

    // Verify that the method has been called exactly once (the default).
    PowerMockito.verifyStatic();
    FileSystem.get(Mockito.any(URI.class), Mockito.any(Configuration.class));
}

From source file:com.pinterest.secor.io.FileReaderWriterFactoryTest.java

License:Apache License

public void testSequenceFileWriter() throws Exception {
    setupSequenceFileReaderConfig();// w ww .  ja v a  2  s . com
    mockSequenceFileWriter(false);

    FileWriter writer = ReflectionUtil.createFileWriter(mConfig.getFileReaderWriterFactory(), mLogFilePath,
            null, mConfig);

    // Verify that the method has been called exactly once (the default).
    PowerMockito.verifyStatic();
    FileSystem.get(Mockito.any(URI.class), Mockito.any(Configuration.class));

    assert writer.getLength() == 123L;

    mockSequenceFileWriter(true);

    writer = ReflectionUtil.createFileWriter(mConfig.getFileReaderWriterFactory(), mLogFilePathGz,
            new GzipCodec(), mConfig);

    // Verify that the method has been called exactly once (the default).
    PowerMockito.verifyStatic();
    FileSystem.get(Mockito.any(URI.class), Mockito.any(Configuration.class));

    assert writer.getLength() == 12L;
}

From source file:com.pinterest.secor.io.FileReaderWriterFactoryTest.java

License:Apache License

public void testDelimitedTextFileWriter() throws Exception {
    setupDelimitedTextFileWriterConfig();
    mockDelimitedTextFileWriter(false);/*  w ww  .  ja  va  2s.c  o  m*/
    FileWriter writer = (FileWriter) ReflectionUtil.createFileWriter(mConfig.getFileReaderWriterFactory(),
            mLogFilePath, null, mConfig);
    assert writer.getLength() == 0L;

    mockDelimitedTextFileWriter(true);
    writer = (FileWriter) ReflectionUtil.createFileWriter(mConfig.getFileReaderWriterFactory(), mLogFilePathGz,
            new GzipCodec(), mConfig);
    assert writer.getLength() == 0L;
}

From source file:com.pinterest.secor.io.FileReaderWriterFactoryTest.java

License:Apache License

public void testDelimitedTextFileReader() throws Exception {
    setupDelimitedTextFileWriterConfig();

    mockDelimitedTextFileWriter(false);/*from w w w  .j  a va 2  s  . c  o m*/

    ReflectionUtil.createFileReader(mConfig.getFileReaderWriterFactory(), mLogFilePath, null, mConfig);

    mockDelimitedTextFileWriter(true);
    ReflectionUtil.createFileReader(mConfig.getFileReaderWriterFactory(), mLogFilePathGz, new GzipCodec(),
            mConfig);
}

From source file:com.pinterest.secor.io.FileReaderWriterTest.java

License:Apache License

public void testSequenceFileReader() throws Exception {
    setupSequenceFileReaderConfig();// w  ww . j a  v  a2 s . c o  m
    mockSequenceFileReaderWriter(false);
    ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(), mLogFilePath, null,
            FileReaderWriter.Type.Reader);

    // Verify that the method has been called exactly once (the default).
    PowerMockito.verifyStatic();
    FileSystem.get(Mockito.any(URI.class), Mockito.any(Configuration.class));

    mockSequenceFileReaderWriter(true);
    ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(), mLogFilePathGz, new GzipCodec(),
            FileReaderWriter.Type.Reader);

    // Verify that the method has been called exactly once (the default).
    PowerMockito.verifyStatic();
    FileSystem.get(Mockito.any(URI.class), Mockito.any(Configuration.class));
}

From source file:com.pinterest.secor.io.FileReaderWriterTest.java

License:Apache License

public void testSequenceFileWriter() throws Exception {
    setupSequenceFileReaderConfig();//from   w  w  w  . ja va2s  .com
    mockSequenceFileReaderWriter(false);

    FileReaderWriter writer = (FileReaderWriter) ReflectionUtil.createFileReaderWriter(
            mConfig.getFileReaderWriter(), mLogFilePath, null, FileReaderWriter.Type.Writer);

    // Verify that the method has been called exactly once (the default).
    PowerMockito.verifyStatic();
    FileSystem.get(Mockito.any(URI.class), Mockito.any(Configuration.class));

    assert writer.getLength() == 123L;

    mockSequenceFileReaderWriter(true);

    writer = (FileReaderWriter) ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(),
            mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Writer);

    // Verify that the method has been called exactly once (the default).
    PowerMockito.verifyStatic();
    FileSystem.get(Mockito.any(URI.class), Mockito.any(Configuration.class));

    assert writer.getLength() == 12L;
}

From source file:com.pinterest.secor.io.FileReaderWriterTest.java

License:Apache License

public void testDelimitedTextFileWriter() throws Exception {
    setupDelimitedTextFileReaderWriterConfig();
    mockDelimitedTextFileReaderWriter(false);
    FileReaderWriter writer = (FileReaderWriter) ReflectionUtil.createFileReaderWriter(
            mConfig.getFileReaderWriter(), mLogFilePath, null, FileReaderWriter.Type.Writer);
    assert writer.getLength() == 0L;

    mockDelimitedTextFileReaderWriter(true);
    writer = (FileReaderWriter) ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(),
            mLogFilePathGz, new GzipCodec(), FileReaderWriter.Type.Writer);
    assert writer.getLength() == 0L;
}

From source file:com.pinterest.secor.io.FileReaderWriterTest.java

License:Apache License

public void testDelimitedTextFileReader() throws Exception {
    setupDelimitedTextFileReaderWriterConfig();

    mockDelimitedTextFileReaderWriter(false);
    ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(), mLogFilePath, null,
            FileReaderWriter.Type.Reader);

    mockDelimitedTextFileReaderWriter(true);
    ReflectionUtil.createFileReaderWriter(mConfig.getFileReaderWriter(), mLogFilePathGz, new GzipCodec(),
            FileReaderWriter.Type.Reader);
}

From source file:com.twitter.elephanttwin.util.HdfsUtils.java

License:Apache License

/**
 * Same as {@link openInputStream}, except with a supplier as an extra level of indirection.
 *
 * @param dataFile//  w  w w . j a  v a2s.co m
 * @return InputSupplier that provides the input stream
 * @throws IOException
 */
public static InputSupplier<InputStream> getInputStreamSupplier(String dataFile) throws IOException {
    Preconditions.checkNotNull(dataFile);

    final InputStream in;
    if (dataFile.endsWith(".lzo")) {
        // Properly handle compressed files.
        LzopCodec codec = new LzopCodec();
        codec.setConf(new Configuration());
        in = codec.createInputStream(openFile(dataFile));
    } else if (dataFile.endsWith(".gz")) {
        GzipCodec codec = new GzipCodec();
        codec.setConf(new Configuration());
        in = codec.createInputStream(openFile(dataFile));
    } else {
        in = openFile(dataFile);
    }

    return new InputSupplier<InputStream>() {
        @Override
        public InputStream getInput() throws IOException {
            // TODO(Jimmy Lin): JohnS noted that this breaks the contract of InputSupplier
            // on 2 counts---the open is not lazy/initiated by the caller of getInput and
            // getInput does not return a fresh stream on each call.
            return in;
        }
    };
}

From source file:fi.tkk.ics.hadoop.bam.TestFastqInputFormat.java

License:Open Source License

@Test
public void testGzCompressedInput() throws IOException {
    // write gzip-compressed data
    GzipCodec codec = new GzipCodec();
    PrintWriter fastqOut = new PrintWriter(
            new BufferedOutputStream(codec.createOutputStream(new FileOutputStream(tempGz))));
    fastqOut.write(twoFastq);//from   w  ww  .j a  v a  2s.  co  m
    fastqOut.close();

    // now try to read it
    split = new FileSplit(new Path(tempGz.toURI().toString()), 0, twoFastq.length(), null);
    FastqRecordReader reader = new FastqRecordReader(conf, split);

    boolean retval = reader.next(key, fragment);
    assertTrue(retval);
    assertEquals("ERR020229.10880 HWI-ST168_161:1:1:1373:2042/1", key.toString());
    assertEquals("TTGGATGATAGGGATTATTTGACTCGAATATTGGAAATAGCTGTTTATATTTTTTAAAAATGGTCTGTAACTGGTGACAGGACGCTTCGAT",
            fragment.getSequence().toString());

    retval = reader.next(key, fragment);
    assertTrue(retval);
    assertEquals("ERR020229.10883 HWI-ST168_161:1:1:1796:2044/1", key.toString());
    assertEquals("TGAGCAGATGTGCTAAAGCTGCTTCTCCCCTAGGATCATTTGTACCTACCAGACTCAGGGAAAGGGGTGAGAATTGGGCCGTGGGGCAAGG",
            fragment.getSequence().toString());
}