io.druid.java.util.common.CompressionUtils.java Source code

Java tutorial

Introduction

Here is the source code for io.druid.java.util.common.CompressionUtils.java

Source

/*
 * Licensed to Metamarkets Group Inc. (Metamarkets) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. Metamarkets licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package io.druid.java.util.common;

import com.google.common.base.Predicate;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.common.io.ByteSink;
import com.google.common.io.ByteSource;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
import io.druid.java.util.common.io.NativeIO;
import io.druid.java.util.common.logger.Logger;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Enumeration;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

public class CompressionUtils {
    private static final Logger log = new Logger(CompressionUtils.class);
    private static final int DEFAULT_RETRY_COUNT = 3;
    private static final String BZ2_SUFFIX = ".bz2";
    private static final String GZ_SUFFIX = ".gz";
    private static final String XZ_SUFFIX = ".xz";
    private static final String ZIP_SUFFIX = ".zip";

    /**
     * Zip the contents of directory into the file indicated by outputZipFile. Sub directories are skipped
     *
     * @param directory     The directory whose contents should be added to the zip in the output stream.
     * @param outputZipFile The output file to write the zipped data to
     * @param fsync         True if the output file should be fsynced to disk
     *
     * @return The number of bytes (uncompressed) read from the input directory.
     *
     * @throws IOException
     */
    public static long zip(File directory, File outputZipFile, boolean fsync) throws IOException {
        if (!isZip(outputZipFile.getName())) {
            log.warn("No .zip suffix[%s], putting files from [%s] into it anyway.", outputZipFile, directory);
        }

        try (final FileOutputStream out = new FileOutputStream(outputZipFile)) {
            long bytes = zip(directory, out);

            // For explanation of why fsyncing here is a good practice:
            // https://github.com/druid-io/druid/pull/5187#pullrequestreview-85188984
            if (fsync) {
                out.getChannel().force(true);
            }

            return bytes;
        }
    }

    /**
     * Zip the contents of directory into the file indicated by outputZipFile. Sub directories are skipped
     *
     * @param directory     The directory whose contents should be added to the zip in the output stream.
     * @param outputZipFile The output file to write the zipped data to
     *
     * @return The number of bytes (uncompressed) read from the input directory.
     *
     * @throws IOException
     */
    public static long zip(File directory, File outputZipFile) throws IOException {
        return zip(directory, outputZipFile, false);
    }

    /**
     * Zips the contents of the input directory to the output stream. Sub directories are skipped
     *
     * @param directory The directory whose contents should be added to the zip in the output stream.
     * @param out       The output stream to write the zip data to. Caller is responsible for closing this stream.
     *
     * @return The number of bytes (uncompressed) read from the input directory.
     *
     * @throws IOException
     */
    public static long zip(File directory, OutputStream out) throws IOException {
        if (!directory.isDirectory()) {
            throw new IOE("directory[%s] is not a directory", directory);
        }

        final ZipOutputStream zipOut = new ZipOutputStream(out);

        long totalSize = 0;
        for (File file : directory.listFiles()) {
            log.info("Adding file[%s] with size[%,d].  Total size so far[%,d]", file, file.length(), totalSize);
            if (file.length() >= Integer.MAX_VALUE) {
                zipOut.finish();
                throw new IOE("file[%s] too large [%,d]", file, file.length());
            }
            zipOut.putNextEntry(new ZipEntry(file.getName()));
            totalSize += Files.asByteSource(file).copyTo(zipOut);
        }
        zipOut.closeEntry();
        // Workaround for http://hg.openjdk.java.net/jdk8/jdk8/jdk/rev/759aa847dcaf
        zipOut.flush();
        zipOut.finish();

        return totalSize;
    }

    /**
     * Unzip the byteSource to the output directory. If cacheLocally is true, the byteSource is cached to local disk before unzipping.
     * This may cause more predictable behavior than trying to unzip a large file directly off a network stream, for example.
     * * @param byteSource The ByteSource which supplies the zip data
     *
     * @param byteSource   The ByteSource which supplies the zip data
     * @param outDir       The output directory to put the contents of the zip
     * @param shouldRetry  A predicate expression to determine if a new InputStream should be acquired from ByteSource and the copy attempted again
     * @param cacheLocally A boolean flag to indicate if the data should be cached locally
     *
     * @return A FileCopyResult containing the result of writing the zip entries to disk
     *
     * @throws IOException
     */
    public static FileUtils.FileCopyResult unzip(final ByteSource byteSource, final File outDir,
            final Predicate<Throwable> shouldRetry, boolean cacheLocally) throws IOException {
        if (!cacheLocally) {
            try {
                return RetryUtils.retry(() -> unzip(byteSource.openStream(), outDir), shouldRetry,
                        DEFAULT_RETRY_COUNT);
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        } else {
            final File tmpFile = File.createTempFile("compressionUtilZipCache", ZIP_SUFFIX);
            try {
                FileUtils.retryCopy(byteSource, tmpFile, shouldRetry, DEFAULT_RETRY_COUNT);
                return unzip(tmpFile, outDir);
            } finally {
                if (!tmpFile.delete()) {
                    log.warn("Could not delete zip cache at [%s]", tmpFile.toString());
                }
            }
        }
    }

    /**
     * Unzip the byteSource to the output directory. If cacheLocally is true, the byteSource is cached to local disk before unzipping.
     * This may cause more predictable behavior than trying to unzip a large file directly off a network stream, for example.
     *
     * @param byteSource   The ByteSource which supplies the zip data
     * @param outDir       The output directory to put the contents of the zip
     * @param cacheLocally A boolean flag to indicate if the data should be cached locally
     *
     * @return A FileCopyResult containing the result of writing the zip entries to disk
     *
     * @throws IOException
     */
    public static FileUtils.FileCopyResult unzip(final ByteSource byteSource, final File outDir,
            boolean cacheLocally) throws IOException {
        return unzip(byteSource, outDir, FileUtils.IS_EXCEPTION, cacheLocally);
    }

    /**
     * Unzip the pulled file to an output directory. This is only expected to work on zips with lone files, and is not intended for zips with directory structures.
     *
     * @param pulledFile The file to unzip
     * @param outDir     The directory to store the contents of the file.
     *
     * @return a FileCopyResult of the files which were written to disk
     *
     * @throws IOException
     */
    public static FileUtils.FileCopyResult unzip(final File pulledFile, final File outDir) throws IOException {
        if (!(outDir.exists() && outDir.isDirectory())) {
            throw new ISE("outDir[%s] must exist and be a directory", outDir);
        }
        log.info("Unzipping file[%s] to [%s]", pulledFile, outDir);
        final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult();
        try (final ZipFile zipFile = new ZipFile(pulledFile)) {
            final Enumeration<? extends ZipEntry> enumeration = zipFile.entries();
            while (enumeration.hasMoreElements()) {
                final ZipEntry entry = enumeration.nextElement();
                final File outFile = new File(outDir, entry.getName());

                validateZipOutputFile(pulledFile.getCanonicalPath(), outFile, outDir);

                result.addFiles(FileUtils.retryCopy(new ByteSource() {
                    @Override
                    public InputStream openStream() throws IOException {
                        return new BufferedInputStream(zipFile.getInputStream(entry));
                    }
                }, outFile, FileUtils.IS_EXCEPTION, DEFAULT_RETRY_COUNT).getFiles());
            }
        }
        return result;
    }

    public static void validateZipOutputFile(String sourceFilename, final File outFile, final File outDir)
            throws IOException {
        // check for evil zip exploit that allows writing output to arbitrary directories
        final File canonicalOutFile = outFile.getCanonicalFile();
        final String canonicalOutDir = outDir.getCanonicalPath();
        if (!canonicalOutFile.toPath().startsWith(canonicalOutDir)) {
            throw new ISE("Unzipped output path[%s] of sourceFile[%s] does not start with outDir[%s].",
                    canonicalOutFile, sourceFilename, canonicalOutDir);
        }
    }

    /**
     * Unzip from the input stream to the output directory, using the entry's file name as the file name in the output directory.
     * The behavior of directories in the input stream's zip is undefined.
     * If possible, it is recommended to use unzip(ByteStream, File) instead
     *
     * @param in     The input stream of the zip data. This stream is closed
     * @param outDir The directory to copy the unzipped data to
     *
     * @return The FileUtils.FileCopyResult containing information on all the files which were written
     *
     * @throws IOException
     */
    public static FileUtils.FileCopyResult unzip(InputStream in, File outDir) throws IOException {
        try (final ZipInputStream zipIn = new ZipInputStream(in)) {
            final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult();
            ZipEntry entry;
            while ((entry = zipIn.getNextEntry()) != null) {
                final File file = new File(outDir, entry.getName());

                validateZipOutputFile("", file, outDir);

                NativeIO.chunkedCopy(zipIn, file);

                result.addFile(file);
                zipIn.closeEntry();
            }
            return result;
        }
    }

    /**
     * gunzip the file to the output file.
     *
     * @param pulledFile The source of the gz data
     * @param outFile    A target file to put the contents
     *
     * @return The result of the file copy
     *
     * @throws IOException
     */
    public static FileUtils.FileCopyResult gunzip(final File pulledFile, File outFile) {
        return gunzip(Files.asByteSource(pulledFile), outFile);
    }

    /**
     * Unzips the input stream via a gzip filter. use gunzip(ByteSource, File, Predicate) if possible
     *
     * @param in      The input stream to run through the gunzip filter. This stream is closed
     * @param outFile The file to output to
     *
     * @throws IOException
     */
    public static FileUtils.FileCopyResult gunzip(InputStream in, File outFile) throws IOException {
        try (GZIPInputStream gzipInputStream = gzipInputStream(in)) {
            NativeIO.chunkedCopy(gzipInputStream, outFile);
            return new FileUtils.FileCopyResult(outFile);
        }
    }

    /**
     * Fixes java bug 7036144 http://bugs.java.com/bugdatabase/view_bug.do?bug_id=7036144 which affects concatenated GZip
     *
     * @param in The raw input stream
     *
     * @return A GZIPInputStream that can handle concatenated gzip streams in the input
     *
     * @see #decompress(InputStream, String) which should be used instead for streams coming from files
     */
    public static GZIPInputStream gzipInputStream(final InputStream in) throws IOException {
        return new GZIPInputStream(new FilterInputStream(in) {
            @Override
            public int available() throws IOException {
                final int otherAvailable = super.available();
                // Hack. Docs say available() should return an estimate,
                // so we estimate about 1KB to work around available == 0 bug in GZIPInputStream
                return otherAvailable == 0 ? 1 << 10 : otherAvailable;
            }
        });
    }

    /**
     * gunzip from the source stream to the destination stream.
     *
     * @param in  The input stream which is to be decompressed. This stream is closed.
     * @param out The output stream to write to. This stream is closed
     *
     * @return The number of bytes written to the output stream.
     *
     * @throws IOException
     */
    public static long gunzip(InputStream in, OutputStream out) throws IOException {
        try (GZIPInputStream gzipInputStream = gzipInputStream(in)) {
            final long result = ByteStreams.copy(gzipInputStream, out);
            out.flush();
            return result;
        } finally {
            out.close();
        }
    }

    /**
     * A gunzip function to store locally
     *
     * @param in          The factory to produce input streams
     * @param outFile     The file to store the result into
     * @param shouldRetry A predicate to indicate if the Throwable is recoverable
     *
     * @return The count of bytes written to outFile
     */
    public static FileUtils.FileCopyResult gunzip(final ByteSource in, final File outFile,
            Predicate<Throwable> shouldRetry) {
        return FileUtils.retryCopy(new ByteSource() {
            @Override
            public InputStream openStream() throws IOException {
                return gzipInputStream(in.openStream());
            }
        }, outFile, shouldRetry, DEFAULT_RETRY_COUNT);
    }

    /**
     * Gunzip from the input stream to the output file
     *
     * @param in      The compressed input stream to read from
     * @param outFile The file to write the uncompressed results to
     *
     * @return A FileCopyResult of the file written
     */
    public static FileUtils.FileCopyResult gunzip(final ByteSource in, File outFile) {
        return gunzip(in, outFile, FileUtils.IS_EXCEPTION);
    }

    /**
     * Copy inputStream to out while wrapping out in a GZIPOutputStream
     * Closes both input and output
     *
     * @param inputStream The input stream to copy data from. This stream is closed
     * @param out         The output stream to wrap in a GZIPOutputStream before copying. This stream is closed
     *
     * @return The size of the data copied
     *
     * @throws IOException
     */
    public static long gzip(InputStream inputStream, OutputStream out) throws IOException {
        try (GZIPOutputStream outputStream = new GZIPOutputStream(out)) {
            final long result = ByteStreams.copy(inputStream, outputStream);
            out.flush();
            return result;
        } finally {
            inputStream.close();
        }
    }

    /**
     * Gzips the input file to the output
     *
     * @param inFile      The file to gzip
     * @param outFile     A target file to copy the uncompressed contents of inFile to
     * @param shouldRetry Predicate on a potential throwable to determine if the copy should be attempted again.
     *
     * @return The result of the file copy
     *
     * @throws IOException
     */
    public static FileUtils.FileCopyResult gzip(final File inFile, final File outFile,
            Predicate<Throwable> shouldRetry) {
        gzip(Files.asByteSource(inFile), Files.asByteSink(outFile), shouldRetry);
        return new FileUtils.FileCopyResult(outFile);
    }

    public static long gzip(final ByteSource in, final ByteSink out, Predicate<Throwable> shouldRetry) {
        return StreamUtils.retryCopy(in, new ByteSink() {
            @Override
            public OutputStream openStream() throws IOException {
                return new GZIPOutputStream(out.openStream());
            }
        }, shouldRetry, DEFAULT_RETRY_COUNT);
    }

    /**
     * GZip compress the contents of inFile into outFile
     *
     * @param inFile  The source of data
     * @param outFile The destination for compressed data
     *
     * @return A FileCopyResult of the resulting file at outFile
     *
     * @throws IOException
     */
    public static FileUtils.FileCopyResult gzip(final File inFile, final File outFile) {
        return gzip(inFile, outFile, FileUtils.IS_EXCEPTION);
    }

    /**
     * Checks to see if fName is a valid name for a "*.zip" file
     *
     * @param fName The name of the file in question
     *
     * @return True if fName is properly named for a .zip file, false otherwise
     */
    public static boolean isZip(String fName) {
        if (Strings.isNullOrEmpty(fName)) {
            return false;
        }
        return fName.endsWith(ZIP_SUFFIX); // Technically a file named `.zip` would be fine
    }

    /**
     * Checks to see if fName is a valid name for a "*.gz" file
     *
     * @param fName The name of the file in question
     *
     * @return True if fName is a properly named .gz file, false otherwise
     */
    public static boolean isGz(String fName) {
        if (Strings.isNullOrEmpty(fName)) {
            return false;
        }
        return fName.endsWith(GZ_SUFFIX) && fName.length() > GZ_SUFFIX.length();
    }

    /**
     * Get the file name without the .gz extension
     *
     * @param fname The name of the gzip file
     *
     * @return fname without the ".gz" extension
     *
     * @throws IAE if fname is not a valid "*.gz" file name
     */
    public static String getGzBaseName(String fname) {
        final String reducedFname = Files.getNameWithoutExtension(fname);
        if (isGz(fname) && !reducedFname.isEmpty()) {
            return reducedFname;
        }
        throw new IAE("[%s] is not a valid gz file name", fname);
    }

    /**
     * Decompress an input stream from a file, based on the filename.
     */
    public static InputStream decompress(final InputStream in, final String fileName) throws IOException {
        if (fileName.endsWith(GZ_SUFFIX)) {
            return gzipInputStream(in);
        } else if (fileName.endsWith(BZ2_SUFFIX)) {
            return new BZip2CompressorInputStream(in, true);
        } else if (fileName.endsWith(XZ_SUFFIX)) {
            return new XZCompressorInputStream(in, true);
        } else if (fileName.endsWith(ZIP_SUFFIX)) {
            // This reads the first file in the archive.
            final ZipInputStream zipIn = new ZipInputStream(in, StandardCharsets.UTF_8);
            try {
                final ZipEntry nextEntry = zipIn.getNextEntry();
                if (nextEntry == null) {
                    zipIn.close();

                    // No files in the archive - return an empty stream.
                    return new ByteArrayInputStream(new byte[0]);
                }
                return zipIn;
            } catch (IOException e) {
                try {
                    zipIn.close();
                } catch (IOException e2) {
                    e.addSuppressed(e2);
                }
                throw e;
            }
        } else {
            return in;
        }
    }

    // Helper method for unit tests (for checking that we fixed https://snyk.io/research/zip-slip-vulnerability)
    public static void makeEvilZip(File outputFile) throws IOException {
        ZipOutputStream zipOutputStream = new ZipOutputStream(new FileOutputStream(outputFile));
        ZipEntry zipEntry = new ZipEntry("../../../../../../../../../../../../../../../tmp/evil.txt");
        zipOutputStream.putNextEntry(zipEntry);
        byte[] output = StringUtils.toUtf8("evil text");
        zipOutputStream.write(output);
        zipOutputStream.closeEntry();
        zipOutputStream.close();
    }
}