List of usage examples for org.apache.commons.compress.archivers.tar TarArchiveInputStream close
public void close() throws IOException
From source file:org.apache.flume.test.util.StagedInstall.java
private void untarTarFile(File tarFile, File destDir) throws Exception { TarArchiveInputStream tarInputStream = null; try {/* ww w . j a v a 2s . co m*/ tarInputStream = new TarArchiveInputStream(new FileInputStream(tarFile)); TarArchiveEntry entry = null; while ((entry = tarInputStream.getNextTarEntry()) != null) { String name = entry.getName(); LOGGER.debug("Next file: " + name); File destFile = new File(destDir, entry.getName()); if (entry.isDirectory()) { destFile.mkdirs(); continue; } File destParent = destFile.getParentFile(); destParent.mkdirs(); OutputStream entryOutputStream = null; try { entryOutputStream = new FileOutputStream(destFile); byte[] buffer = new byte[2048]; int length = 0; while ((length = tarInputStream.read(buffer, 0, 2048)) != -1) { entryOutputStream.write(buffer, 0, length); } } catch (Exception ex) { LOGGER.error("Exception while expanding tar file", ex); throw ex; } finally { if (entryOutputStream != null) { try { entryOutputStream.close(); } catch (Exception ex) { LOGGER.warn("Failed to close entry output stream", ex); } } } } } catch (Exception ex) { LOGGER.error("Exception caught while untarring tar file: " + tarFile.getAbsolutePath(), ex); throw ex; } finally { if (tarInputStream != null) { try { tarInputStream.close(); } catch (Exception ex) { LOGGER.warn("Unable to close tar input stream: " + tarFile.getCanonicalPath(), ex); } } } }
From source file:org.apache.gobblin.data.management.copy.writer.TarArchiveInputStreamDataWriter.java
/** * Untars the passed in {@link FileAwareInputStream} to the task's staging directory. Uses the name of the root * {@link TarArchiveEntry} in the stream as the directory name for the untarred file. The method also commits the data * by moving the file from staging to output directory. * * @see org.apache.gobblin.data.management.copy.writer.FileAwareInputStreamDataWriter#write(org.apache.gobblin.data.management.copy.FileAwareInputStream) *//*from w w w . ja va 2 s . c om*/ @Override public void writeImpl(InputStream inputStream, Path writeAt, CopyableFile copyableFile) throws IOException { this.closer.register(inputStream); TarArchiveInputStream tarIn = new TarArchiveInputStream(inputStream); final ReadableByteChannel inputChannel = Channels.newChannel(tarIn); TarArchiveEntry tarEntry; // flush the first entry in the tar, which is just the root directory tarEntry = tarIn.getNextTarEntry(); String tarEntryRootName = StringUtils.remove(tarEntry.getName(), Path.SEPARATOR); log.info("Unarchiving at " + writeAt); try { while ((tarEntry = tarIn.getNextTarEntry()) != null) { // the API tarEntry.getName() is misleading, it is actually the path of the tarEntry in the tar file String newTarEntryPath = tarEntry.getName().replace(tarEntryRootName, writeAt.getName()); Path tarEntryStagingPath = new Path(writeAt.getParent(), newTarEntryPath); if (!FileUtils.isSubPath(writeAt.getParent(), tarEntryStagingPath)) { throw new IOException( String.format("Extracted file: %s is trying to write outside of output directory: %s", tarEntryStagingPath, writeAt.getParent())); } if (tarEntry.isDirectory() && !this.fs.exists(tarEntryStagingPath)) { this.fs.mkdirs(tarEntryStagingPath); } else if (!tarEntry.isDirectory()) { FSDataOutputStream out = this.fs.create(tarEntryStagingPath, true); final WritableByteChannel outputChannel = Channels.newChannel(out); try { StreamCopier copier = new StreamCopier(inputChannel, outputChannel); if (isInstrumentationEnabled()) { copier.withCopySpeedMeter(this.copySpeedMeter); } this.bytesWritten.addAndGet(copier.copy()); if (isInstrumentationEnabled()) { log.info("File {}: copied {} bytes, average rate: {} B/s", copyableFile.getOrigin().getPath(), this.copySpeedMeter.getCount(), this.copySpeedMeter.getMeanRate()); } else { log.info("File {} copied.", copyableFile.getOrigin().getPath()); } } finally { out.close(); outputChannel.close(); } } } } finally { tarIn.close(); inputChannel.close(); inputStream.close(); } }
From source file:org.apache.hadoop.hive.common.CompressionUtils.java
/** * Untar an input file into an output file. * * The output file is created in the output folder, having the same name as the input file, minus * the '.tar' extension.//from w w w. jav a 2 s . c om * * @param inputFileName the input .tar file * @param outputDirName the output directory file. * @throws IOException * @throws FileNotFoundException * * @return The {@link List} of {@link File}s with the untared content. * @throws ArchiveException */ public static List<File> unTar(final String inputFileName, final String outputDirName, boolean flatten) throws FileNotFoundException, IOException, ArchiveException { File inputFile = new File(inputFileName); File outputDir = new File(outputDirName); final List<File> untaredFiles = new LinkedList<File>(); final InputStream is; if (inputFileName.endsWith(".gz")) { is = new GzipCompressorInputStream(new FileInputStream(inputFile)); } else { is = new FileInputStream(inputFile); } final TarArchiveInputStream debInputStream = (TarArchiveInputStream) new ArchiveStreamFactory() .createArchiveInputStream("tar", is); TarArchiveEntry entry = null; while ((entry = (TarArchiveEntry) debInputStream.getNextEntry()) != null) { final File outputFile = new File(outputDir, entry.getName()); if (entry.isDirectory()) { if (flatten) { // no sub-directories continue; } LOG.debug(String.format("Attempting to write output directory %s.", outputFile.getAbsolutePath())); if (!outputFile.exists()) { LOG.debug(String.format("Attempting to create output directory %s.", outputFile.getAbsolutePath())); if (!outputFile.mkdirs()) { throw new IllegalStateException( String.format("Couldn't create directory %s.", outputFile.getAbsolutePath())); } } } else { final OutputStream outputFileStream; if (flatten) { File flatOutputFile = new File(outputDir, outputFile.getName()); LOG.debug(String.format("Creating flat output file %s.", flatOutputFile.getAbsolutePath())); outputFileStream = new FileOutputStream(flatOutputFile); } else if (!outputFile.getParentFile().exists()) { LOG.debug(String.format("Attempting to create output directory %s.", outputFile.getParentFile().getAbsoluteFile())); if (!outputFile.getParentFile().getAbsoluteFile().mkdirs()) { throw new IllegalStateException(String.format("Couldn't create directory %s.", outputFile.getParentFile().getAbsolutePath())); } LOG.debug(String.format("Creating output file %s.", outputFile.getAbsolutePath())); outputFileStream = new FileOutputStream(outputFile); } else { outputFileStream = new FileOutputStream(outputFile); } IOUtils.copy(debInputStream, outputFileStream); outputFileStream.close(); } untaredFiles.add(outputFile); } debInputStream.close(); return untaredFiles; }
From source file:org.apache.helix.provisioning.yarn.AppLauncher.java
/** * Generates the classpath after the archive file gets extracted under 'serviceName' folder * @param serviceName// www .j av a 2s .c o m * @param archiveFile * @return */ private String generateClasspathAfterExtraction(String serviceName, File archiveFile) { if (!isArchive(archiveFile.getAbsolutePath())) { return "./"; } StringBuilder classpath = new StringBuilder(); // put the jar files under the archive in the classpath try { final InputStream is = new FileInputStream(archiveFile); final TarArchiveInputStream debInputStream = (TarArchiveInputStream) new ArchiveStreamFactory() .createArchiveInputStream("tar", is); TarArchiveEntry entry = null; while ((entry = (TarArchiveEntry) debInputStream.getNextEntry()) != null) { if (entry.isFile()) { classpath.append(File.pathSeparatorChar); classpath.append("./" + serviceName + "/" + entry.getName()); } } debInputStream.close(); } catch (Exception e) { LOG.error("Unable to read archive file:" + archiveFile, e); } return classpath.toString(); }
From source file:org.apache.kylin.common.util.TarGZUtil.java
public static void uncompressTarGZ(File tarFile, File dest) throws IOException { dest.mkdir();//ww w .j a v a 2 s . c o m TarArchiveInputStream tarIn = null; tarIn = new TarArchiveInputStream( new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(tarFile)))); TarArchiveEntry tarEntry = tarIn.getNextTarEntry(); // tarIn is a TarArchiveInputStream while (tarEntry != null) {// create a file with the same name as the tarEntry File destPath = new File(dest, tarEntry.getName()); System.out.println("working: " + destPath.getCanonicalPath()); if (tarEntry.isDirectory()) { destPath.mkdirs(); } else { destPath.createNewFile(); //byte [] btoRead = new byte[(int)tarEntry.getSize()]; byte[] btoRead = new byte[1024]; //FileInputStream fin // = new FileInputStream(destPath.getCanonicalPath()); BufferedOutputStream bout = new BufferedOutputStream(new FileOutputStream(destPath)); int len = 0; while ((len = tarIn.read(btoRead)) != -1) { bout.write(btoRead, 0, len); } bout.close(); btoRead = null; } tarEntry = tarIn.getNextTarEntry(); } tarIn.close(); }
From source file:org.apache.sqoop.test.utils.CompressionUtils.java
/** * Untar given stream (tar.gz archive) to given directory. * * Directory structure will be preserved. * * @param inputStream InputStream of tar.gz archive * @param targetDirectory Target directory for tarball content * @throws IOException/*from w w w .j av a 2s.co m*/ */ public static void untarStreamToDirectory(InputStream inputStream, String targetDirectory) throws IOException { assert inputStream != null; assert targetDirectory != null; LOG.info("Untaring archive to directory: " + targetDirectory); TarArchiveInputStream in = new TarArchiveInputStream(new GzipCompressorInputStream(inputStream)); TarArchiveEntry entry = null; int BUFFER_SIZE = 2048; while ((entry = (TarArchiveEntry) in.getNextEntry()) != null) { LOG.info("Untaring file: " + entry.getName()); if (entry.isDirectory()) { (new File(HdfsUtils.joinPathFragments(targetDirectory, entry.getName()))).mkdirs(); } else { int count; byte data[] = new byte[BUFFER_SIZE]; FileOutputStream fos = new FileOutputStream( HdfsUtils.joinPathFragments(targetDirectory, entry.getName())); BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE); while ((count = in.read(data, 0, BUFFER_SIZE)) != -1) { dest.write(data, 0, count); } dest.close(); } } in.close(); }
From source file:org.apache.tajo.yarn.command.LaunchCommand.java
private String getTajoRootInTar(String tar) throws IOException { TarArchiveInputStream inputStream = null; try {/*from w ww . jav a2s . c o m*/ inputStream = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(tar))); for (TarArchiveEntry entry = inputStream.getNextTarEntry(); entry != null;) { String entryName = entry.getName(); if (entry.isDirectory() && entryName.startsWith("tajo-")) { return entryName.substring(0, entryName.length() - 1); } entry = inputStream.getNextTarEntry(); } } finally { if (inputStream != null) { inputStream.close(); } } throw new IOException("Unable to get tajo home dir from " + tar); }
From source file:org.canova.api.util.ArchiveUtils.java
/** * Extracts files to the specified destination * @param file the file to extract to//from w w w . j a v a 2 s . co m * @param dest the destination directory * @throws java.io.IOException */ public static void unzipFileTo(String file, String dest) throws IOException { File target = new File(file); if (!target.exists()) throw new IllegalArgumentException("Archive doesnt exist"); FileInputStream fin = new FileInputStream(target); int BUFFER = 2048; byte data[] = new byte[BUFFER]; if (file.endsWith(".zip")) { //getFromOrigin the zip file content ZipInputStream zis = new ZipInputStream(fin); //getFromOrigin the zipped file list entry ZipEntry ze = zis.getNextEntry(); while (ze != null) { String fileName = ze.getName(); File newFile = new File(dest + File.separator + fileName); if (ze.isDirectory()) { newFile.mkdirs(); zis.closeEntry(); ze = zis.getNextEntry(); continue; } log.info("file unzip : " + newFile.getAbsoluteFile()); //create all non exists folders //else you will hit FileNotFoundException for compressed folder FileOutputStream fos = new FileOutputStream(newFile); int len; while ((len = zis.read(data)) > 0) { fos.write(data, 0, len); } fos.flush(); fos.close(); zis.closeEntry(); ze = zis.getNextEntry(); } zis.close(); } else if (file.endsWith(".tar")) { BufferedInputStream in = new BufferedInputStream(fin); TarArchiveInputStream tarIn = new TarArchiveInputStream(in); TarArchiveEntry entry = null; /** Read the tar entries using the getNextEntry method **/ while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { log.info("Extracting: " + entry.getName()); /** If the entry is a directory, createComplex the directory. **/ if (entry.isDirectory()) { File f = new File(dest + File.separator + entry.getName()); f.mkdirs(); } /** * If the entry is a file,write the decompressed file to the disk * and close destination stream. **/ else { int count; FileOutputStream fos = new FileOutputStream(dest + File.separator + entry.getName()); BufferedOutputStream destStream = new BufferedOutputStream(fos, BUFFER); while ((count = tarIn.read(data, 0, BUFFER)) != -1) { destStream.write(data, 0, count); } destStream.flush(); ; IOUtils.closeQuietly(destStream); } } /** Close the input stream **/ tarIn.close(); } else if (file.endsWith(".tar.gz") || file.endsWith(".tgz")) { BufferedInputStream in = new BufferedInputStream(fin); GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in); TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn); TarArchiveEntry entry = null; /** Read the tar entries using the getNextEntry method **/ while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { log.info("Extracting: " + entry.getName()); /** If the entry is a directory, createComplex the directory. **/ if (entry.isDirectory()) { File f = new File(dest + File.separator + entry.getName()); f.mkdirs(); } /** * If the entry is a file,write the decompressed file to the disk * and close destination stream. **/ else { int count; FileOutputStream fos = new FileOutputStream(dest + File.separator + entry.getName()); BufferedOutputStream destStream = new BufferedOutputStream(fos, BUFFER); while ((count = tarIn.read(data, 0, BUFFER)) != -1) { destStream.write(data, 0, count); } destStream.flush(); IOUtils.closeQuietly(destStream); } } /** Close the input stream **/ tarIn.close(); } else if (file.endsWith(".gz")) { GZIPInputStream is2 = new GZIPInputStream(fin); File extracted = new File(target.getParent(), target.getName().replace(".gz", "")); if (extracted.exists()) extracted.delete(); extracted.createNewFile(); OutputStream fos = FileUtils.openOutputStream(extracted); IOUtils.copyLarge(is2, fos); is2.close(); fos.flush(); fos.close(); } }
From source file:org.cloudifysource.esc.util.TarGzUtils.java
/** * Extract a tar.gz file./*from w w w.j av a 2 s. c om*/ * * @param source * The file to extract from. * @param destination * The destination folder. * @throws IOException * An error occured during the extraction. */ public static void extract(final File source, final String destination) throws IOException { LOGGER.fine(String.format("Extracting %s to %s", source.getName(), destination)); if (!FilenameUtils.getExtension(source.getName().toLowerCase()).equals("gz")) { throw new IllegalArgumentException("Expecting tar.gz file: " + source.getAbsolutePath()); } if (!new File(destination).isDirectory()) { throw new IllegalArgumentException("Destination should be a folder: " + destination); } /** create a TarArchiveInputStream object. **/ FileInputStream fin = new FileInputStream(source); BufferedInputStream in = new BufferedInputStream(fin); GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in); TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn); TarArchiveEntry entry = null; /** Read the tar entries using the getNextEntry method **/ while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { LOGGER.finer("Extracting: " + entry.getName()); /** If the entry is a directory, create the directory. **/ if (entry.isDirectory()) { File f = new File(destination, entry.getName()); f.mkdirs(); } else { int count; byte[] data = new byte[BUFFER]; FileOutputStream fos = new FileOutputStream(new File(destination, entry.getName())); BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER); while ((count = tarIn.read(data, 0, BUFFER)) != -1) { dest.write(data, 0, count); } dest.close(); } } /** Close the input stream **/ tarIn.close(); }
From source file:org.cmuchimps.gort.modules.helper.CompressionHelper.java
public static void untar(File f) { if (f == null || !f.exists() || !f.canRead()) { return;/*ww w . j a va2 s . c om*/ } else { System.out.println("Untarring file: " + f.getAbsolutePath()); } File parent = f.getParentFile(); if (parent == null || !parent.exists() || !parent.canWrite()) { return; } FileInputStream fin = null; BufferedInputStream in = null; TarArchiveInputStream tarIn = null; TarArchiveEntry entry = null; try { fin = new FileInputStream(f); in = new BufferedInputStream(fin); tarIn = new TarArchiveInputStream(in); while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { final File outputFile = new File(parent, entry.getName()); if (entry.isDirectory()) { if (!outputFile.exists()) { outputFile.mkdirs(); } } else { final FileOutputStream outputFileStream = new FileOutputStream(outputFile); IOUtils.copy(tarIn, outputFileStream); outputFileStream.close(); } //System.out.println("Processed: " + outputFile.getAbsolutePath()); } } catch (FileNotFoundException ex) { ex.printStackTrace(); ; } catch (IOException e) { e.printStackTrace(); } finally { if (in != null) { try { in.close(); } catch (IOException ex) { ex.printStackTrace(); } } if (fin != null) { try { fin.close(); } catch (IOException ex) { ex.printStackTrace(); } } if (tarIn != null) { try { tarIn.close(); } catch (IOException ex) { ex.printStackTrace(); } } } }