List of usage examples for org.apache.commons.compress.archivers.tar TarArchiveInputStream read
public int read(byte[] buf, int offset, int numToRead) throws IOException
From source file:in.neoandroid.neoupdate.neoUpdate.java
private boolean downloadFile(NewAsset asset, String toPath, TarArchiveInputStream tin, TarArchiveEntry ae) { if (enableDebug) Log.d(TAG, "Start download: " + asset.path + ":NPK: " + (tin != null)); boolean resume = (db.updateAndGetStatus(asset.path, asset.md5) == neoUpdateDB.UPDATE_STATUS.UPDATE_RESUME); String newPath;/*ww w . j a va 2 s . co m*/ if (toPath != null) newPath = toPath; else newPath = mapPath(asset.path); createSubDirectories(newPath); File newFile = new File(newPath); long fromBytes = 0; if (resume) fromBytes = newFile.length(); try { FileOutputStream os = new FileOutputStream(newFile, resume); db.setMd5(asset.path, asset.md5); if (tin != null && ae != null) { // Via NPK final int BUFF_SIZE = (8 * 1024); // Buffer size of 8KB byte[] buffer = new byte[BUFF_SIZE]; int n = 0; long size = ae.getSize(); if (resume && fromBytes > 0 && fromBytes < size) { tin.skip(fromBytes); size -= fromBytes; } while (size > 0) { n = BUFF_SIZE; if (n > size) n = (int) size; n = tin.read(buffer, 0, n); if (n < 0) break; if (n > 0) os.write(buffer, 0, n); } } else if (nConnections <= 0) { // Via Local File System FileInputStream is = new FileInputStream(baseUrl + asset.path); is.getChannel().transferTo(fromBytes, is.getChannel().size() - fromBytes, os.getChannel()); is.close(); } else { // Via Internet HttpResponse resp = HttpWithPostData(asset.path, fromBytes); resp.getEntity().writeTo(os); } db.setDownloaded(asset.path, true); os.close(); } catch (Exception e) { if (enableDebug) e.printStackTrace(); return false; } return true; }
From source file:org.apache.flume.test.util.StagedInstall.java
private void untarTarFile(File tarFile, File destDir) throws Exception { TarArchiveInputStream tarInputStream = null; try {/*from w ww .j a va2 s . co m*/ tarInputStream = new TarArchiveInputStream(new FileInputStream(tarFile)); TarArchiveEntry entry = null; while ((entry = tarInputStream.getNextTarEntry()) != null) { String name = entry.getName(); LOGGER.debug("Next file: " + name); File destFile = new File(destDir, entry.getName()); if (entry.isDirectory()) { destFile.mkdirs(); continue; } File destParent = destFile.getParentFile(); destParent.mkdirs(); OutputStream entryOutputStream = null; try { entryOutputStream = new FileOutputStream(destFile); byte[] buffer = new byte[2048]; int length = 0; while ((length = tarInputStream.read(buffer, 0, 2048)) != -1) { entryOutputStream.write(buffer, 0, length); } } catch (Exception ex) { LOGGER.error("Exception while expanding tar file", ex); throw ex; } finally { if (entryOutputStream != null) { try { entryOutputStream.close(); } catch (Exception ex) { LOGGER.warn("Failed to close entry output stream", ex); } } } } } catch (Exception ex) { LOGGER.error("Exception caught while untarring tar file: " + tarFile.getAbsolutePath(), ex); throw ex; } finally { if (tarInputStream != null) { try { tarInputStream.close(); } catch (Exception ex) { LOGGER.warn("Unable to close tar input stream: " + tarFile.getCanonicalPath(), ex); } } } }
From source file:org.apache.sqoop.test.utils.CompressionUtils.java
/** * Untar given stream (tar.gz archive) to given directory. * * Directory structure will be preserved. * * @param inputStream InputStream of tar.gz archive * @param targetDirectory Target directory for tarball content * @throws IOException/*from w w w . ja v a2 s. co m*/ */ public static void untarStreamToDirectory(InputStream inputStream, String targetDirectory) throws IOException { assert inputStream != null; assert targetDirectory != null; LOG.info("Untaring archive to directory: " + targetDirectory); TarArchiveInputStream in = new TarArchiveInputStream(new GzipCompressorInputStream(inputStream)); TarArchiveEntry entry = null; int BUFFER_SIZE = 2048; while ((entry = (TarArchiveEntry) in.getNextEntry()) != null) { LOG.info("Untaring file: " + entry.getName()); if (entry.isDirectory()) { (new File(HdfsUtils.joinPathFragments(targetDirectory, entry.getName()))).mkdirs(); } else { int count; byte data[] = new byte[BUFFER_SIZE]; FileOutputStream fos = new FileOutputStream( HdfsUtils.joinPathFragments(targetDirectory, entry.getName())); BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE); while ((count = in.read(data, 0, BUFFER_SIZE)) != -1) { dest.write(data, 0, count); } dest.close(); } } in.close(); }
From source file:org.caleydo.data.importer.tcga.FirehoseProvider.java
private TCGAFileInfo extractFileFromTarGzArchive(URL inUrl, String fileToExtract, File outputDirectory, boolean hasTumor) { log.info(inUrl + " download and extract: " + fileToExtract); File targetFile = new File(outputDirectory, fileToExtract); // use cached if (targetFile.exists() && !settings.isCleanCache()) { log.fine(inUrl + " cache hit"); return new TCGAFileInfo(targetFile, inUrl, fileToExtract); }/*ww w .j a va 2s. co m*/ File notFound = new File(outputDirectory, fileToExtract + "-notfound"); if (notFound.exists() && !settings.isCleanCache()) { log.warning(inUrl + " marked as not found"); return null; } String alternativeName = fileToExtract; if (hasTumor) { alternativeName = "/" + tumor.getBaseName() + fileToExtract; fileToExtract = "/" + tumor + fileToExtract; } TarArchiveInputStream tarIn = null; OutputStream out = null; try { InputStream in = new BufferedInputStream(inUrl.openStream()); // ok we have the file tarIn = new TarArchiveInputStream(new GZIPInputStream(in)); // search the correct entry ArchiveEntry act = tarIn.getNextEntry(); while (act != null && !act.getName().endsWith(fileToExtract) && !act.getName().endsWith(alternativeName)) { act = tarIn.getNextEntry(); } if (act == null) // no entry found throw new FileNotFoundException("no entry named: " + fileToExtract + " found"); byte[] buf = new byte[4096]; int n; targetFile.getParentFile().mkdirs(); // use a temporary file to recognize if we have aborted between run String tmpFile = targetFile.getAbsolutePath() + ".tmp"; out = new BufferedOutputStream(new FileOutputStream(tmpFile)); while ((n = tarIn.read(buf, 0, 4096)) > -1) out.write(buf, 0, n); out.close(); Files.move(new File(tmpFile).toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); log.info(inUrl + " extracted " + fileToExtract); return new TCGAFileInfo(targetFile, inUrl, fileToExtract); } catch (FileNotFoundException e) { log.log(Level.WARNING, inUrl + " can't extract" + fileToExtract + ": file not found", e); // file was not found, create a marker to remember this for quicker checks notFound.getParentFile().mkdirs(); try { notFound.createNewFile(); } catch (IOException e1) { log.log(Level.WARNING, inUrl + " can't create not-found marker", e); } return null; } catch (Exception e) { log.log(Level.SEVERE, inUrl + " can't extract" + fileToExtract + ": " + e.getMessage(), e); return null; } finally { Closeables.closeQuietly(tarIn); Closeables.closeQuietly(out); } }
From source file:org.canova.api.util.ArchiveUtils.java
/** * Extracts files to the specified destination * @param file the file to extract to//from ww w . ja v a2 s.co m * @param dest the destination directory * @throws java.io.IOException */ public static void unzipFileTo(String file, String dest) throws IOException { File target = new File(file); if (!target.exists()) throw new IllegalArgumentException("Archive doesnt exist"); FileInputStream fin = new FileInputStream(target); int BUFFER = 2048; byte data[] = new byte[BUFFER]; if (file.endsWith(".zip")) { //getFromOrigin the zip file content ZipInputStream zis = new ZipInputStream(fin); //getFromOrigin the zipped file list entry ZipEntry ze = zis.getNextEntry(); while (ze != null) { String fileName = ze.getName(); File newFile = new File(dest + File.separator + fileName); if (ze.isDirectory()) { newFile.mkdirs(); zis.closeEntry(); ze = zis.getNextEntry(); continue; } log.info("file unzip : " + newFile.getAbsoluteFile()); //create all non exists folders //else you will hit FileNotFoundException for compressed folder FileOutputStream fos = new FileOutputStream(newFile); int len; while ((len = zis.read(data)) > 0) { fos.write(data, 0, len); } fos.flush(); fos.close(); zis.closeEntry(); ze = zis.getNextEntry(); } zis.close(); } else if (file.endsWith(".tar")) { BufferedInputStream in = new BufferedInputStream(fin); TarArchiveInputStream tarIn = new TarArchiveInputStream(in); TarArchiveEntry entry = null; /** Read the tar entries using the getNextEntry method **/ while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { log.info("Extracting: " + entry.getName()); /** If the entry is a directory, createComplex the directory. **/ if (entry.isDirectory()) { File f = new File(dest + File.separator + entry.getName()); f.mkdirs(); } /** * If the entry is a file,write the decompressed file to the disk * and close destination stream. **/ else { int count; FileOutputStream fos = new FileOutputStream(dest + File.separator + entry.getName()); BufferedOutputStream destStream = new BufferedOutputStream(fos, BUFFER); while ((count = tarIn.read(data, 0, BUFFER)) != -1) { destStream.write(data, 0, count); } destStream.flush(); ; IOUtils.closeQuietly(destStream); } } /** Close the input stream **/ tarIn.close(); } else if (file.endsWith(".tar.gz") || file.endsWith(".tgz")) { BufferedInputStream in = new BufferedInputStream(fin); GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in); TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn); TarArchiveEntry entry = null; /** Read the tar entries using the getNextEntry method **/ while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { log.info("Extracting: " + entry.getName()); /** If the entry is a directory, createComplex the directory. **/ if (entry.isDirectory()) { File f = new File(dest + File.separator + entry.getName()); f.mkdirs(); } /** * If the entry is a file,write the decompressed file to the disk * and close destination stream. **/ else { int count; FileOutputStream fos = new FileOutputStream(dest + File.separator + entry.getName()); BufferedOutputStream destStream = new BufferedOutputStream(fos, BUFFER); while ((count = tarIn.read(data, 0, BUFFER)) != -1) { destStream.write(data, 0, count); } destStream.flush(); IOUtils.closeQuietly(destStream); } } /** Close the input stream **/ tarIn.close(); } else if (file.endsWith(".gz")) { GZIPInputStream is2 = new GZIPInputStream(fin); File extracted = new File(target.getParent(), target.getName().replace(".gz", "")); if (extracted.exists()) extracted.delete(); extracted.createNewFile(); OutputStream fos = FileUtils.openOutputStream(extracted); IOUtils.copyLarge(is2, fos); is2.close(); fos.flush(); fos.close(); } }
From source file:org.cloudifysource.esc.util.TarGzUtils.java
/** * Extract a tar.gz file./*from w w w . j a v a 2 s. c o m*/ * * @param source * The file to extract from. * @param destination * The destination folder. * @throws IOException * An error occured during the extraction. */ public static void extract(final File source, final String destination) throws IOException { LOGGER.fine(String.format("Extracting %s to %s", source.getName(), destination)); if (!FilenameUtils.getExtension(source.getName().toLowerCase()).equals("gz")) { throw new IllegalArgumentException("Expecting tar.gz file: " + source.getAbsolutePath()); } if (!new File(destination).isDirectory()) { throw new IllegalArgumentException("Destination should be a folder: " + destination); } /** create a TarArchiveInputStream object. **/ FileInputStream fin = new FileInputStream(source); BufferedInputStream in = new BufferedInputStream(fin); GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in); TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn); TarArchiveEntry entry = null; /** Read the tar entries using the getNextEntry method **/ while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { LOGGER.finer("Extracting: " + entry.getName()); /** If the entry is a directory, create the directory. **/ if (entry.isDirectory()) { File f = new File(destination, entry.getName()); f.mkdirs(); } else { int count; byte[] data = new byte[BUFFER]; FileOutputStream fos = new FileOutputStream(new File(destination, entry.getName())); BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER); while ((count = tarIn.read(data, 0, BUFFER)) != -1) { dest.write(data, 0, count); } dest.close(); } } /** Close the input stream **/ tarIn.close(); }
From source file:org.dataconservancy.packaging.tool.impl.generator.BagItPackageAssemblerTest.java
/** * Test that the bag-info.txt file contains the parameter information passed in, including * parameters passed in after the initialization. * * Note that the package being generated in this case is empty, so the bag size and payload * oxum values will be 0./* w w w . j a v a 2 s. c o m*/ * @throws CompressorException * @throws ArchiveException * @throws IOException */ @Test public void testBagItInfoFile() throws CompressorException, ArchiveException, IOException { final String paramName = "TEST_PARAMETER"; final String paramValue = "test parameter"; underTest.addParameter(paramName, paramValue); Package pkg = underTest.assemblePackage(); CompressorInputStream cis = new CompressorStreamFactory() .createCompressorInputStream(CompressorStreamFactory.GZIP, pkg.serialize()); TarArchiveInputStream ais = (TarArchiveInputStream) (new ArchiveStreamFactory() .createArchiveInputStream(ArchiveStreamFactory.TAR, cis)); String bagInfo = ""; TarArchiveEntry entry = ais.getNextTarEntry(); while (entry != null) { if (entry.getName().contains("bag-info.txt")) { byte[] content = new byte[(int) entry.getSize()]; ais.read(content, 0, (int) entry.getSize()); bagInfo = new String(content); break; } entry = ais.getNextTarEntry(); } // Test that expected initial parameters are present String expected = GeneralParameterNames.PACKAGE_NAME + ": " + packageName; assertTrue("Expected to find: " + expected, bagInfo.contains(expected)); // These two values should be 0 since there is nothing in the test package this time. expected = BagItParameterNames.BAG_SIZE + ": 0"; assertTrue("Expected to find: " + expected, bagInfo.contains(expected)); expected = BagItParameterNames.PAYLOAD_OXUM + ": 0"; assertTrue("Expected to find: " + expected, bagInfo.contains(expected)); // Test the post-init parameter expected = paramName + ": " + paramValue; assertTrue("Expected to find: " + expected, bagInfo.contains(expected)); }
From source file:org.datavec.api.util.ArchiveUtils.java
/** * Extracts files to the specified destination * @param file the file to extract to//from w w w .ja v a 2s. c o m * @param dest the destination directory * @throws java.io.IOException */ public static void unzipFileTo(String file, String dest) throws IOException { File target = new File(file); if (!target.exists()) throw new IllegalArgumentException("Archive doesnt exist"); FileInputStream fin = new FileInputStream(target); int BUFFER = 2048; byte data[] = new byte[BUFFER]; if (file.endsWith(".zip") || file.endsWith(".jar")) { //getFromOrigin the zip file content ZipInputStream zis = new ZipInputStream(fin); //getFromOrigin the zipped file list entry ZipEntry ze = zis.getNextEntry(); while (ze != null) { String fileName = ze.getName(); File newFile = new File(dest + File.separator + fileName); if (ze.isDirectory()) { newFile.mkdirs(); zis.closeEntry(); ze = zis.getNextEntry(); continue; } log.info("file unzip : " + newFile.getAbsoluteFile()); //create all non exists folders //else you will hit FileNotFoundException for compressed folder FileOutputStream fos = new FileOutputStream(newFile); int len; while ((len = zis.read(data)) > 0) { fos.write(data, 0, len); } fos.flush(); fos.close(); zis.closeEntry(); ze = zis.getNextEntry(); } zis.close(); } else if (file.endsWith(".tar")) { BufferedInputStream in = new BufferedInputStream(fin); TarArchiveInputStream tarIn = new TarArchiveInputStream(in); TarArchiveEntry entry = null; /** Read the tar entries using the getNextEntry method **/ while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { log.info("Extracting: " + entry.getName()); /** If the entry is a directory, createComplex the directory. **/ if (entry.isDirectory()) { File f = new File(dest + File.separator + entry.getName()); f.mkdirs(); } /** * If the entry is a file,write the decompressed file to the disk * and close destination stream. **/ else { int count; FileOutputStream fos = new FileOutputStream(dest + File.separator + entry.getName()); BufferedOutputStream destStream = new BufferedOutputStream(fos, BUFFER); while ((count = tarIn.read(data, 0, BUFFER)) != -1) { destStream.write(data, 0, count); } destStream.flush(); ; IOUtils.closeQuietly(destStream); } } /** Close the input stream **/ tarIn.close(); } else if (file.endsWith(".tar.gz") || file.endsWith(".tgz")) { BufferedInputStream in = new BufferedInputStream(fin); GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in); TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn); TarArchiveEntry entry = null; /** Read the tar entries using the getNextEntry method **/ while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { log.info("Extracting: " + entry.getName()); /** If the entry is a directory, createComplex the directory. **/ if (entry.isDirectory()) { File f = new File(dest + File.separator + entry.getName()); f.mkdirs(); } /** * If the entry is a file,write the decompressed file to the disk * and close destination stream. **/ else { int count; FileOutputStream fos = new FileOutputStream(dest + File.separator + entry.getName()); BufferedOutputStream destStream = new BufferedOutputStream(fos, BUFFER); while ((count = tarIn.read(data, 0, BUFFER)) != -1) { destStream.write(data, 0, count); } destStream.flush(); IOUtils.closeQuietly(destStream); } } /** Close the input stream **/ tarIn.close(); } else if (file.endsWith(".gz")) { GZIPInputStream is2 = new GZIPInputStream(fin); File extracted = new File(target.getParent(), target.getName().replace(".gz", "")); if (extracted.exists()) extracted.delete(); extracted.createNewFile(); OutputStream fos = FileUtils.openOutputStream(extracted); IOUtils.copyLarge(is2, fos); is2.close(); fos.flush(); fos.close(); } }
From source file:org.deeplearning4j.util.ArchiveUtils.java
/** * Extracts files to the specified destination * @param file the file to extract to//from w w w .j av a 2s . c o m * @param dest the destination directory * @throws IOException */ public static void unzipFileTo(String file, String dest) throws IOException { File target = new File(file); if (!target.exists()) throw new IllegalArgumentException("Archive doesnt exist"); FileInputStream fin = new FileInputStream(target); int BUFFER = 2048; byte data[] = new byte[BUFFER]; if (file.endsWith(".zip")) { //getFromOrigin the zip file content ZipInputStream zis = new ZipInputStream(fin); //getFromOrigin the zipped file list entry ZipEntry ze = zis.getNextEntry(); while (ze != null) { String fileName = ze.getName(); File newFile = new File(dest + File.separator + fileName); log.info("file unzip : " + newFile.getAbsoluteFile()); //createComplex all non exists folders //else you will hit FileNotFoundException for compressed folder new File(newFile.getParent()).mkdirs(); FileOutputStream fos = new FileOutputStream(newFile); int len; while ((len = zis.read(data)) > 0) { fos.write(data, 0, len); } fos.close(); ze = zis.getNextEntry(); } zis.closeEntry(); zis.close(); } else if (file.endsWith(".tar.gz") || file.endsWith(".tgz")) { BufferedInputStream in = new BufferedInputStream(fin); GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in); TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn); TarArchiveEntry entry = null; /** Read the tar entries using the getNextEntry method **/ while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { log.info("Extracting: " + entry.getName()); /** If the entry is a directory, createComplex the directory. **/ if (entry.isDirectory()) { File f = new File(dest + File.separator + entry.getName()); f.mkdirs(); } /** * If the entry is a file,write the decompressed file to the disk * and close destination stream. **/ else { int count; FileOutputStream fos = new FileOutputStream(dest + File.separator + entry.getName()); BufferedOutputStream destStream = new BufferedOutputStream(fos, BUFFER); while ((count = tarIn.read(data, 0, BUFFER)) != -1) { destStream.write(data, 0, count); } destStream.flush(); IOUtils.closeQuietly(destStream); } } /** Close the input stream **/ tarIn.close(); } else if (file.endsWith(".gz")) { GZIPInputStream is2 = new GZIPInputStream(fin); File extracted = new File(target.getParent(), target.getName().replace(".gz", "")); if (extracted.exists()) extracted.delete(); extracted.createNewFile(); OutputStream fos = FileUtils.openOutputStream(extracted); IOUtils.copyLarge(is2, fos); is2.close(); fos.flush(); fos.close(); } target.delete(); }
From source file:org.fuin.esmp.EventStoreDownloadMojo.java
private void unTarGz(final File archive, final File destDir) throws MojoExecutionException { try {/*from w w w . ja v a 2 s. c om*/ final TarArchiveInputStream tarIn = new TarArchiveInputStream( new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(archive)))); try { TarArchiveEntry entry; while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { LOG.info("Extracting: " + entry.getName()); final File file = new File(destDir, entry.getName()); if (entry.isDirectory()) { createIfNecessary(file); } else { int count; final byte[] data = new byte[MB]; final FileOutputStream fos = new FileOutputStream(file); final BufferedOutputStream dest = new BufferedOutputStream(fos, MB); try { while ((count = tarIn.read(data, 0, MB)) != -1) { dest.write(data, 0, count); } } finally { dest.close(); } entry.getMode(); } applyFileMode(file, new FileMode(entry.getMode())); } } finally { tarIn.close(); } } catch (final IOException ex) { throw new MojoExecutionException("Error uncompressing event store archive: " + archive, ex); } }