List of usage examples for org.apache.commons.compress.archivers.zip ZipArchiveEntry getCrc
public long getCrc()
From source file:fr.ens.biologie.genomique.eoulsan.modules.mapping.hadoop.ReadsMapperHadoopModule.java
/** * Compute the checksum of a ZIP file./*w w w . j a va 2 s . c o m*/ * @param in input stream * @return the checksum as a string * @throws IOException if an error occurs while creating the checksum */ private static String computeZipCheckSum(final InputStream in) throws IOException { ZipArchiveInputStream zais = new ZipArchiveInputStream(in); // Create Hash function final Hasher hs = Hashing.md5().newHasher(); // Store entries in a map final Map<String, long[]> map = new HashMap<>(); ZipArchiveEntry e; while ((e = zais.getNextZipEntry()) != null) { map.put(e.getName(), new long[] { e.getSize(), e.getCrc() }); } zais.close(); // Add values to hash function in an ordered manner for (String filename : new TreeSet<>(map.keySet())) { hs.putString(filename, StandardCharsets.UTF_8); for (long l : map.get(filename)) { hs.putLong(l); } } return hs.hash().toString(); }
From source file:at.spardat.xma.xdelta.JarDelta.java
/** * Entry to new name./*from www . jav a 2 s. co m*/ * * @param source the source * @param name the name * @return the zip archive entry * @throws ZipException the zip exception */ public static ZipArchiveEntry entryToNewName(ZipArchiveEntry source, String name) throws ZipException { if (source.getName().equals(name)) return new ZipArchiveEntry(source); ZipArchiveEntry ret = new ZipArchiveEntry(name); byte[] extra = source.getExtra(); if (extra != null) { ret.setExtraFields(ExtraFieldUtils.parse(extra, true, ExtraFieldUtils.UnparseableExtraField.READ)); } else { ret.setExtra(ExtraFieldUtils.mergeLocalFileDataData(source.getExtraFields(true))); } ret.setInternalAttributes(source.getInternalAttributes()); ret.setExternalAttributes(source.getExternalAttributes()); ret.setExtraFields(source.getExtraFields(true)); ret.setCrc(source.getCrc()); ret.setMethod(source.getMethod()); ret.setSize(source.getSize()); return ret; }
From source file:at.spardat.xma.xdelta.JarDelta.java
/** * Find best source./*from w ww . j av a 2s . c o m*/ * * @param source the source * @param target the target * @param targetEntry the target entry * @return the zip archive entry * @throws IOException Signals that an I/O exception has occurred. */ public ZipArchiveEntry findBestSource(ZipFile source, ZipFile target, ZipArchiveEntry targetEntry) throws IOException { ArrayList<ZipArchiveEntry> ret = new ArrayList<>(); for (ZipArchiveEntry next : source.getEntries(targetEntry.getName())) { if (next.getCrc() == targetEntry.getCrc()) return next; ret.add(next); } if (ret.size() == 0) return null; if (ret.size() == 1 || targetEntry.isDirectory()) return ret.get(0); //More than one and no matching crc --- need to calculate xdeltas and pick the shortest ZipArchiveEntry retEntry = null; for (ZipArchiveEntry sourceEntry : ret) { try (ByteArrayOutputStream outbytes = new ByteArrayOutputStream()) { Delta d = new Delta(); DiffWriter diffWriter = new GDiffWriter(new DataOutputStream(outbytes)); int sourceSize = (int) sourceEntry.getSize(); byte[] sourceBytes = new byte[sourceSize]; try (InputStream sourceStream = source.getInputStream(sourceEntry)) { for (int erg = sourceStream.read(sourceBytes); erg < sourceBytes.length; erg += sourceStream .read(sourceBytes, erg, sourceBytes.length - erg)) ; } d.compute(sourceBytes, target.getInputStream(targetEntry), diffWriter); byte[] nextDiff = outbytes.toByteArray(); if (calculatedDelta == null || calculatedDelta.length > nextDiff.length) { retEntry = sourceEntry; calculatedDelta = nextDiff; } } } return retEntry; }
From source file:at.spardat.xma.xdelta.JarDelta.java
/** * Test if the content of two byte arrays is completly identical. * * @param sourceEntry the source entry//from w ww . ja v a2s . c o m * @param targetEntry the target entry * @return true if source and target contain the same bytes. */ public boolean equal(ZipArchiveEntry sourceEntry, ZipArchiveEntry targetEntry) { return (sourceEntry.getSize() == targetEntry.getSize()) && (sourceEntry.getCrc() == targetEntry.getCrc()); }
From source file:at.spardat.xma.xdelta.JarPatcher.java
/** * Gets the entry.//from w w w. ja v a2 s.co m * * @param source the source * @param name the name * @param crc the crc * @return the entry */ private ZipArchiveEntry getEntry(ZipFile source, String name, long crc) { for (ZipArchiveEntry next : source.getEntries(name)) { if (next.getCrc() == crc) return next; } if (!JarDelta.zipFilesPattern.matcher(name).matches()) { return null; } else { return source.getEntry(name); } }
From source file:at.spardat.xma.xdelta.JarPatcher.java
/** * Close entry./* w w w.j a va 2s .c o m*/ * * @param output the output * @param outEntry the out entry * @param crc the crc * @throws IOException Signals that an I/O exception has occurred. */ private void closeEntry(ZipArchiveOutputStream output, ZipArchiveEntry outEntry, long crc) throws IOException { output.flush(); output.closeArchiveEntry(); if (outEntry.getCrc() != crc) throw new IOException("CRC mismatch for " + outEntry.getName()); }
From source file:at.spardat.xma.xdelta.JarDelta.java
/** * Compute delta.//from w ww . ja v a2 s . c om * * @param source the source * @param target the target * @param output the output * @param list the list * @param prefix the prefix * @throws IOException Signals that an I/O exception has occurred. */ public void computeDelta(ZipFile source, ZipFile target, ZipArchiveOutputStream output, PrintWriter list, String prefix) throws IOException { try { for (Enumeration<ZipArchiveEntry> enumer = target.getEntries(); enumer.hasMoreElements();) { calculatedDelta = null; ZipArchiveEntry targetEntry = enumer.nextElement(); ZipArchiveEntry sourceEntry = findBestSource(source, target, targetEntry); String nextEntryName = prefix + targetEntry.getName(); if (sourceEntry != null && zipFilesPattern.matcher(sourceEntry.getName()).matches() && !equal(sourceEntry, targetEntry)) { nextEntryName += "!"; } nextEntryName += "|" + Long.toHexString(targetEntry.getCrc()); if (sourceEntry != null) { nextEntryName += ":" + Long.toHexString(sourceEntry.getCrc()); } else { nextEntryName += ":0"; } list.println(nextEntryName); if (targetEntry.isDirectory()) { if (sourceEntry == null) { ZipArchiveEntry outputEntry = entryToNewName(targetEntry, prefix + targetEntry.getName()); output.putArchiveEntry(outputEntry); output.closeArchiveEntry(); } } else { if (sourceEntry == null || sourceEntry.getSize() <= Delta.DEFAULT_CHUNK_SIZE || targetEntry.getSize() <= Delta.DEFAULT_CHUNK_SIZE) { // new Entry od. alter Eintrag od. neuer Eintrag leer ZipArchiveEntry outputEntry = entryToNewName(targetEntry, prefix + targetEntry.getName()); output.putArchiveEntry(outputEntry); try (InputStream in = target.getInputStream(targetEntry)) { int read = 0; while (-1 < (read = in.read(buffer))) { output.write(buffer, 0, read); } output.flush(); } output.closeArchiveEntry(); } else { if (!equal(sourceEntry, targetEntry)) { if (zipFilesPattern.matcher(sourceEntry.getName()).matches()) { File embeddedTarget = File.createTempFile("jardelta-tmp", ".zip"); File embeddedSource = File.createTempFile("jardelta-tmp", ".zip"); try (FileOutputStream out = new FileOutputStream(embeddedSource); InputStream in = source.getInputStream(sourceEntry); FileOutputStream out2 = new FileOutputStream(embeddedTarget); InputStream in2 = target.getInputStream(targetEntry)) { int read = 0; while (-1 < (read = in.read(buffer))) { out.write(buffer, 0, read); } out.flush(); read = 0; while (-1 < (read = in2.read(buffer))) { out2.write(buffer, 0, read); } out2.flush(); computeDelta(new ZipFile(embeddedSource), new ZipFile(embeddedTarget), output, list, prefix + sourceEntry.getName() + "!"); } finally { embeddedSource.delete(); embeddedTarget.delete(); } } else { ZipArchiveEntry outputEntry = new ZipArchiveEntry( prefix + targetEntry.getName() + ".gdiff"); outputEntry.setTime(targetEntry.getTime()); outputEntry.setComment("" + targetEntry.getCrc()); output.putArchiveEntry(outputEntry); if (calculatedDelta != null) { output.write(calculatedDelta); output.flush(); } else { try (ByteArrayOutputStream outbytes = new ByteArrayOutputStream()) { Delta d = new Delta(); DiffWriter diffWriter = new GDiffWriter(new DataOutputStream(outbytes)); int sourceSize = (int) sourceEntry.getSize(); byte[] sourceBytes = new byte[sourceSize]; try (InputStream sourceStream = source.getInputStream(sourceEntry)) { for (int erg = sourceStream.read( sourceBytes); erg < sourceBytes.length; erg += sourceStream .read(sourceBytes, erg, sourceBytes.length - erg)) ; } d.compute(sourceBytes, target.getInputStream(targetEntry), diffWriter); output.write(outbytes.toByteArray()); } } output.closeArchiveEntry(); } } } } } } finally { source.close(); target.close(); } }
From source file:cz.muni.fi.xklinec.zipstream.Mallory.java
/** * Entry point. // w w w .j a va 2 s .c o m * * @param args * @throws FileNotFoundException * @throws IOException * @throws NoSuchFieldException * @throws ClassNotFoundException * @throws NoSuchMethodException * @throws java.lang.InterruptedException * @throws java.lang.CloneNotSupportedException */ public void doMain(String[] args) throws FileNotFoundException, IOException, NoSuchFieldException, ClassNotFoundException, NoSuchMethodException, InterruptedException, CloneNotSupportedException { // command line argument parser CmdLineParser parser = new CmdLineParser(this); // if you have a wider console, you could increase the value; // here 80 is also the default parser.setUsageWidth(80); try { // parse the arguments. parser.parseArgument(args); } catch (CmdLineException e) { // if there's a problem in the command line, // you'll get this exception. this will report // an error message. System.err.println(e.getMessage()); System.err.println("java Mallory [options...] arguments..."); // print the list of available options parser.printUsage(System.err); System.err.println(); // print option sample. This is useful some time System.err.println(" Example: java Mallory " + parser.printExample(ExampleMode.ALL)); return; } if (arguments.size() == 2) { final String a0 = arguments.get(0); final String a1 = arguments.get(1); if (!quiet) System.err .println(String.format("Will use file [%s] as input file and [%s] as output file", a0, a1)); fis = new FileInputStream(a0); fos = new FileOutputStream(a1); } else if (arguments.isEmpty()) { if (!quiet) System.err .println(String.format("Will use file [STDIN] as input file and [STDOUT] as output file")); fis = System.in; fos = System.out; } else { if (!quiet) System.err.println("I do not understand the usage."); return; } if (zipAlign) { System.err.println("WARNING: ZIP Align feature not implemented yet..."); return; } // Deflater to re-compress uncompressed data read from ZIP stream. def = new Deflater(9, true); sentFiles = new HashSet<String>(); // Buffer input stream so input stream is read in chunks bis = new BufferedInputStream(fis); bos = new BufferedOutputStream(fos); // Effective temporary dir - if separate is required if (separateTempDir) { effectiveTempDir = File.createTempFile("temp_apk_dir_", "", new File(TEMP_DIR)); effectiveTempDir.delete(); effectiveTempDir.mkdir(); } else { effectiveTempDir = new File(TEMP_DIR); } // Generate temporary APK filename tempApk = File.createTempFile("temp_apk_", ".apk", effectiveTempDir); if (tempApk.canWrite() == false) { throw new IOException("Temp file is not writable!"); } FileOutputStream tos = new FileOutputStream(tempApk); // What we want here is to read input stream from the socket/pipe // whatever, process it in ZIP logic and simultaneously to copy // all read data to the temporary file - this reminds tee command // logic. This functionality can be found in TeeInputStream. TeeInputStream tis = new TeeInputStream(bis, tos); // Providing tis to ZipArchiveInputStream will copy all read data // to temporary tos file. zip = new ZipArchiveInputStream(tis); // List of all sent files, with data and hashes alMap = new HashMap<String, PostponedEntry>(); // Output stream // If there is defined slow down stream, it is used for user output to // mitigate tampering time gap. OutputStream osToUse = bos; SlowDownStream sdStream = null; if (slowDownStream) { // New slow down output stream with internal pipe buffer 15MB. sdStream = new SlowDownStream(osToUse, 15 * 1024 * 1024); // If size of the APK is known, use it to set slow down parameters correctly. if (apkSize > 0) { setSlowDownParams(); } if (!quiet) { System.err.println(String.format("Slown down stream will be used; apkSize=%d buffer=%d timeout=%d", apkSize, slowDownBuffer, slowDownTimeout)); } sdStream.setFlushBufferSize(slowDownBuffer); sdStream.setFlushBufferTimeout(slowDownTimeout); sdStream.start(); osToUse = sdStream; } zop = new ZipArchiveOutputStream(osToUse); zop.setLevel(9); if (!quiet) { System.err.println("Patterns that will be excluded:"); for (String regex : exclude) { System.err.println(" '" + regex + "'"); } System.err.println(); } // Read the archive ZipArchiveEntry ze = zip.getNextZipEntry(); while (ze != null) { ZipExtraField[] extra = ze.getExtraFields(true); byte[] lextra = ze.getLocalFileDataExtra(); UnparseableExtraFieldData uextra = ze.getUnparseableExtraFieldData(); byte[] uextrab = uextra != null ? uextra.getLocalFileDataData() : null; byte[] ex = ze.getExtra(); // ZipArchiveOutputStream.DEFLATED // // Data for entry byte[] byteData = Utils.readAll(zip); byte[] deflData = new byte[0]; int infl = byteData.length; int defl = 0; // If method is deflated, get the raw data (compress again). // Since ZIPStream automatically decompresses deflated files in read(). if (ze.getMethod() == ZipArchiveOutputStream.DEFLATED) { def.reset(); def.setInput(byteData); def.finish(); byte[] deflDataTmp = new byte[byteData.length * 2]; defl = def.deflate(deflDataTmp); deflData = new byte[defl]; System.arraycopy(deflDataTmp, 0, deflData, 0, defl); } if (!quiet) System.err.println(String.format( "ZipEntry: meth=%d " + "size=%010d isDir=%5s " + "compressed=%07d extra=%d lextra=%d uextra=%d ex=%d " + "comment=[%s] " + "dataDesc=%s " + "UTF8=%s " + "infl=%07d defl=%07d " + "name [%s]", ze.getMethod(), ze.getSize(), ze.isDirectory(), ze.getCompressedSize(), extra != null ? extra.length : -1, lextra != null ? lextra.length : -1, uextrab != null ? uextrab.length : -1, ex != null ? ex.length : -1, ze.getComment(), ze.getGeneralPurposeBit().usesDataDescriptor(), ze.getGeneralPurposeBit().usesUTF8ForNames(), infl, defl, ze.getName())); final String curName = ze.getName(); // Store zip entry to the map for later check after the APK is recompiled. // Hashes will be compared with the modified APK files after the process. PostponedEntry al = new PostponedEntry(ze, byteData, deflData); alMap.put(curName, al); // META-INF files should be always on the end of the archive, // thus add postponed files right before them if (isPostponed(ze)) { // Capturing interesting files for us and store for later. // If the file is not interesting, send directly to the stream. if (!quiet) System.err.println(" Interesting file, postpone sending!!!"); } else { // recompute CRC? if (recomputeCrc) { crc.reset(); crc.update(byteData); final long newCrc = crc.getValue(); if (!quiet && ze.getCrc() != newCrc && ze.getCrc() != -1) { System.err.println(" Warning: file CRC mismatch!!! Original: [" + ze.getCrc() + "] real: [" + newCrc + "]"); } ze.setCrc(newCrc); } // Write ZIP entry to the archive zop.putArchiveEntry(ze); // Add file data to the stream zop.write(byteData, 0, infl); zop.closeArchiveEntry(); zop.flush(); // Mark file as sent. addSent(curName); } ze = zip.getNextZipEntry(); } // Flush buffers zop.flush(); fos.flush(); // Cleaning up stuff, all reading streams can be closed now. zip.close(); bis.close(); fis.close(); tis.close(); tos.close(); // // APK is finished here, all non-interesting files were sent to the // zop strem (socket to the victim). Now APK transformation will // be performed, diff, sending rest of the files to zop. // boolean doPadding = paddExtra > 0 || outBytes > 0; long flen = tempApk.length(); if (outBytes <= 0) { outBytes = flen + paddExtra; } if (!quiet) { System.err.println("\nAPK reading finished, going to tamper downloaded " + " APK file [" + tempApk.toString() + "]; filezise=[" + flen + "]"); System.err.println(String.format("Sent so far: %d kB in %f %% after adding padding it is %f %%", zop.getWritten() / 1024, 100.0 * (double) zop.getWritten() / (double) flen, 100.0 * (double) zop.getWritten() / ((double) (outBytes > 0 ? outBytes : flen)))); } // New APK was generated, new filename = "tempApk_tampered" newApk = new File(outFile == null ? getFileName(tempApk.getAbsolutePath()) : outFile); if (cmd == null) { // Simulation of doing some evil stuff on the temporary apk Thread.sleep(3000); if (!quiet) System.err.println( "Tampered APK file: " + " [" + newApk.toString() + "]; filezise=[" + newApk.length() + "]"); // // Since no tampering was performed right now we will simulate it by just simple // copying the APK file // FileUtils.copyFile(tempApk, newApk); } else { try { // Execute command String cmd2exec; switch (cmdFormat) { case 0: cmd2exec = cmd + " " + tempApk.getAbsolutePath(); break; case 1: cmd2exec = cmd.replaceAll(INPUT_APK_PLACEHOLDER, tempApk.getAbsolutePath()); break; case 2: cmd2exec = cmd.replaceAll(INPUT_APK_PLACEHOLDER, tempApk.getAbsolutePath()); cmd2exec = cmd2exec.replaceAll(OUTPUT_APK_PLACEHOLDER, newApk.getAbsolutePath()); break; default: throw new IllegalArgumentException("Unknown command format number"); } if (!quiet) { System.err.println("Command to be executed: " + cmd2exec); System.err.println("\n<CMDOUTPUT>"); } long cmdStartTime = System.currentTimeMillis(); CmdExecutionResult resExec = execute(cmd2exec, OutputOpt.EXECUTE_STD_COMBINE, null, quiet ? null : System.err); long cmdStopTime = System.currentTimeMillis(); if (!quiet) { System.err.println("</CMDOUTPUT>\n"); System.err.println("Command executed. Return value: " + resExec.exitValue + "; tamperingTime=" + (cmdStopTime - cmdStartTime)); } } catch (IOException e) { if (!quiet) e.printStackTrace(System.err); } } // // Now read new APK file with ZipInputStream and push new/modified files to the ZOP. // fis = new FileInputStream(newApk); bis = new BufferedInputStream(fis); zip = new ZipArchiveInputStream(bis); // Merge tampered APK to the final, but in this first time // do it to the external buffer in order to get final apk size. // Backup ZOP state to the clonned instance. zop.flush(); long writtenBeforeDiff = zop.getWritten(); ZipArchiveOutputStream zop_back = zop; zop = zop.cloneThis(); // Set temporary byte array output stream, so original output stream is not // touched in this phase. ByteArrayOutputStream bbos = new ByteArrayOutputStream(); zop.setOut(bbos); mergeTamperedApk(false, false); zop.flush(); // Now output stream almost contains APK file, central directory is not written yet. long writtenAfterDiff = zop.getWritten(); if (!quiet) System.err.println(String.format("Tampered apk size yet; writtenBeforeDiff=%d writtenAfterDiff=%d", writtenBeforeDiff, writtenAfterDiff)); // Write central directory header to temporary buffer to discover its size. zop.writeFinish(); zop.flush(); bbos.flush(); // Read new values long writtenAfterCentralDir = zop.getWritten(); long centralDirLen = zop.getCdLength(); byte[] buffAfterMerge = bbos.toByteArray(); //int endOfCentralDir = (int) (buffAfterMerge.length - (writtenAfterCentralDir-writtenBeforeDiff)); long endOfCentralDir = END_OF_CENTRAL_DIR_SIZE; // Determine number of bytes to add to APK. // padlen is number of bytes missing in APK to meet desired size in bytes. padlen = doPadding ? (outBytes - (writtenAfterCentralDir + endOfCentralDir)) : 0; // Compute number of files needed for padding. int padfiles = (int) Math.ceil((double) padlen / (double) (PAD_BLOCK_MAX)); if (!quiet) System.err.println(String.format("Remaining to pad=%d, padfiles=%d " + "writtenAfterCentralDir=%d " + "centralDir=%d endOfCentralDir=%d centralDirOffset=%d " + "buffSize=%d total=%d desired=%d ", padlen, padfiles, writtenAfterCentralDir, centralDirLen, endOfCentralDir, zop.getCdOffset(), buffAfterMerge.length, writtenAfterCentralDir + endOfCentralDir, outBytes)); if (padlen < 0) { throw new IllegalStateException("Padlen cannot be negative, please increase padding size"); } // Close input streams for tampered APK try { zip.close(); bis.close(); fis.close(); } catch (Exception e) { if (!quiet) e.printStackTrace(System.err); } // Merge again, now with pre-defined padding size. fis = new FileInputStream(newApk); bis = new BufferedInputStream(fis); zip = new ZipArchiveInputStream(bis); // Revert changes - use clonned writer stream. zop = zop_back; long writtenBeforeDiff2 = zop.getWritten(); // Merge tampered APK, now for real, now with computed padding. mergeTamperedApk(true, true); zop.flush(); long writtenAfterMerge2 = zop.getWritten(); // Finish really zop.finish(); zop.flush(); long writtenReally = zop.getWritten(); long centralDirLen2 = zop.getCdLength(); if (!quiet) System.err.println(String.format( "Write stats; " + "writtenBeforeDiff=%d writtenAfterDiff=%d " + "writtenAfterCentralDir=%d centralDir=%d endOfCd=%d centralDirOffset=%d " + "padlen=%d total=%d desired=%d", writtenBeforeDiff2, writtenAfterMerge2, writtenReally, centralDirLen2, endOfCentralDir, zop.getCdOffset(), padlen, writtenReally + endOfCentralDir, outBytes)); // Will definitelly close (and finish if not yet) ZOP stream // and close underlying stream. zop.close(); if (sdStream != null) { if (!quiet) { System.err.println("Waiting for sdStream to finish..."); } // Wait for stream to finish dumping with pre-set speed, if it takes // too long (1 minute) switch slown down stream to dumping mode // without any waiting. long startedDump = System.currentTimeMillis(); while (sdStream.isRunning()) { long curTime = System.currentTimeMillis(); if (startedDump != -1 && (curTime - startedDump) > 1000 * 120) { startedDump = -1; sdStream.flushPipes(); } Thread.sleep(10); } if (!quiet) { System.err.println("SD stream finished, terminating..."); } } // Should always be same if (!quiet && doPadding && writtenBeforeDiff != writtenBeforeDiff2) { System.err.println(String.format("Warning! Size before merge from pass1 and pass2 does not match.")); } // If size is different, something went wrong. if (!quiet && doPadding && ((writtenReally + endOfCentralDir) != outBytes)) { System.err.println(String.format("Warning! Output size differs from desired size.")); } bos.close(); fos.close(); // Delete temporary files if required if (deleteArtefacts) { try { if (newApk.exists()) { newApk.delete(); if (!quiet) System.err.println("Tampered APK removed. " + newApk.getAbsolutePath()); } if (tempApk.exists()) { tempApk.delete(); if (!quiet) System.err.println("Original APK removed. " + tempApk.getAbsolutePath()); } if (separateTempDir && effectiveTempDir.exists()) { FileUtils.deleteDirectory(effectiveTempDir); if (!quiet) System.err.println("Temporary directory removed. " + effectiveTempDir.getAbsolutePath()); } if (!quiet) System.err.println("Temporary files were removed."); } catch (IOException e) { if (!quiet) e.printStackTrace(System.err); } } if (!quiet) System.err.println("THE END!"); }