List of usage examples for java.util.zip CRC32 CRC32
public CRC32()
From source file:org.lmdbjava.bench.Common.java
public void setup(final BenchmarkParams b) throws IOException { keySize = intKey ? BYTES : STRING_KEY_LENGTH; crc = new CRC32(); final IntHashSet set = new IntHashSet(num, MIN_VALUE); keys = new int[num]; for (int i = 0; i < num; i++) { if (sequential) { keys[i] = i;/*from ww w . ja va 2 s .co m*/ } else { while (true) { int candidateKey = RND.nextInt(); if (candidateKey < 0) { candidateKey *= -1; } if (!set.contains(candidateKey)) { set.add(candidateKey); keys[i] = candidateKey; break; } } } } rmdir(TMP_BENCH); tmp = create(b, ""); compact = create(b, "-compacted"); }
From source file:eionet.gdem.utils.ZipUtil.java
/** * * @param f//from w ww . ja v a 2 s.c o m * - File that will be zipped * @param outZip * - ZipOutputStream represents the zip file, where the files will be placed * @param sourceDir * - root directory, where the zipping started * @param doCompress * - don't comress the file, if doCompress=true * @throws IOException If an error occurs. */ public static void zipFile(File f, ZipOutputStream outZip, String sourceDir, boolean doCompress) throws IOException { // Read the source file into byte array byte[] fileBytes = Utils.getBytesFromFile(f); // create a new zip entry String strAbsPath = f.getPath(); String strZipEntryName = strAbsPath.substring(sourceDir.length() + 1, strAbsPath.length()); strZipEntryName = Utils.Replace(strZipEntryName, File.separator, "/"); ZipEntry anEntry = new ZipEntry(strZipEntryName); // Don't compress the file, if not needed if (!doCompress) { // ZipEntry can't calculate crc size automatically, if we use STORED method. anEntry.setMethod(ZipEntry.STORED); anEntry.setSize(fileBytes.length); CRC32 crc321 = new CRC32(); crc321.update(fileBytes); anEntry.setCrc(crc321.getValue()); } // place the zip entry in the ZipOutputStream object outZip.putNextEntry(anEntry); // now write the content of the file to the ZipOutputStream outZip.write(fileBytes); outZip.flush(); // Close the current entry outZip.closeEntry(); }
From source file:org.apache.lucene.store.Directory.java
public synchronized String getCacheKey(String[] filelist) { CacheKeyBuffer kkk = new CacheKeyBuffer(filelist, this.dir_uuid, System.currentTimeMillis() / 600000l, this.getP()); String rtn = CACHE_BUFFER.get(kkk); if (rtn == null) { StringBuffer buff = new StringBuffer(); buff.append(this.getClass().getName()).append("_"); buff.append(this.dir_uuid).append("_"); CRC32 crc32 = new CRC32(); crc32.update(0);//w w w . ja v a 2 s .c o m long filesize = 0; if (filelist != null) { buff.append(filelist.length).append("_"); for (String s : filelist) { crc32.update(new String(s).getBytes()); try { filesize += this.fileLength(s); } catch (Throwable e) { logger.error("filelength", e); } } } long crcvalue = crc32.getValue(); buff.append(crcvalue).append("_"); buff.append(filesize).append("_"); rtn = buff.toString(); CACHE_BUFFER.put(kkk, rtn); } return rtn; }
From source file:org.apache.mnemonic.DurablePersonNGTest.java
@Test(expectedExceptions = { OutOfHybridMemory.class })
public void testGenPeople() throws OutOfHybridMemory, RetrieveDurableEntityError {
Random rand = Utils.createRandom();
NonVolatileMemAllocator act = new NonVolatileMemAllocator(
Utils.getNonVolatileMemoryAllocatorService("pmalloc"), 1024L * 1024 * 1024, "./pobj_person.dat",
true);/*from w ww . jav a 2s . co m*/
cKEYCAPACITY = act.handlerCapacity();
act.setBufferReclaimer(new Reclaim<ByteBuffer>() {
@Override
public boolean reclaim(ByteBuffer mres, Long sz) {
System.out.println(String.format("Reclaim Memory Buffer: %X Size: %s",
System.identityHashCode(mres), null == sz ? "NULL" : sz.toString()));
return false;
}
});
act.setChunkReclaimer(new Reclaim<Long>() {
@Override
public boolean reclaim(Long mres, Long sz) {
System.out.println(String.format("Reclaim Memory Chunk: %X Size: %s",
System.identityHashCode(mres), null == sz ? "NULL" : sz.toString()));
return false;
}
});
for (long i = 0; i < cKEYCAPACITY; ++i) {
act.setHandler(i, 0L);
}
Person<Integer> mother;
Person<Integer> person;
Checksum pic_cs = new CRC32();
pic_cs.reset();
Checksum fp_cs = new CRC32();
fp_cs.reset();
long keyidx = 0;
long val;
try {
while (true) {
// if (keyidx >= KEYCAPACITY) break;
keyidx %= cKEYCAPACITY;
System.out.printf("************ Generating People on Key %d ***********\n", keyidx);
val = act.getHandler(keyidx);
if (0L != val) {
PersonFactory.restore(act, val, true);
}
person = PersonFactory.create(act);
person.setAge((short) rand.nextInt(50));
person.setName(String.format("Name: [%s]", UUID.randomUUID().toString()), true);
person.setName(String.format("Name: [%s]", UUID.randomUUID().toString()), true);
person.setName(String.format("Name: [%s]", UUID.randomUUID().toString()), true);
person.setName(String.format("Name: [%s]", UUID.randomUUID().toString()), true);
person.setPicture(genuptBuffer(act, pic_cs, genRandSize()), true);
person.setPreference(genuptChunk(act, fp_cs, genRandSize()), true);
act.setHandler(keyidx, person.getHandler());
pic_checksum = pic_cs.getValue();
fp_checksum = fp_cs.getValue();
for (int deep = 0; deep < rand.nextInt(100); ++deep) {
mother = PersonFactory.create(act);
mother.setAge((short) (50 + rand.nextInt(50)));
mother.setName(String.format("Name: [%s]", UUID.randomUUID().toString()), true);
mother.setPicture(genuptBuffer(act, pic_cs, genRandSize()), true);
mother.setPreference(genuptChunk(act, fp_cs, genRandSize()), true);
person.setMother(mother, true);
pic_checksum = pic_cs.getValue();
fp_checksum = fp_cs.getValue();
person = mother;
}
++keyidx;
}
} finally {
act.close();
}
}
From source file:org.game.cs.core.condenser.steam.packets.SteamPacketFactory.java
/** * Reassembles the data of a split and/or compressed packet into a single * packet object//from w w w . j a v a 2 s. c om * * @param splitPackets An array of packet data * @param isCompressed whether the data of this packet is compressed * @param uncompressedSize The size of the decompressed packet data * @param packetChecksum The CRC32 checksum of the decompressed * packet data * @throws SteamCondenserException if decompressing the packet data fails * @throws PacketFormatException if the calculated CRC32 checksum does not * match the expected value * @return SteamPacket The reassembled packet * @see SteamPacketFactory#getPacketFromData */ public static SteamPacket reassemblePacket(ArrayList<byte[]> splitPackets, boolean isCompressed, int uncompressedSize, int packetChecksum) throws SteamCondenserException { byte[] packetData, tmpData; packetData = new byte[0]; for (byte[] splitPacket : splitPackets) { tmpData = packetData; packetData = new byte[tmpData.length + splitPacket.length]; System.arraycopy(tmpData, 0, packetData, 0, tmpData.length); System.arraycopy(splitPacket, 0, packetData, tmpData.length, splitPacket.length); } if (isCompressed) { try { ByteArrayInputStream stream = new ByteArrayInputStream(packetData); stream.read(); stream.read(); BZip2CompressorInputStream bzip2 = new BZip2CompressorInputStream(stream); byte[] uncompressedPacketData = new byte[uncompressedSize]; bzip2.read(uncompressedPacketData, 0, uncompressedSize); CRC32 crc32 = new CRC32(); crc32.update(uncompressedPacketData); int crc32checksum = (int) crc32.getValue(); if (crc32checksum != packetChecksum) { throw new PacketFormatException("CRC32 checksum mismatch of uncompressed packet data."); } packetData = uncompressedPacketData; } catch (IOException e) { throw new SteamCondenserException(e.getMessage(), e); } } packetData = new String(packetData).substring(4).getBytes(); return SteamPacketFactory.getPacketFromData(packetData); }
From source file:org.apache.pluto.util.assemble.io.JarStreamingAssembly.java
/** * Reads the source JarInputStream, copying entries to the destination JarOutputStream. * The web.xml and portlet.xml are cached, and after the entire archive is copied * (minus the web.xml) a re-written web.xml is generated and written to the * destination JAR.// ww w . jav a 2 s .co m * * @param source the WAR source input stream * @param dest the WAR destination output stream * @param dispatchServletClass the name of the dispatch class * @throws IOException */ public static void assembleStream(JarInputStream source, JarOutputStream dest, String dispatchServletClass) throws IOException { try { //Need to buffer the web.xml and portlet.xml files for the rewritting JarEntry servletXmlEntry = null; byte[] servletXmlBuffer = null; byte[] portletXmlBuffer = null; JarEntry originalJarEntry; //Read the source archive entry by entry while ((originalJarEntry = source.getNextJarEntry()) != null) { final JarEntry newJarEntry = smartClone(originalJarEntry); originalJarEntry = null; //Capture the web.xml JarEntry and contents as a byte[], don't write it out now or //update the CRC or length of the destEntry. if (Assembler.SERVLET_XML.equals(newJarEntry.getName())) { servletXmlEntry = newJarEntry; servletXmlBuffer = IOUtils.toByteArray(source); } //Capture the portlet.xml contents as a byte[] else if (Assembler.PORTLET_XML.equals(newJarEntry.getName())) { portletXmlBuffer = IOUtils.toByteArray(source); dest.putNextEntry(newJarEntry); IOUtils.write(portletXmlBuffer, dest); } //Copy all other entries directly to the output archive else { dest.putNextEntry(newJarEntry); IOUtils.copy(source, dest); } dest.closeEntry(); dest.flush(); } // If no portlet.xml was found in the archive, skip the assembly step. if (portletXmlBuffer != null) { // container for assembled web.xml bytes final byte[] webXmlBytes; // Checks to make sure the web.xml was found in the archive if (servletXmlBuffer == null) { throw new FileNotFoundException( "File '" + Assembler.SERVLET_XML + "' could not be found in the source input stream."); } //Create streams of the byte[] data for the updater method final InputStream webXmlIn = new ByteArrayInputStream(servletXmlBuffer); final InputStream portletXmlIn = new ByteArrayInputStream(portletXmlBuffer); final ByteArrayOutputStream webXmlOut = new ByteArrayOutputStream(servletXmlBuffer.length); //Update the web.xml WebXmlStreamingAssembly.assembleStream(webXmlIn, portletXmlIn, webXmlOut, dispatchServletClass); IOUtils.copy(webXmlIn, webXmlOut); webXmlBytes = webXmlOut.toByteArray(); //If no compression is being used (STORED) we have to manually update the size and crc if (servletXmlEntry.getMethod() == ZipEntry.STORED) { servletXmlEntry.setSize(webXmlBytes.length); final CRC32 webXmlCrc = new CRC32(); webXmlCrc.update(webXmlBytes); servletXmlEntry.setCrc(webXmlCrc.getValue()); } //write out the assembled web.xml entry and contents dest.putNextEntry(servletXmlEntry); IOUtils.write(webXmlBytes, dest); if (LOG.isDebugEnabled()) { LOG.debug("Jar stream " + source + " successfully assembled."); } } else { if (LOG.isDebugEnabled()) { LOG.debug("No portlet XML file was found, assembly was not required."); } //copy the original, unmodified web.xml entry to the destination dest.putNextEntry(servletXmlEntry); IOUtils.write(servletXmlBuffer, dest); if (LOG.isDebugEnabled()) { LOG.debug("Jar stream " + source + " successfully assembled."); } } } finally { dest.flush(); dest.close(); } }
From source file:org.broad.igv.util.Utilities.java
/** * @param buffer//from w w w . j a v a2s .c o m * @return * @throws java.io.IOException */ static private long getCrc(byte[] buffer) throws IOException { CRC32 crc = new CRC32(); crc.reset(); crc.update(buffer, 0, buffer.length); return crc.getValue(); }
From source file:com.github.koraktor.steamcondenser.servers.packets.SteamPacketFactory.java
/** * Reassembles the data of a split and/or compressed packet into a single * packet object//from w ww . j a va2 s . c o m * * @param splitPackets An array of packet data * @param isCompressed whether the data of this packet is compressed * @param uncompressedSize The size of the decompressed packet data * @param packetChecksum The CRC32 checksum of the decompressed * packet data * @throws SteamCondenserException if decompressing the packet data fails * @throws PacketFormatException if the calculated CRC32 checksum does not * match the expected value * @return SteamPacket The reassembled packet * @see SteamPacketFactory#getPacketFromData */ public static SteamPacket reassemblePacket(ArrayList<byte[]> splitPackets, boolean isCompressed, int uncompressedSize, int packetChecksum) throws SteamCondenserException { byte[] packetData, tmpData; packetData = new byte[0]; for (byte[] splitPacket : splitPackets) { tmpData = packetData; packetData = new byte[tmpData.length + splitPacket.length]; System.arraycopy(tmpData, 0, packetData, 0, tmpData.length); System.arraycopy(splitPacket, 0, packetData, tmpData.length, splitPacket.length); } if (isCompressed) { try { ByteArrayInputStream stream = new ByteArrayInputStream(packetData); stream.read(); stream.read(); BZip2CompressorInputStream bzip2 = new BZip2CompressorInputStream(stream); byte[] uncompressedPacketData = new byte[uncompressedSize]; bzip2.read(uncompressedPacketData, 0, uncompressedSize); CRC32 crc32 = new CRC32(); crc32.update(uncompressedPacketData); int crc32checksum = (int) crc32.getValue(); if (crc32checksum != packetChecksum) { throw new PacketFormatException("CRC32 checksum mismatch of uncompressed packet data."); } packetData = uncompressedPacketData; } catch (IOException e) { throw new SteamCondenserException(e.getMessage(), e); } } tmpData = packetData; packetData = new byte[tmpData.length - 4]; System.arraycopy(tmpData, 4, packetData, 0, tmpData.length - 4); return SteamPacketFactory.getPacketFromData(packetData); }
From source file:nl.nn.adapterframework.pipes.ChecksumPipe.java
protected ChecksumGenerator createChecksumGenerator() throws NoSuchAlgorithmException { if (CHECKSUM_MD5.equals(getType())) { return new MessageDigestChecksumGenerator(getType()); } else if (CHECKSUM_CRC32.equals(getType())) { return new ZipChecksumGenerator(new CRC32()); } else if (CHECKSUM_ADLER32.equals(getType())) { return new ZipChecksumGenerator(new Adler32()); } else if (CHECKSUM_SHA.equals(getType())) { return new MessageDigestChecksumGenerator(getType()); } else {//from ww w . j a va 2 s . c o m throw new NoSuchAlgorithmException("unsupported algorithm [" + getType() + "]"); } }
From source file:com.hadoop.compression.lzo.LzopInputStream.java
/** * Read and verify an lzo header, setting relevant block checksum options * and ignoring most everything else./*from w ww . ja v a 2 s . com*/ * @param in InputStream * @throws IOException if there is a error in lzo header */ protected void readHeader(InputStream in) throws IOException { readFully(in, buf, 0, 9); if (!Arrays.equals(buf, LzopCodec.LZO_MAGIC)) { throw new IOException("Invalid LZO header"); } Arrays.fill(buf, (byte) 0); Adler32 adler = new Adler32(); CRC32 crc32 = new CRC32(); int hitem = readHeaderItem(in, buf, 2, adler, crc32); // lzop version if (hitem > LzopCodec.LZOP_VERSION) { LOG.debug("Compressed with later version of lzop: " + Integer.toHexString(hitem) + " (expected 0x" + Integer.toHexString(LzopCodec.LZOP_VERSION) + ")"); } hitem = readHeaderItem(in, buf, 2, adler, crc32); // lzo library version if (hitem < LzoDecompressor.MINIMUM_LZO_VERSION) { throw new IOException("Compressed with incompatible lzo version: 0x" + Integer.toHexString(hitem) + " (expected at least 0x" + Integer.toHexString(LzoDecompressor.MINIMUM_LZO_VERSION) + ")"); } hitem = readHeaderItem(in, buf, 2, adler, crc32); // lzop extract version if (hitem > LzopCodec.LZOP_VERSION) { throw new IOException("Compressed with incompatible lzop version: 0x" + Integer.toHexString(hitem) + " (expected 0x" + Integer.toHexString(LzopCodec.LZOP_VERSION) + ")"); } hitem = readHeaderItem(in, buf, 1, adler, crc32); // method if (hitem < 1 || hitem > 3) { throw new IOException("Invalid strategy: " + Integer.toHexString(hitem)); } readHeaderItem(in, buf, 1, adler, crc32); // ignore level // flags hitem = readHeaderItem(in, buf, 4, adler, crc32); try { for (DChecksum f : dflags) { if (0 == (f.getHeaderMask() & hitem)) { dflags.remove(f); } else { dcheck.put(f, (int) f.getChecksumClass().newInstance().getValue()); } } for (CChecksum f : cflags) { if (0 == (f.getHeaderMask() & hitem)) { cflags.remove(f); } else { ccheck.put(f, (int) f.getChecksumClass().newInstance().getValue()); } } } catch (InstantiationException e) { throw new RuntimeException("Internal error", e); } catch (IllegalAccessException e) { throw new RuntimeException("Internal error", e); } ((LzopDecompressor) decompressor).initHeaderFlags(dflags, cflags); boolean useCRC32 = 0 != (hitem & 0x00001000); // F_H_CRC32 boolean extraField = 0 != (hitem & 0x00000040); // F_H_EXTRA_FIELD if (0 != (hitem & 0x400)) { // F_MULTIPART throw new IOException("Multipart lzop not supported"); } if (0 != (hitem & 0x800)) { // F_H_FILTER throw new IOException("lzop filter not supported"); } if (0 != (hitem & 0x000FC000)) { // F_RESERVED throw new IOException("Unknown flags in header"); } // known !F_H_FILTER, so no optional block readHeaderItem(in, buf, 4, adler, crc32); // ignore mode readHeaderItem(in, buf, 4, adler, crc32); // ignore mtime readHeaderItem(in, buf, 4, adler, crc32); // ignore gmtdiff hitem = readHeaderItem(in, buf, 1, adler, crc32); // fn len if (hitem > 0) { // skip filename int filenameLen = Math.max(4, hitem); // buffer must be at least 4 bytes for readHeaderItem to work. readHeaderItem(in, new byte[filenameLen], hitem, adler, crc32); } int checksum = (int) (useCRC32 ? crc32.getValue() : adler.getValue()); hitem = readHeaderItem(in, buf, 4, adler, crc32); // read checksum if (hitem != checksum) { throw new IOException("Invalid header checksum: " + Long.toHexString(checksum) + " (expected 0x" + Integer.toHexString(hitem) + ")"); } if (extraField) { // lzop 1.08 ultimately ignores this LOG.debug("Extra header field not processed"); adler.reset(); crc32.reset(); hitem = readHeaderItem(in, buf, 4, adler, crc32); readHeaderItem(in, new byte[hitem], hitem, adler, crc32); checksum = (int) (useCRC32 ? crc32.getValue() : adler.getValue()); if (checksum != readHeaderItem(in, buf, 4, adler, crc32)) { throw new IOException("Invalid checksum for extra header field"); } } }