List of usage examples for java.util.zip CheckedOutputStream getChecksum
public Checksum getChecksum()
From source file:org.commoncrawl.service.listcrawler.CrawlHistoryManager.java
/** * append a ProxyCrawlHistoryItem to the active log * //from ww w. j a v a2 s .co m * @param item * @throws IOException */ void appendItemToLog(ProxyCrawlHistoryItem item) throws IOException { try { // open the log file ... DataOutputStream logStream = new DataOutputStream(new FileOutputStream(getActiveLogFilePath(), true)); try { // reset crc calculator (single thread so no worries on synchronization) _crc16Out.reset(); // reset output stream _outputBuffer.reset(); // create checked stream CheckedOutputStream checkedStream = new CheckedOutputStream(_outputBuffer, _crc16Out); DataOutputStream dataOutputStream = new DataOutputStream(checkedStream); // write out item item.serialize(dataOutputStream, new BinaryProtocol()); dataOutputStream.flush(); // ok now write out sync,crc,length then data logStream.write(getLocalLogSyncBytes()); logStream.writeInt((int) checkedStream.getChecksum().getValue()); logStream.writeShort((short) _outputBuffer.getLength()); logStream.write(_outputBuffer.getData(), 0, _outputBuffer.getLength()); logStream.flush(); logStream.close(); logStream = null; // now we need to update the file header updateLogFileHeader(getActiveLogFilePath(), 1, LOG_ITEM_HEADER_SIZE + _outputBuffer.getLength()); URLFP fingerprint = URLUtils.getURLFPFromURL(item.getOriginalURL(), true); // update local log synchronized (_localLogItems) { if (fingerprint != null) { _localLogItems.put(fingerprint, item); } } ImmutableSet<CrawlList> lists = null; // and now walk lists updating them as necessary synchronized (_crawlLists) { lists = new ImmutableSet.Builder<CrawlList>().addAll(_crawlLists.values()).build(); } for (CrawlList list : lists) { try { list.updateItemState(fingerprint, item); } catch (Exception e) { // ok, IF an error occurs updating the list metadata.. we need to // coninue along. // it is critical for this thread to not die in such a circumstane LOG.fatal("Error Updating List(" + list.getListId() + "):" + CCStringUtils.stringifyException(e)); System.out.println("Exception in List Update(" + list.getListId() + "):" + CCStringUtils.stringifyException(e)); } } } finally { if (logStream != null) { logStream.close(); } } } finally { } }
From source file:org.commoncrawl.service.listcrawler.CacheWriterThread.java
@Override public void run() { boolean shutdown = false; while (!shutdown) { try {//from w w w . ja v a 2 s. co m final CacheWriteRequest request = _writeRequestQueue.take(); switch (request._requestType) { case ExitThreadRequest: { // shutdown condition ... CacheManager.LOG.info("Disk Writer Thread Received Shutdown. Exiting!"); shutdown = true; } break; case WriteRequest: { long timeStart = System.currentTimeMillis(); try { // reset crc calculator (single thread so no worries on synchronization) _crc32Out.reset(); // figure out if we need to compress the item ... if ((request._item.getFlags() & CacheItem.Flags.Flag_IsCompressed) == 0 && request._item.getContent().getCount() != 0) { LOG.info("Incoming Cache Request Content for:" + request._item.getUrl() + " is not compressed. Compressing..."); ByteStream compressedBytesOut = new ByteStream(request._item.getContent().getCount()); ThriftyGZIPOutputStream gzipOutputStream = new ThriftyGZIPOutputStream( compressedBytesOut); gzipOutputStream.write(request._item.getContent().getReadOnlyBytes(), 0, request._item.getContent().getCount()); gzipOutputStream.finish(); LOG.info("Finished Compressing Incoming Content for:" + request._item.getUrl() + " BytesIn:" + request._item.getContent().getCount() + " BytesOut:" + compressedBytesOut.size()); // replace buffer request._item.setContent( new FlexBuffer(compressedBytesOut.getBuffer(), 0, compressedBytesOut.size())); request._item.setFlags((request._item.getFlags() | CacheItem.Flags.Flag_IsCompressed)); } // create streams ... ByteStream bufferOutputStream = new ByteStream(8192); CheckedOutputStream checkedStream = new CheckedOutputStream(bufferOutputStream, _crc32Out); DataOutputStream dataOutputStream = new DataOutputStream(checkedStream); // remember if this item has content ... boolean hasContent = request._item.isFieldDirty(CacheItem.Field_CONTENT); // now mark the content field as clean, so that it will not be serialized in our current serialization attempt ... request._item.setFieldClean(CacheItem.Field_CONTENT); // and go ahead and write out the data to the intermediate buffer while also computing partial checksum request._item.write(dataOutputStream); request._item.setFieldDirty(CacheItem.Field_CONTENT); // ok, now ... write out file header ... CacheItemHeader itemHeader = new CacheItemHeader(_manager.getLocalLogSyncBytes()); itemHeader._status = CacheItemHeader.STATUS_ALIVE; itemHeader._lastAccessTime = System.currentTimeMillis(); itemHeader._fingerprint = request._itemFingerprint; // compute total length ... // first the header bytes in the cacheItem itemHeader._dataLength = bufferOutputStream.size(); // next the content length (encoded - as in size + bytes) ... itemHeader._dataLength += 4 + request._item.getContent().getCount(); // lastly the crc value iteself ... itemHeader._dataLength += 8; // open the log file ... DataOutputBuffer logStream = new DataOutputBuffer(); // ok, go ahead and write the header itemHeader.writeHeader(logStream); // ok now write out the item data minus content... logStream.write(bufferOutputStream.getBuffer(), 0, bufferOutputStream.size()); // now create a checked stream for the content ... CheckedOutputStream checkedStream2 = new CheckedOutputStream(logStream, checkedStream.getChecksum()); dataOutputStream = new DataOutputStream(checkedStream2); // content size dataOutputStream.writeInt(request._item.getContent().getCount()); // now write out the content (via checked stream so that we can calc checksum on content) dataOutputStream.write(request._item.getContent().getReadOnlyBytes(), 0, request._item.getContent().getCount()); // ok ... lastly write out the checksum bytes ... dataOutputStream.writeLong(checkedStream2.getChecksum().getValue()); // and FINALLY, write out the total item bytes (so that we can seek in reverse to read last request log logStream.writeInt(CacheItemHeader.SIZE + itemHeader._dataLength); // ok flush everyting to the memory stream dataOutputStream.flush(); //ok - time to acquire the log semaphore //LOG.info("Acquiring Local Log Semaphore"); _manager.getLocalLogAccessSemaphore().acquireUninterruptibly(); try { // now time to acquire the write semaphore ... _manager.getLocalLogWriteAccessSemaphore().acquireUninterruptibly(); // get the current file position long recordOffset = _manager.getLocalLogFilePos(); try { long ioTimeStart = System.currentTimeMillis(); RandomAccessFile logFile = new RandomAccessFile(_manager.getActiveLogFilePath(), "rw"); try { // seek to our known record offset logFile.seek(recordOffset); // write out the data logFile.write(logStream.getData(), 0, logStream.getLength()); } finally { logFile.close(); } // now we need to update the file header _manager.updateLogFileHeader(_manager.getActiveLogFilePath(), 1, CacheItemHeader.SIZE + itemHeader._dataLength + 4 /*trailing bytes*/); CacheManager.LOG .info("#### Wrote Cache Item in:" + (System.currentTimeMillis() - timeStart) + " iotime:" + (System.currentTimeMillis() - ioTimeStart) + " QueueSize:" + _writeRequestQueue.size()); } finally { // release write semaphore quickly _manager.getLocalLogWriteAccessSemaphore().release(); } // now inform the manager of the completed request ... _manager.writeRequestComplete(request, recordOffset); } finally { //LOG.info("Releasing Local Log Semaphore"); _manager.getLocalLogAccessSemaphore().release(); } } catch (IOException e) { CacheManager.LOG.error("### FUC# BATMAN! - GONNA LOSE THIS REQUEST!!!!:" + CCStringUtils.stringifyException(e)); _manager.writeRequestFailed(request, e); } } break; } } catch (InterruptedException e) { } } }
From source file:com.mozilla.SUTAgentAndroid.service.DoCommand.java
public String Zip(String zipFileName, String srcName) { String fixedZipFileName = fixFileName(zipFileName); String fixedSrcName = fixFileName(srcName); String sRet = ""; try {//from w w w . ja v a2 s.c o m FileOutputStream dest = new FileOutputStream(fixedZipFileName); CheckedOutputStream checksum = new CheckedOutputStream(dest, new Adler32()); ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(checksum)); out.setMethod(ZipOutputStream.DEFLATED); sRet += AddFilesToZip(out, fixedSrcName, ""); out.close(); System.out.println("checksum: " + checksum.getChecksum().getValue()); sRet += "checksum: " + checksum.getChecksum().getValue(); } catch (Exception e) { e.printStackTrace(); } return (sRet); }