Java tutorial
/* ************************************************************************ # # DivConq # # http://divconq.com/ # # Copyright: # Copyright 2014 eTimeline, LLC. All rights reserved. # # License: # See the license.txt file in the project's top-level directory for details. # # Authors: # * Andy White # ************************************************************************ */ package divconq.ctp.stream; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.util.ArrayList; import java.util.List; import java.util.zip.CRC32; import java.util.zip.DataFormatException; import java.util.zip.Deflater; import java.util.zip.Inflater; import org.apache.commons.compress.compressors.gzip.GzipUtils; import divconq.ctp.f.FileDescriptor; import divconq.hub.Hub; import divconq.lang.op.OperationContext; import divconq.script.StackEntry; import divconq.util.FileUtil; import divconq.util.StringUtil; import divconq.xml.XElement; public class UngzipStream extends BaseStream implements IStreamSource { protected static final int FHCRC = 0x02; protected static final int FEXTRA = 0x04; protected static final int FNAME = 0x08; protected static final int FCOMMENT = 0x10; protected static final int FRESERVED = 0xE0; private enum GzipState { HEADER_START, FLG_READ, XLEN_READ, SKIP_FNAME, SKIP_COMMENT, PROCESS_FHCRC, PRROCESS_CONTENT, PROCESS_FOOTER, DONE } protected GzipState gzipState = GzipState.HEADER_START; protected int flags = -1; protected int xlen = -1; protected Inflater inflater = null; protected byte[] dictionary = null; protected CRC32 crc = new CRC32(); protected List<ByteBuf> outlist = new ArrayList<>(); protected String nameHint = null; //protected String ourpath = null; //protected long ourmod = 0; //protected byte ourperm = 0; protected FileDescriptor ufile = null; protected boolean eofsent = false; protected ByteBuf remnant = null; public UngzipStream() { this.inflater = new Inflater(true); this.crc = new CRC32(); } public UngzipStream(byte[] dictionary) { this(); this.dictionary = dictionary; } @Override public void init(StackEntry stack, XElement el) { this.nameHint = stack.stringFromElement(el, "NameHint"); } @Override public void close() { Inflater inf = this.inflater; if (inf != null) inf.end(); this.inflater = null; ByteBuf rem = this.remnant; if (rem != null) { rem.release(); this.remnant = null; } // not truly thread safe, consider for (ByteBuf bb : this.outlist) bb.release(); this.outlist.clear(); super.close(); } // make sure we don't return without first releasing the file reference content @Override public ReturnOption handle(FileDescriptor file, ByteBuf data) { if (file == FileDescriptor.FINAL) return this.downstream.handle(file, data); if (this.ufile == null) this.initializeFileValues(file); // inflate the payload into 1 or more outgoing buffers set in a queue ByteBuf in = data; if (in != null) { ByteBuf rem = this.remnant; ByteBuf src = ((rem != null) && rem.isReadable()) ? Unpooled.copiedBuffer(rem, in) : in; this.inflate(src); // if there are any unread bytes here we need to store them and combine with the next "handle" // this would be rare since the header and footer are small, but it is possible and should be handled // file content has its own "in progress" buffer so no need to worry about that this.remnant = src.isReadable() ? src.copy() : null; // TODO wrap or slice here? we need copy above if (in != null) in.release(); if (rem != null) rem.release(); if (OperationContext.get().getTaskRun().isKilled()) return ReturnOption.DONE; } // write all buffers in the queue while (this.outlist.size() > 0) { ReturnOption ret = this.nextMessage(); if (ret != ReturnOption.CONTINUE) return ret; } // if we reached done and we wrote all the buffers, then send the EOF marker if not already if ((this.gzipState == GzipState.DONE) && !this.eofsent) return this.nextMessage(); // otherwise we need more data return ReturnOption.CONTINUE; } // TODO return null if not EOF and if no bytes to send - there is no point in sending messages // downstream without content or EOF public ReturnOption nextMessage() { ByteBuf out = null; if (this.outlist.size() > 0) out = this.outlist.remove(0); boolean eof = (this.outlist.size() == 0) && (this.gzipState == GzipState.DONE); this.ufile.setEof(eof); // set it on but never off if (eof) this.eofsent = true; return this.downstream.handle(this.ufile, out); } public void initializeFileValues(FileDescriptor src) { this.ufile = new FileDescriptor(); if (StringUtil.isNotEmpty(this.nameHint)) this.ufile.setPath("/" + this.nameHint); else if (src.hasPath()) this.ufile.setPath("/" + GzipUtils.getUncompressedFilename(src.path().getFileName())); else this.ufile.setPath("/" + FileUtil.randomFilename("bin")); this.ufile.setModTime(src.getModTime()); this.ufile.setPermissions(src.getPermissions()); } // return true when completely done protected void inflate(ByteBuf in) { switch (this.gzipState) { case HEADER_START: if (in.readableBytes() < 10) return; // read magic numbers int magic0 = in.readByte(); int magic1 = in.readByte(); if (magic0 != 31) { OperationContext.get().getTaskRun().kill("Input is not in the GZIP format"); return; } this.crc.update(magic0); this.crc.update(magic1); int method = in.readUnsignedByte(); if (method != Deflater.DEFLATED) { OperationContext.get().getTaskRun() .kill("Unsupported compression method " + method + " in the GZIP header"); return; } this.crc.update(method); this.flags = in.readUnsignedByte(); this.crc.update(this.flags); if ((this.flags & FRESERVED) != 0) { OperationContext.get().getTaskRun().kill("Reserved flags are set in the GZIP header"); return; } // mtime (int) this.crc.update(in.readByte()); this.crc.update(in.readByte()); this.crc.update(in.readByte()); this.crc.update(in.readByte()); this.crc.update(in.readUnsignedByte()); // extra flags this.crc.update(in.readUnsignedByte()); // operating system this.gzipState = GzipState.FLG_READ; case FLG_READ: if ((this.flags & FEXTRA) != 0) { if (in.readableBytes() < 2) return; int xlen1 = in.readUnsignedByte(); int xlen2 = in.readUnsignedByte(); this.crc.update(xlen1); this.crc.update(xlen2); this.xlen |= xlen1 << 8 | xlen2; } this.gzipState = GzipState.XLEN_READ; case XLEN_READ: if (this.xlen != -1) { if (in.readableBytes() < xlen) return; byte[] xtra = new byte[xlen]; in.readBytes(xtra); this.crc.update(xtra); } this.gzipState = GzipState.SKIP_FNAME; case SKIP_FNAME: if ((this.flags & FNAME) != 0) { boolean gotend = false; while (in.isReadable()) { int b = in.readUnsignedByte(); this.crc.update(b); if (b == 0x00) { gotend = true; break; } } if (!gotend) return; } this.gzipState = GzipState.SKIP_COMMENT; case SKIP_COMMENT: if ((this.flags & FCOMMENT) != 0) { boolean gotend = false; while (in.isReadable()) { int b = in.readUnsignedByte(); this.crc.update(b); if (b == 0x00) { gotend = true; break; } } if (!gotend) return; } this.gzipState = GzipState.PROCESS_FHCRC; case PROCESS_FHCRC: if ((this.flags & FHCRC) != 0) { if (in.readableBytes() < 4) return; long crcValue = 0; for (int i = 0; i < 4; ++i) crcValue |= (long) in.readUnsignedByte() << i * 8; long readCrc = crc.getValue(); if (crcValue != readCrc) { OperationContext.get().getTaskRun() .kill("CRC value missmatch. Expected: " + crcValue + ", Got: " + readCrc); return; } } this.crc.reset(); this.gzipState = GzipState.PRROCESS_CONTENT; case PRROCESS_CONTENT: int readableBytes = in.readableBytes(); if (readableBytes < 1) return; if (in.hasArray()) { this.inflater.setInput(in.array(), in.arrayOffset() + in.readerIndex(), readableBytes); } else { byte[] array = new byte[readableBytes]; in.getBytes(in.readerIndex(), array); this.inflater.setInput(array); } int maxOutputLength = this.inflater.getRemaining() << 1; ByteBuf decompressed = Hub.instance.getBufferAllocator().heapBuffer(maxOutputLength); boolean readFooter = false; byte[] outArray = decompressed.array(); try { while (!this.inflater.needsInput()) { int writerIndex = decompressed.writerIndex(); int outIndex = decompressed.arrayOffset() + writerIndex; int length = decompressed.writableBytes(); if (length == 0) { // completely filled the buffer allocate a new one and start to fill it this.outlist.add(decompressed); decompressed = Hub.instance.getBufferAllocator().heapBuffer(maxOutputLength); outArray = decompressed.array(); continue; } int outputLength = this.inflater.inflate(outArray, outIndex, length); if (outputLength > 0) { decompressed.writerIndex(writerIndex + outputLength); this.crc.update(outArray, outIndex, outputLength); } else { if (this.inflater.needsDictionary()) { if (this.dictionary == null) { OperationContext.get().getTaskRun().kill( "decompression failure, unable to set dictionary as non was specified"); return; } this.inflater.setDictionary(this.dictionary); } } if (this.inflater.finished()) { readFooter = true; break; } } in.skipBytes(readableBytes - this.inflater.getRemaining()); } catch (DataFormatException x) { OperationContext.get().getTaskRun().kill("decompression failure: " + x); return; } finally { if (decompressed.isReadable()) { this.outlist.add(decompressed); } else { decompressed.release(); } } if (!readFooter) break; this.gzipState = GzipState.PROCESS_FOOTER; case PROCESS_FOOTER: if (in.readableBytes() < 8) return; long crcValue = 0; for (int i = 0; i < 4; ++i) crcValue |= (long) in.readUnsignedByte() << i * 8; long readCrc = this.crc.getValue(); if (crcValue != readCrc) { OperationContext.get().getTaskRun() .kill("CRC value missmatch. Expected: " + crcValue + ", Got: " + readCrc); return; } // read ISIZE and verify int dataLength = 0; for (int i = 0; i < 4; ++i) dataLength |= in.readUnsignedByte() << i * 8; int readLength = this.inflater.getTotalOut(); if (dataLength != readLength) { OperationContext.get().getTaskRun() .kill("Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength); return; } this.gzipState = GzipState.DONE; case DONE: break; } } // TODO if there is more from us then handle that before going upstream @Override public void read() { // write all buffers in the queue while (this.outlist.size() > 0) { if (this.nextMessage() != ReturnOption.CONTINUE) return; } // if we reached done and we wrote all the buffers, then send the EOF marker if not already if ((this.gzipState == GzipState.DONE) && !this.eofsent) { if (this.nextMessage() != ReturnOption.CONTINUE) return; } this.upstream.read(); } }