List of usage examples for java.nio ByteBuffer getInt
public abstract int getInt();
From source file:srebrinb.compress.sevenzip.SevenZFile.java
private Archive readHeaders(final byte[] password) throws IOException { ByteBuffer buf = ByteBuffer.allocate(12 /* signature + 2 bytes version + 4 bytes CRC */) .order(ByteOrder.LITTLE_ENDIAN); readFully(buf);/*from w ww . j a v a 2 s .c o m*/ final byte[] signature = new byte[6]; buf.get(signature); if (!Arrays.equals(signature, sevenZSignature)) { throw new IOException("Bad 7z signature"); } // 7zFormat.txt has it wrong - it's first major then minor final byte archiveVersionMajor = buf.get(); final byte archiveVersionMinor = buf.get(); if (archiveVersionMajor != 0) { throw new IOException( String.format("Unsupported 7z version (%d,%d)", archiveVersionMajor, archiveVersionMinor)); } final long startHeaderCrc = 0xffffFFFFL & buf.getInt(); final StartHeader startHeader = readStartHeader(startHeaderCrc); final int nextHeaderSizeInt = (int) startHeader.nextHeaderSize; if (nextHeaderSizeInt != startHeader.nextHeaderSize) { throw new IOException("cannot handle nextHeaderSize " + startHeader.nextHeaderSize); } channel.position(SIGNATURE_HEADER_SIZE + startHeader.nextHeaderOffset); buf = ByteBuffer.allocate(nextHeaderSizeInt).order(ByteOrder.LITTLE_ENDIAN); readFully(buf); final CRC32 crc = new CRC32(); crc.update(buf.array()); if (startHeader.nextHeaderCrc != crc.getValue()) { throw new IOException("NextHeader CRC mismatch"); } Archive archive = new Archive(); int nid = getUnsignedByte(buf); if (nid == NID.kEncodedHeader) { buf = readEncodedHeader(buf, archive, password); // Archive gets rebuilt with the new header archive = new Archive(); nid = getUnsignedByte(buf); } if (nid == NID.kHeader) { readHeader(buf, archive); } else { throw new IOException("Broken or unsupported archive: no Header"); } return archive; }
From source file:org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.java
/** * Execute the specified listener, committing or rolling back the transaction afterwards (if necessary). * * @param channel the Rabbit Channel to operate on * @param messageIn the received Rabbit Message * @throws Exception Any Exception./*from w ww. jav a2s.c om*/ * * @see #invokeListener * @see #handleListenerException */ protected void executeListener(Channel channel, Message messageIn) throws Exception { if (!isRunning()) { if (logger.isWarnEnabled()) { logger.warn( "Rejecting received message because the listener container has been stopped: " + messageIn); } throw new MessageRejectedWhileStoppingException(); } try { Message message = messageIn; if (this.afterReceivePostProcessors != null) { for (MessagePostProcessor processor : this.afterReceivePostProcessors) { message = processor.postProcessMessage(message); } } Object batchFormat = message.getMessageProperties().getHeaders() .get(MessageProperties.SPRING_BATCH_FORMAT); if (MessageProperties.BATCH_FORMAT_LENGTH_HEADER4.equals(batchFormat) && this.deBatchingEnabled) { ByteBuffer byteBuffer = ByteBuffer.wrap(message.getBody()); MessageProperties messageProperties = message.getMessageProperties(); messageProperties.getHeaders().remove(MessageProperties.SPRING_BATCH_FORMAT); while (byteBuffer.hasRemaining()) { int length = byteBuffer.getInt(); if (length < 0 || length > byteBuffer.remaining()) { throw new ListenerExecutionFailedException("Bad batched message received", new MessageConversionException( "Insufficient batch data at offset " + byteBuffer.position()), message); } byte[] body = new byte[length]; byteBuffer.get(body); messageProperties.setContentLength(length); // Caveat - shared MessageProperties. Message fragment = new Message(body, messageProperties); invokeListener(channel, fragment); } } else { invokeListener(channel, message); } } catch (Exception ex) { if (messageIn.getMessageProperties().isFinalRetryForMessageWithNoId()) { if (this.statefulRetryFatalWithNullMessageId) { throw new FatalListenerExecutionException( "Illegal null id in message. Failed to manage retry for message: " + messageIn); } else { throw new ListenerExecutionFailedException("Cannot retry message more than once without an ID", new AmqpRejectAndDontRequeueException("Not retryable; rejecting and not requeuing", ex), messageIn); } } handleListenerException(ex); throw ex; } }
From source file:hivemall.topicmodel.ProbabilisticTopicModelBaseUDTF.java
protected final void runIterativeTraining(@Nonnegative final int iterations) throws HiveException { final ByteBuffer buf = this.inputBuf; final NioStatefulSegment dst = this.fileIO; assert (buf != null); assert (dst != null); final long numTrainingExamples = model.getDocCount(); long numTrain = numTrainingExamples / miniBatchSize; if (numTrainingExamples % miniBatchSize != 0L) { numTrain++;//w w w.j a va 2s. c om } final Reporter reporter = getReporter(); final Counters.Counter iterCounter = (reporter == null) ? null : reporter.getCounter("hivemall.topicmodel.ProbabilisticTopicModel$Counter", "iteration"); try { if (dst.getPosition() == 0L) {// run iterations w/o temporary file if (buf.position() == 0) { return; // no training example } buf.flip(); int iter = 2; float perplexity = cumPerplexity / numTrain; float perplexityPrev; for (; iter <= iterations; iter++) { perplexityPrev = perplexity; cumPerplexity = 0.f; reportProgress(reporter); setCounterValue(iterCounter, iter); while (buf.remaining() > 0) { int recordBytes = buf.getInt(); assert (recordBytes > 0) : recordBytes; int wcLength = buf.getInt(); final String[] wordCounts = new String[wcLength]; for (int j = 0; j < wcLength; j++) { wordCounts[j] = NIOUtils.getString(buf); } update(wordCounts); } buf.rewind(); // mean perplexity over `numTrain` mini-batches perplexity = cumPerplexity / numTrain; logger.info("Mean perplexity over mini-batches: " + perplexity); if (Math.abs(perplexityPrev - perplexity) < eps) { break; } } logger.info("Performed " + Math.min(iter, iterations) + " iterations of " + NumberUtils.formatNumber(numTrainingExamples) + " training examples on memory (thus " + NumberUtils.formatNumber(numTrainingExamples * Math.min(iter, iterations)) + " training updates in total) "); } else {// read training examples in the temporary file and invoke train for each example // write training examples in buffer to a temporary file if (buf.remaining() > 0) { writeBuffer(buf, dst); } try { dst.flush(); } catch (IOException e) { throw new HiveException("Failed to flush a file: " + dst.getFile().getAbsolutePath(), e); } if (logger.isInfoEnabled()) { File tmpFile = dst.getFile(); logger.info( "Wrote " + numTrainingExamples + " records to a temporary file for iterative training: " + tmpFile.getAbsolutePath() + " (" + FileUtils.prettyFileSize(tmpFile) + ")"); } // run iterations int iter = 2; float perplexity = cumPerplexity / numTrain; float perplexityPrev; for (; iter <= iterations; iter++) { perplexityPrev = perplexity; cumPerplexity = 0.f; setCounterValue(iterCounter, iter); buf.clear(); dst.resetPosition(); while (true) { reportProgress(reporter); // TODO prefetch // writes training examples to a buffer in the temporary file final int bytesRead; try { bytesRead = dst.read(buf); } catch (IOException e) { throw new HiveException("Failed to read a file: " + dst.getFile().getAbsolutePath(), e); } if (bytesRead == 0) { // reached file EOF break; } assert (bytesRead > 0) : bytesRead; // reads training examples from a buffer buf.flip(); int remain = buf.remaining(); if (remain < SizeOf.INT) { throw new HiveException("Illegal file format was detected"); } while (remain >= SizeOf.INT) { int pos = buf.position(); int recordBytes = buf.getInt(); remain -= SizeOf.INT; if (remain < recordBytes) { buf.position(pos); break; } int wcLength = buf.getInt(); final String[] wordCounts = new String[wcLength]; for (int j = 0; j < wcLength; j++) { wordCounts[j] = NIOUtils.getString(buf); } update(wordCounts); remain -= recordBytes; } buf.compact(); } // mean perplexity over `numTrain` mini-batches perplexity = cumPerplexity / numTrain; logger.info("Mean perplexity over mini-batches: " + perplexity); if (Math.abs(perplexityPrev - perplexity) < eps) { break; } } logger.info("Performed " + Math.min(iter, iterations) + " iterations of " + NumberUtils.formatNumber(numTrainingExamples) + " training examples on a secondary storage (thus " + NumberUtils.formatNumber(numTrainingExamples * Math.min(iter, iterations)) + " training updates in total)"); } } catch (Throwable e) { throw new HiveException("Exception caused in the iterative training", e); } finally { // delete the temporary file and release resources try { dst.close(true); } catch (IOException e) { throw new HiveException("Failed to close a file: " + dst.getFile().getAbsolutePath(), e); } this.inputBuf = null; this.fileIO = null; } }
From source file:srebrinb.compress.sevenzip.SevenZFile.java
private void readPackInfo(final ByteBuffer header, final Archive archive) throws IOException { archive.packPos = readUint64(header); final long numPackStreams = readUint64(header); int nid = getUnsignedByte(header); if (nid == NID.kSize) { archive.packSizes = new long[(int) numPackStreams]; for (int i = 0; i < archive.packSizes.length; i++) { archive.packSizes[i] = readUint64(header); }/* w w w . j av a 2s .c om*/ nid = getUnsignedByte(header); } if (nid == NID.kCRC) { archive.packCrcsDefined = readAllOrBits(header, (int) numPackStreams); archive.packCrcs = new long[(int) numPackStreams]; for (int i = 0; i < (int) numPackStreams; i++) { if (archive.packCrcsDefined.get(i)) { archive.packCrcs[i] = 0xffffFFFFL & header.getInt(); } } nid = getUnsignedByte(header); } if (nid != NID.kEnd) { throw new IOException("Badly terminated PackInfo (" + nid + ")"); } }
From source file:org.carbondata.processing.util.LevelSortIndexWriterThread.java
private MemberSortModel[] getLevelData() throws IOException { DataInputStream fileChannel = null; long currPositionIndex = 0; long size = 0; ByteBuffer buffer = null; // CHECKSTYLE:OFF boolean enableEncoding = Boolean .valueOf(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ENABLE_BASE64_ENCODING, CarbonCommonConstants.ENABLE_BASE64_ENCODING_DEFAULT)); // CHECKSTYLE:ON try {// ww w . j av a 2s. c om fileChannel = FileFactory.getDataInputStream(levelFilePath, FileFactory.getFileType(levelFilePath)); CarbonFile memberFile = FileFactory.getCarbonFile(levelFilePath, FileFactory.getFileType(levelFilePath)); size = memberFile.getSize() - 4; long skipSize = size; long actualSkipSize = 0; while (actualSkipSize != size) { actualSkipSize += fileChannel.skip(skipSize); skipSize = skipSize - actualSkipSize; } maxSurrogate = fileChannel.readInt(); } catch (IOException e) { LOGGER.error(e, "problem while reading the level file"); throw e; } finally { CarbonUtil.closeStreams(fileChannel); } try { fileChannel = FileFactory.getDataInputStream(levelFilePath, FileFactory.getFileType(levelFilePath)); // CHECKSTYLE:OFF buffer = ByteBuffer.allocate((int) size); // CHECKSTYLE:ON fileChannel.readFully(buffer.array()); buffer.rewind(); } catch (IOException e) { LOGGER.error(e, "problem while reading the level file"); throw e; } finally { CarbonUtil.closeStreams(fileChannel); } minSurrogate = buffer.getInt(); MemberSortModel[] surogateKeyArrays = new MemberSortModel[maxSurrogate - minSurrogate + 1]; int surrogateKeyIndex = minSurrogate; currPositionIndex += 4; int current = 0; while (currPositionIndex < size) { int len = buffer.getInt(); // CHECKSTYLE:OFF // CHECKSTYLE:ON currPositionIndex += 4; byte[] rowBytes = new byte[len]; buffer.get(rowBytes); currPositionIndex += len; String memberName = null;// CHECKSTYLE:OFF if (!memberDataType.equals(DataType.STRING)) { if (enableEncoding) { memberName = new String(Base64.decodeBase64(rowBytes), Charset.defaultCharset()); } else { memberName = new String(rowBytes, Charset.defaultCharset()); } surogateKeyArrays[current] = new MemberSortModel(surrogateKeyIndex, memberName, null, memberDataType); } else { if (enableEncoding) { rowBytes = Base64.decodeBase64(rowBytes); } surogateKeyArrays[current] = new MemberSortModel(surrogateKeyIndex, null, rowBytes, memberDataType); } surrogateKeyIndex++; current++; } return surogateKeyArrays; }
From source file:org.apache.bookkeeper.bookie.Bookie.java
void readJournal() throws IOException, BookieException { long startTs = MathUtils.now(); journal.replay(new JournalScanner() { @Override/*from w ww . jav a 2 s . c o m*/ public void process(int journalVersion, long offset, ByteBuffer recBuff) throws IOException { long ledgerId = recBuff.getLong(); long entryId = recBuff.getLong(); try { LOG.debug("Replay journal - ledger id : {}, entry id : {}.", ledgerId, entryId); if (entryId == METAENTRY_ID_LEDGER_KEY) { if (journalVersion >= JournalChannel.V3) { int masterKeyLen = recBuff.getInt(); byte[] masterKey = new byte[masterKeyLen]; recBuff.get(masterKey); masterKeyCache.put(ledgerId, masterKey); } else { throw new IOException("Invalid journal. Contains journalKey " + " but layout version (" + journalVersion + ") is too old to hold this"); } } else if (entryId == METAENTRY_ID_FENCE_KEY) { if (journalVersion >= JournalChannel.V4) { byte[] key = masterKeyCache.get(ledgerId); if (key == null) { key = ledgerStorage.readMasterKey(ledgerId); } LedgerDescriptor handle = handles.getHandle(ledgerId, key); handle.setFenced(); } else { throw new IOException("Invalid journal. Contains fenceKey " + " but layout version (" + journalVersion + ") is too old to hold this"); } } else { byte[] key = masterKeyCache.get(ledgerId); if (key == null) { key = ledgerStorage.readMasterKey(ledgerId); } LedgerDescriptor handle = handles.getHandle(ledgerId, key); recBuff.rewind(); handle.addEntry(recBuff); } } catch (NoLedgerException nsle) { LOG.debug("Skip replaying entries of ledger {} since it was deleted.", ledgerId); } catch (BookieException be) { throw new IOException(be); } } }); long elapsedTs = MathUtils.now() - startTs; LOG.info("Finished replaying journal in {} ms.", elapsedTs); }
From source file:org.zaproxy.zap.extension.ascanrulesAlpha.GitMetadata.java
/** * get data for a given SHA1 object, using either the loose or packed formats * * @param basemsg the base message to use when retrieving additional resources * @param gitbasepath the Git base path//from w w w .j a v a2s .c o m * @param filesha1 the SHA1 associated with the file in Git * @param trypacked try the packed format, or try the loose format * @return the binary data associated with the file in Git, as specified by the filesha1 * parameter * @throws Exception */ public byte[] getObjectData(HttpMessage basemsg, String gitbasepath, String filesha1, boolean trypacked) throws Exception { URI originaluri = basemsg.getRequestHeader().getURI(); if (!trypacked) { // try the unpacked (loose) format URI gitobjecturi = new URI(originaluri.getScheme(), originaluri.getAuthority(), gitbasepath + "objects/" + filesha1.substring(0, 2) + "/" + filesha1.substring(2), null, null); if (log.isDebugEnabled()) log.debug("The internal Git (loose) file name is " + gitobjecturi.getURI()); byte[] data = getURIResponseBody(gitobjecturi, true, basemsg); ByteBuffer dataBuffer = ByteBuffer.wrap(data); StringBuilder sb = new StringBuilder(); while (true) { byte b = dataBuffer.get(); if (b == ' ') break; sb.append((char) b); } String objecttype = new String(sb); if (!objecttype.equals("blob")) { throw new Exception( "The Git 'loose' file '" + gitobjecturi + "' is not of type 'blob': '" + objecttype + "'"); } // read the size of data in the file (which appears as ASCII digits in the text), until // we get a 0x00 sb = new StringBuilder(); while (true) { byte b = dataBuffer.get(); if (b == 0x00) break; sb.append((char) b); } int dataSize = Integer.parseInt(new String(sb)); // now read that number of bytes from the bytebuffer, or at least attempt to.. byte[] blobDecoded = new byte[dataSize]; dataBuffer.get(blobDecoded); // that's it. we're done. return the decoded data, which will hopefully be source code // :) return blobDecoded; } else { // try the packed format // With the Git "packed" format, there are Git "pack index" files, and Git "pack" files. // They come as a set. You need both to get the contents of the file you're looking for. // The name of the Git "pack" files and "pack index" files is based on the SHA1 sum of // the SHA1 objects that it contains, and is not guessable. // This is an issue if you do not already know what pack files live in the directory // (unless you have a directory listing, for instance). // Luckily, in practice, in most cases (although not always) the name of the "pack" file // is contained in an ".git/objects/info/packs" file in the Git repo metadata. // The ".git/objects/info/packs" can also contain the names of multiple pack files, // which I have not seen in practice. That scenario is not currently supported here. // Both the "pack" and "pack index" files have an associated version number, but not // necessarily the same version number as each other. // There are constraints and interdependencies on these version numbers, however. // The Git "pack index" file currently comes in versions 1,2, and 3 (as of January 30, // 2014). // version 1 "pack index" files are not seen in the wild, but can be created using later // versions of Git, if necessary. Version 1 is supported here. // (Version 1 "pack index" files are seen in conjunction with Version 2 "pack" files, // but there is no reason (that I know of) why they should not also support Version 3 or // 4 pack files). // version 2 "pack index" files use either a version 2 or version 3 "pack" file. All // these versions are supported here. // (Version 1 and 2 "pack index" file formats have structural differences, but not // not wildly dis-similar). // version 3 "pack index" file cannot yet be created by any currently known version of // Git, but the format is documented. // (Version 3 "pack index" files require a version 4 "pack file". Both these versions // are tentatively supported here, although this code has never been tested) // The Git "pack" file currently comes in versions 1,2,3, and 4 (as of January 30, // 2014). // Version 1 "pack" files do not appear to be documented. They are not supported here. // Version 2 "pack files" are used with version 2 "pack index" files. This is a common // scenario in the wild. Both versions are supported here. // Version 3 "pack files" are (also) used with version 2 "pack index" files. Both // versions are supported here. // (Version 3 "pack files" are identical in format to version 2, with only the // version number differing) // Version 4 "pack files" are used in conjunction with version 3 "pack index" files. // Both these versions are tentatively supported here, although this code has never been // tested. // There are also separate version numbers in the Git "index file" (unrelated to the // "pack index" files mentioned above), which are probably similarly inter-related. // I do not have a mapping of the Git version number (1.7.6 / 1.8.5, for instance) to // any of the the internal file version numbers that they create (by default) or // support. So sue me. URI uri = new URI(originaluri.getScheme(), originaluri.getAuthority(), gitbasepath + "objects/info/packs", null, null); if (log.isDebugEnabled()) log.debug("The internal Git file containing the name of the pack file is " + uri); byte[] packinfofiledata = null; try { packinfofiledata = getURIResponseBody(uri, false, basemsg); } catch (FileNotFoundException e) { log.error("We could not read '" + uri + "' to get the name of the pack file containing the content: " + e.getMessage()); throw e; } ByteBuffer dataBuffer = ByteBuffer.wrap(packinfofiledata); StringBuilder sb = new StringBuilder(); while (true) { byte b = dataBuffer.get(); if (b == ' ') break; sb.append((char) b); } String objecttype = new String(sb); if (!objecttype.equals("P")) { throw new Exception("The pack info file is not of type 'P': '" + objecttype + "'"); } // the file should begin with "P ", and everything after that is the pack file name // (and exclude the 2 trailing newlines as well) // TODO: handle the case where this file contains the name of multiple pack files. // Currently, i have no test cases. Maybe in extremely large Git repositories? byte[] packfilenamebytes = new byte[packinfofiledata.length - 4]; dataBuffer.get(packfilenamebytes); String packfilename = new String(packfilenamebytes); // validate that the file name looks like "pack*.pack" Matcher packfilenamematcher = Pattern.compile("^pack-[0-9a-f]{40}\\.pack$").matcher(packfilename); if (!packfilenamematcher.find()) { throw new Exception( "The pack file name '" + packfilename + "' does not match the expected pattern"); } // Now generate the full name of the pack file, and the pack index. URI packuri = new URI(originaluri.getScheme(), originaluri.getAuthority(), gitbasepath + "objects/pack/" + packfilename, null, null); URI packindexuri = new URI(originaluri.getScheme(), originaluri.getAuthority(), gitbasepath + "objects/pack/" + packfilename.substring(0, packfilename.length() - 5) + ".idx", null, null); // retrieve the content for the "pack index" file! byte[] packfileindexdata = null; try { packfileindexdata = getURIResponseBody(packindexuri, false, basemsg); } catch (FileNotFoundException e) { System.out.println("We could not read '" + packindexuri + "', which is necessary to get the packed contents of the SHA1 requested: " + e.getMessage()); throw e; } // retrieve the content for the "pack" file! byte[] packfiledata = null; try { packfiledata = getURIResponseBody(packuri, false, basemsg); } catch (FileNotFoundException e) { System.out.println("We could not read '" + packuri + "', which should contain the packed contents of the SHA1 requested: " + e.getMessage()); throw e; } // now that we know we have both the "pack index" and the "pack" (data) file, parse the // data // first parse out some signature data info from the "pack" file ByteBuffer packfileheaderBuffer = ByteBuffer.wrap(packfiledata, 0, 12); byte[] packfileheaderSignatureArray = new byte[4]; // 4 bytes packfileheaderBuffer.get(packfileheaderSignatureArray); if (!new String(packfileheaderSignatureArray).equals("PACK")) { throw new Exception("The pack file header does not appear to be valid"); } int packFileVersion = packfileheaderBuffer.getInt(); // 4 bytes int packEntryCount = packfileheaderBuffer.getInt(); // 4 bytes if (packFileVersion != 2 && packFileVersion != 3 && packFileVersion != 4) { throw new Exception( "Only Git Pack File versions 2, 3, and 4 are currently supported. Git Pack File Version " + packFileVersion + " was found. Contact the zaproxy (OWASP Zap) dev team"); } // for pack file version 4, read the SHA1 tables from the "pack" file at this point // these used to live in the "pack index" file, in earlier versions. // Note: since at this point in time, there is no way to generate a v3 pack index file + // v4 pack file // so this particular block of code remains hypothetical. it seems to comply with the // documented version 4 "pack" file format, however, and it // works for version 2 "pack index" and version 2 "pack" files, which appears to be the // most common combination seen in the wild. int sha1Index = Integer.MAX_VALUE; int packEntryOffsetArray[] = null; int packEntryOffsetArrayOrdered[] = null; int indexEntryCount = 0; if (packFileVersion >= 4) { sha1Index = Integer.MAX_VALUE; // the tables in the V4 tables in the pack file are variable length, so just grab // the data after the main header for now ByteBuffer packfileTablesBuffer = ByteBuffer.wrap(packfiledata, 12, packfiledata.length - 12); // read the series of 20 byte sha1 entries. // ours *should* be in here somewhere.. find it // make sure to read *all* of the entries from the file (or else seek to the end of // the data), so the parsing logic is not broken. // TODO: use a binary search to find this in a more efficient manner for (int i = 0; i < packEntryCount; i++) { byte[] packTableData = new byte[20]; packfileTablesBuffer.get(packTableData); String packTableSha1 = Hex.encodeHexString(packTableData); // TODO: use more efficient byte based comparison to find the SHA1 here (and in // similar code in pack index version 2 logic, later.. if (packTableSha1.equals(filesha1)) { if (log.isDebugEnabled()) log.debug("FOUND our SHA1 " + packTableSha1 + " at entry " + i + " in the v4 pack tables"); sha1Index = i; // we do not need to "read past" all the entries. break; } } } // try to parse the "pack index" as a version 1 "pack index" file, which has a different // layout to subsequent versions. // use a separate ByteBuffer for this, in case things don't work out (because they // probably will not work out) :) try { ByteBuffer packindexfileV1dataBuffer = ByteBuffer.wrap(packfileindexdata); byte packEntrySizeArray[] = new byte[256 * 4]; packindexfileV1dataBuffer.get(packEntrySizeArray); if ( /*packEntrySizeArray[0]== 0xFF && */ packEntrySizeArray[1] == 't' && packEntrySizeArray[2] == 'O' && packEntrySizeArray[3] == 'c') { // the signature is a non-V1 signature. throw new NotV1GitPackIndexFileException(); } // get the last 4 bytes as an int, network order. indexEntryCount = (packEntrySizeArray[(255 * 4) + 3] << 0); indexEntryCount |= (packEntrySizeArray[(255 * 4) + 2] << 8); indexEntryCount |= (packEntrySizeArray[(255 * 4) + 1] << 16); indexEntryCount |= (packEntrySizeArray[(255 * 4) + 0] << 24); // validate that this matches the number of entries in the "pack" file. if (indexEntryCount != packEntryCount) { throw new Exception("The entry count (" + indexEntryCount + ") from the version 1 pack index file does not match the entry count (" + packEntryCount + ") from the pack file "); } if (log.isDebugEnabled()) log.debug("Got a pack index entry count of " + indexEntryCount + " from the version 1 pack index file"); // read the indexEntryCount * (4+20) byte entries (4 + 20 blackbirds baked in a // pie!) sha1Index = Integer.MAX_VALUE; packEntryOffsetArray = new int[indexEntryCount]; packEntryOffsetArrayOrdered = new int[indexEntryCount]; // TODO: use a binary search to find this in a more efficient manner for (int i = 0; i < indexEntryCount; i++) { // read 4 bytes offset (the offset of the SHA1's data in the "pack" file) packEntryOffsetArray[i] = packindexfileV1dataBuffer.getInt(); packEntryOffsetArrayOrdered[i] = packEntryOffsetArray[i]; // read 20 bytes SHA1 byte[] indexEntryIdBuffer = new byte[20]; packindexfileV1dataBuffer.get(indexEntryIdBuffer); String indexEntrySha1 = Hex.encodeHexString(indexEntryIdBuffer); if (indexEntrySha1.equals(filesha1)) { if (log.isDebugEnabled()) log.debug("FOUND our SHA1 " + indexEntrySha1 + " at entry " + i + " in the SHA1 table"); sha1Index = i; } } // final sanity check, if all of the above panned out for version 1 index file. // Note: we *think* that that "pack index" file version 1 is compatible with "pack" // file version 3 and 4, but really, we don't know for sure.. Again, so sue me. int packindexFileVersion = 1; if (packFileVersion != 2 && packFileVersion != 3 && packFileVersion != 4) { throw new Exception("Pack index file version (" + packindexFileVersion + ") is incompatible with pack file version (" + packFileVersion + ")"); } } catch (NotV1GitPackIndexFileException e) { // so it's not a version 1 "pack index" file. Try parsing it as a version 2, 3, 4 // (or later versions, once there are more versions, and we support them) if (log.isDebugEnabled()) log.debug( "The 'pack index' file looks like a > version 1 'pack index' file. Trying to parse it as later formats instead"); // Parse the "pack index" file header ByteBuffer packindexfiledataBuffer = ByteBuffer.wrap(packfileindexdata); byte[] packindexfileheaderSignatureArray = new byte[4]; packindexfiledataBuffer.get(packindexfileheaderSignatureArray); if ( /*packindexfileheaderSignatureArray[0]!= 0xFF || */ packindexfileheaderSignatureArray[1] != 't' || packindexfileheaderSignatureArray[2] != 'O' || packindexfileheaderSignatureArray[3] != 'c') { throw new Exception( "The pack index file header does not appear to be valid for pack index file version 2, 3, or 4: '" + new String(packindexfileheaderSignatureArray) + "' was found"); } // Note: version 1 is handled separately, so need to check for it here. int packindexFileVersion = packindexfiledataBuffer.getInt(); if (packindexFileVersion != 2 && packindexFileVersion != 3) { throw new Exception("Pack index file version(" + packindexFileVersion + ") is not supported"); } if ((packFileVersion == 2 || packFileVersion == 3) && packindexFileVersion != 2) { throw new Exception("Pack index file version (" + packindexFileVersion + ") is incompatible with pack file version (" + packFileVersion + ")"); } if (packindexFileVersion == 3 && packFileVersion != 4) { throw new Exception("Pack index file version (" + packindexFileVersion + ") is only compatible with pack file version 4. Pack file version (" + packFileVersion + ") was found"); } int packEntrySizeArray[] = new int[256]; for (int i = 0; i < 256; i++) { packEntrySizeArray[i] = packindexfiledataBuffer.getInt(); } // get the total number of entries, as being the number of entries from the final // fanout table entry. indexEntryCount = packEntrySizeArray[255]; // validate that this matches the number of entries in the pack file, according to // its header. if (indexEntryCount != packEntryCount) { throw new Exception("The entry count (" + indexEntryCount + ") from the pack index does not match the entry count (" + packEntryCount + ") from the pack file"); } // in version 3 of the pack index file, the SHA1 table moves from the pack index // file to the pack file (necessitating a version 4 pack file, as noted earlier) // in versions < 3 of the index file, the SHA1 data lives in the index file in some // manner (differs between version 1, and versions 2,3). if (packindexFileVersion < 3) { sha1Index = Integer.MAX_VALUE; // read the series of 20 byte sha1 entries. // ours *should* be in here somewhere.. find it // make sure to read *all* of the entries from the file (or else seek to the end // of the data), so the parsing logic is not broken. // TODO: use a binary search to find this in a more efficient manner for (int i = 0; i < indexEntryCount; i++) { byte[] indexEntryIdBuffer = new byte[20]; packindexfiledataBuffer.get(indexEntryIdBuffer); String indexEntrySha1 = Hex.encodeHexString(indexEntryIdBuffer); if (indexEntrySha1.equals(filesha1)) { if (log.isDebugEnabled()) log.debug("FOUND our SHA1 " + indexEntrySha1 + " at entry " + i + " in the SHA11 table"); sha1Index = i; } } } // read the CRCs for the various entries (and throw them away, for now) byte[] crcs = new byte[indexEntryCount * 4]; packindexfiledataBuffer.get(crcs); // read the offsets for the various entries. We need to know the offset into the // pack file of the SHA11 entry we are looking at // NB: the various tables in the "pack index" file are sorted by the corresponding // SHA1. // 2 adjacent entries in the offset table (for consequtive SHA11 entries) could have // wildly different offsets into the "pack" file // and the offsets in the table are therefore not sorted by offset. // In order to calculate the deflated length of an entry in the pack file (which is // not stored anywhere), // we need to generate an extra offset table, ordered by the offset. We will then // look for the next ordered offset, and store it alongside // the offset of the SHA1 we're interested in. packEntryOffsetArray = new int[indexEntryCount]; packEntryOffsetArrayOrdered = new int[indexEntryCount]; for (int i = 0; i < indexEntryCount; i++) { packEntryOffsetArray[i] = packindexfiledataBuffer.getInt(); packEntryOffsetArrayOrdered[i] = packEntryOffsetArray[i]; } } // now we're out of the pack index file version 1 or 2/3 specific stuff.. the rest of // the logic is fairly common (except for the "pack" file version 4 stuff, of course! :) Arrays.sort(packEntryOffsetArrayOrdered); // take account of the 20 byte sha1 checksum after all the individual entries int nextOffset = packfiledata.length - 20; // get the first offset greater than the offset of our sha1. since the table is ordered // by offset, these 2 offsets gives us the deflated length of the entry for (int i = 0; i < indexEntryCount; i++) { if (packEntryOffsetArrayOrdered[i] > packEntryOffsetArray[sha1Index]) { nextOffset = packEntryOffsetArrayOrdered[i]; // if (log.isDebugEnabled()) log.debug("Found the entry with the next offset: "+ // nextOffset); if (nextOffset > (packfiledata.length - 1)) throw new Exception("A 'next' offset of " + nextOffset + " is not feasible for a pack file with length " + packfiledata.length); break; } } // given the "pack" file offsets, we know the deflated length of the entry in there. int entryLength = (nextOffset - packEntryOffsetArray[sha1Index]); if (log.isDebugEnabled()) { log.debug("Our offset into the pack file is " + packEntryOffsetArray[sha1Index]); log.debug("The offset of the next entry into the pack file is " + nextOffset); log.debug("The deflated entry length, based on offset differences, is " + entryLength); } // get the data from the pack file and return it. byte[] inflatedData = getPackedObjectData(packfiledata, packEntryOffsetArray[sha1Index], entryLength, packFileVersion); return inflatedData; } }
From source file:edu.harvard.iq.dvn.ingest.statdataio.impl.plugins.dta.DTAFileReader.java
private void decodeHeader(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeHeader(): start *****"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }//from w ww . j av a 2 s . c om dbgLog.fine("reading the header segument 1: 4 byte\n"); byte[] magic_number = new byte[DTA_MAGIC_NUMBER_LENGTH]; int nbytes = stream.read(magic_number, 0, DTA_MAGIC_NUMBER_LENGTH); if (nbytes == 0) { throw new IOException(); } if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("hex dump: 1st 4bytes =>" + new String(Hex.encodeHex(magic_number)) + "<-"); if (magic_number[2] != 1) { dbgLog.fine("3rd byte is not 1: given file is not stata-dta type"); throw new IllegalArgumentException("given file is not stata-dta type"); } else if ((magic_number[1] != 1) && (magic_number[1] != 2)) { dbgLog.fine("2nd byte is neither 0 nor 1: this file is not stata-dta type"); throw new IllegalArgumentException("given file is not stata-dta type"); } else if (!STATA_RELEASE_NUMBER.containsKey((int) magic_number[0])) { dbgLog.fine("1st byte (" + magic_number[0] + ") is not within the ingestable range [rel. 3-10]:" + "we cannot ingest this Stata file."); throw new IllegalArgumentException("given file is not stata-dta type"); } else { releaseNumber = (int) magic_number[0]; smd.getFileInformation().put("releaseNumber", releaseNumber); smd.getFileInformation().put("byteOrder", (int) magic_number[1]); smd.getFileInformation().put("OSByteOrder", ByteOrder.nativeOrder().toString()); smd.getFileInformation().put("mimeType", MIME_TYPE[0]); smd.getFileInformation().put("fileFormat", MIME_TYPE[0]); init(); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("this file is stata-dta type: " + STATA_RELEASE_NUMBER.get((int) magic_number[0]) + "(Number=" + magic_number[0] + ")"); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("Endian(file)(Big: 1; Little:2)=" + magic_number[1]); if ((int) magic_number[1] == 2) { isLittleEndian = true; dbgLog.fine("Reveral of the bytes is necessary to decode " + "multi-byte fields"); } if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("Endian of this platform:" + ByteOrder.nativeOrder().toString()); } dbgLog.fine("reading the remaining header segument 2: 60 or 109-byte"); byte[] header = new byte[headerLength]; nbytes = stream.read(header, 0, headerLength); //printHexDump(header, "header:\n"); // 1. number of variables: short (2 bytes) ByteBuffer bbnvar = ByteBuffer.wrap(header, 0, NVAR_FIELD_LENGTH); ByteBuffer dupnvar = bbnvar.duplicate(); short short_nvar = dupnvar.getShort(); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("get original short view(nvar)=" + short_nvar); if (isLittleEndian) { bbnvar.order(ByteOrder.LITTLE_ENDIAN); } short shrt_nvar = bbnvar.getShort(); smd.getFileInformation().put("varQnty", new Integer(shrt_nvar)); // setup variableTypeList int nvar = shrt_nvar; variableTypelList = new String[nvar]; // 2. number of observations: int (4 bytes) ByteBuffer nobs = ByteBuffer.wrap(header, NVAR_FIELD_LENGTH, NOBS_FIELD_LENGTH); ByteBuffer dupnobs = nobs.duplicate(); int int_dupnobs = dupnobs.getInt(); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("raw nobs=" + int_dupnobs); if (isLittleEndian) { nobs.order(ByteOrder.LITTLE_ENDIAN); } int int_nobs = nobs.getInt(); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("reversed nobs=" + int_nobs); smd.getFileInformation().put("caseQnty", new Integer(int_nobs)); // 3. data_label: 32 or 81 bytes int dl_offset = NVAR_FIELD_LENGTH + NOBS_FIELD_LENGTH; if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("dl_offset=" + dl_offset); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("data_label_length=" + dataLabelLength); String data_label = new String(Arrays.copyOfRange(header, dl_offset, (dl_offset + dataLabelLength)), "ISO-8859-1"); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("data_label_length=" + data_label.length()); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("loation of the null character=" + data_label.indexOf(0)); String dataLabel = getNullStrippedString(data_label); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("data_label_length=" + dataLabel.length()); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("data_label=[" + dataLabel + "]"); smd.getFileInformation().put("dataLabel", dataLabel); // 4. time_stamp: ASCII String (18 bytes) // added after release 4 if (releaseNumber > 104) { int ts_offset = dl_offset + dataLabelLength; String time_stamp = new String(Arrays.copyOfRange(header, ts_offset, ts_offset + TIME_STAMP_LENGTH), "ISO-8859-1"); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("time_stamp_length=" + time_stamp.length()); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("loation of the null character=" + time_stamp.indexOf(0)); String timeStamp = getNullStrippedString(time_stamp); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("timeStamp_length=" + timeStamp.length()); if (dbgLog.isLoggable(Level.FINE)) dbgLog.fine("timeStamp=[" + timeStamp + "]"); smd.getFileInformation().put("timeStamp", timeStamp); smd.getFileInformation().put("fileDate", timeStamp); smd.getFileInformation().put("fileTime", timeStamp); smd.getFileInformation().put("varFormat_schema", "STATA"); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("smd dump:" + smd.toString()); dbgLog.fine("***** decodeHeader(): end *****"); } }
From source file:com.healthmarketscience.jackcess.impl.ColumnImpl.java
/** * Deserialize a raw byte value for this column into an Object * @param data The raw byte value/* w w w.j a v a 2s.co m*/ * @param order Byte order in which the raw value is stored * @return The deserialized Object * @usage _advanced_method_ */ public Object read(byte[] data, ByteOrder order) throws IOException { ByteBuffer buffer = ByteBuffer.wrap(data).order(order); switch (getType()) { case BOOLEAN: throw new IOException("Tried to read a boolean from data instead of null mask."); case BYTE: return Byte.valueOf(buffer.get()); case INT: return Short.valueOf(buffer.getShort()); case LONG: return Integer.valueOf(buffer.getInt()); case DOUBLE: return Double.valueOf(buffer.getDouble()); case FLOAT: return Float.valueOf(buffer.getFloat()); case SHORT_DATE_TIME: return readDateValue(buffer); case BINARY: return data; case TEXT: return decodeTextValue(data); case MONEY: return readCurrencyValue(buffer); case NUMERIC: return readNumericValue(buffer); case GUID: return readGUIDValue(buffer, order); case UNKNOWN_0D: case UNKNOWN_11: // treat like "binary" data return data; case COMPLEX_TYPE: return new ComplexValueForeignKeyImpl(this, buffer.getInt()); default: throw new IOException("Unrecognized data type: " + _type); } }
From source file:edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta.DTAFileReader.java
private void decodeHeader(BufferedInputStream stream) throws IOException { dbgLog.fine("***** decodeHeader(): start *****"); if (stream == null) { throw new IllegalArgumentException("stream == null!"); }//from w w w .j av a2 s . c o m dbgLog.fine("reading the header segument 1: 4 byte\n"); byte[] magic_number = new byte[DTA_MAGIC_NUMBER_LENGTH]; int nbytes = stream.read(magic_number, 0, DTA_MAGIC_NUMBER_LENGTH); if (nbytes == 0) { throw new IOException(); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("hex dump: 1st 4bytes =>" + new String(Hex.encodeHex(magic_number)) + "<-"); } if (magic_number[2] != 1) { dbgLog.fine("3rd byte is not 1: given file is not stata-dta type"); throw new IllegalArgumentException("The file is not in a STATA format that we can read or support."); } else if ((magic_number[1] != 1) && (magic_number[1] != 2)) { dbgLog.fine("2nd byte is neither 0 nor 1: this file is not stata-dta type"); throw new IllegalArgumentException("given file is not stata-dta type"); } else if (!STATA_RELEASE_NUMBER.containsKey((int) magic_number[0])) { dbgLog.fine("1st byte (" + magic_number[0] + ") is not within the ingestable range [rel. 3-10]:" + "we cannot ingest this Stata file."); throw new IllegalArgumentException("given file is not stata-dta type"); } else { releaseNumber = magic_number[0]; init(); dataTable.setOriginalFileFormat(MIME_TYPE[0]); /* * releaseNumber: * for storing in the datatable, we are converting the numeric Stata * release number into a more user friendly "version number"; * e.g., "release number 115" = "Stata v. 12" * -- L.A. 4.0 */ dataTable.setOriginalFormatVersion(STATA_RELEASE_NUMBER.get(releaseNumber)); dataTable.setUnf("UNF:6:FILEFILEFILEFILE"); if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("this file is stata-dta type: " + STATA_RELEASE_NUMBER.get(releaseNumber) + " (that means Stata version " + releaseNumber + ")"); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("Endian(file)(Big: 1; Little:2)=" + magic_number[1]); } /* * byte order: defined in the second byte of the "magic number": */ if (magic_number[1] == 2) { isLittleEndian = true; dbgLog.fine("Reversal of the bytes is necessary to decode " + "multi-byte fields"); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("Endian of this platform:" + ByteOrder.nativeOrder().toString()); } } dbgLog.fine("reading the remaining header segument 2: 60 or 109-byte"); byte[] header = new byte[headerLength]; nbytes = stream.read(header, 0, headerLength); // 1. number of variables: short (2 bytes) ByteBuffer bbnvar = ByteBuffer.wrap(header, 0, NVAR_FIELD_LENGTH); ByteBuffer dupnvar = bbnvar.duplicate(); short short_nvar = dupnvar.getShort(); if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("get original short view(nvar)=" + short_nvar); } if (isLittleEndian) { bbnvar.order(ByteOrder.LITTLE_ENDIAN); } short shrt_nvar = bbnvar.getShort(); dataTable.setVarQuantity(new Long(shrt_nvar)); int nvar = shrt_nvar; if (dbgLog.isLoggable(Level.INFO)) { dbgLog.info("number of variables(nvar)=" + nvar); } // 4.0 Initialize dataverse variable objects: List<DataVariable> variableList = new ArrayList<>(); for (int i = 0; i < nvar; i++) { DataVariable dv = new DataVariable(); dv.setInvalidRanges(new ArrayList<>()); dv.setSummaryStatistics(new ArrayList<>()); dv.setUnf("UNF:6:XXX"); dv.setCategories(new ArrayList<>()); variableList.add(dv); dv.setFileOrder(i); dv.setDataTable(dataTable); } dataTable.setDataVariables(variableList); // setup variableTypeList variableTypes = new String[nvar]; // and the date/time format list: dateVariableFormats = new String[nvar]; // 2. number of observations: int (4 bytes) ByteBuffer nobs = ByteBuffer.wrap(header, NVAR_FIELD_LENGTH, NOBS_FIELD_LENGTH); ByteBuffer dupnobs = nobs.duplicate(); int int_dupnobs = dupnobs.getInt(); if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("raw nobs=" + int_dupnobs); } if (isLittleEndian) { nobs.order(ByteOrder.LITTLE_ENDIAN); } int int_nobs = nobs.getInt(); if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("reversed nobs=" + int_nobs); } // smd.getFileInformation().put("caseQnty", new Integer(int_nobs)); dataTable.setCaseQuantity(new Long(int_nobs)); /* the "data label" - note that we are not using this label for anything (wonder what it is though? can we use it somewhere?) but we still need to extract it from the byte stream, since the offsets of the objects stored further up are calculated relative to it. -- L.A., 4.0 */ // 3. data_label: 32 or 81 bytes int dl_offset = NVAR_FIELD_LENGTH + NOBS_FIELD_LENGTH; if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("dl_offset=" + dl_offset); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("data_label_length=" + dataLabelLength); } String data_label = new String(Arrays.copyOfRange(header, dl_offset, (dl_offset + dataLabelLength)), "ISO-8859-1"); if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("data_label_length=" + data_label.length()); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("loation of the null character=" + data_label.indexOf(0)); } String dataLabel = getNullStrippedString(data_label); if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("data_label_length=" + dataLabel.length()); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("data_label=[" + dataLabel + "]"); } // smd.getFileInformation().put("dataLabel", dataLabel); /* end of "data label" */ // 4. time_stamp: ASCII String (18 bytes) // added after release 4 if (releaseNumber > 104) { int ts_offset = dl_offset + dataLabelLength; String time_stamp = new String(Arrays.copyOfRange(header, ts_offset, ts_offset + TIME_STAMP_LENGTH), "ISO-8859-1"); if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("time_stamp_length=" + time_stamp.length()); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("loation of the null character=" + time_stamp.indexOf(0)); } String timeStamp = getNullStrippedString(time_stamp); if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("timeStamp_length=" + timeStamp.length()); } if (dbgLog.isLoggable(Level.FINE)) { dbgLog.fine("timeStamp=[" + timeStamp + "]"); } } }