List of usage examples for java.io InputStream mark
public synchronized void mark(int readlimit)
From source file:org.infoscoop.request.filter.DetectTypeFilter.java
private boolean isXml(String contentType, InputStream is) throws IOException { if (contentType != null && (contentType.indexOf("text/xml") >= 0 || contentType.indexOf("application/xml") >= 0 || contentType.indexOf("application/rss+xml") >= 0 || contentType.indexOf("application/rdf+xml") >= 0 || contentType.indexOf("application/atom+xml") >= 0)) { return true; }//from w w w . j a v a2 s .c om is.mark(1); byte[] xmldec = new byte[500]; is.read(xmldec); String xmlDecStr = new String(xmldec); is.reset(); if (xmlDecStr.indexOf("<?xml") >= 0) { return true; } return false; }
From source file:com.microsoft.azure.storage.core.Utility.java
/** * Encrypts an input stream up to a given length. * Exits early if the encrypted data is longer than the abandon length. * // w ww. ja v a 2s .com * @param sourceStream * A <code>InputStream</code> object that represents the stream to measure. * @param targetStream * A <code>ByteArrayOutputStream</code> object that represents the stream to write the encrypted data. * @param cipher * The <code>Cipher</code> to use to encrypt the data. * @param writeLength * The number of bytes to read and encrypt from the sourceStream. * @param abandonLength * The number of bytes to read before the analysis is abandoned. Set this value to <code>-1</code> to * force the entire stream to be read. This parameter is provided to support upload thresholds. * @return * The size of the encrypted stream, or -1 if the encrypted stream would be over the abandonLength. * @throws IOException * If an I/O error occurs. */ public static long encryptStreamIfUnderThreshold(final InputStream sourceStream, final ByteArrayOutputStream targetStream, Cipher cipher, long writeLength, long abandonLength) throws IOException { if (abandonLength < 0) { abandonLength = Long.MAX_VALUE; } if (!sourceStream.markSupported()) { throw new IllegalArgumentException(SR.INPUT_STREAM_SHOULD_BE_MARKABLE); } sourceStream.mark(Constants.MAX_MARK_LENGTH); if (writeLength < 0) { writeLength = Long.MAX_VALUE; } CipherOutputStream encryptStream = new CipherOutputStream(targetStream, cipher); int count = -1; long totalEncryptedLength = targetStream.size(); final byte[] retrievedBuff = new byte[Constants.BUFFER_COPY_LENGTH]; int nextCopy = (int) Math.min(retrievedBuff.length, writeLength - totalEncryptedLength); count = sourceStream.read(retrievedBuff, 0, nextCopy); while (nextCopy > 0 && count != -1) { // Note: We are flushing the CryptoStream on every write here. This way, we don't end up encrypting more data than we intend here, if // we go over the abandonLength. encryptStream.write(retrievedBuff, 0, count); encryptStream.flush(); totalEncryptedLength = targetStream.size(); if (totalEncryptedLength > abandonLength) { // Abandon operation break; } nextCopy = (int) Math.min(retrievedBuff.length, writeLength - totalEncryptedLength); count = sourceStream.read(retrievedBuff, 0, nextCopy); } sourceStream.reset(); sourceStream.mark(Constants.MAX_MARK_LENGTH); encryptStream.close(); totalEncryptedLength = targetStream.size(); if (totalEncryptedLength > abandonLength) { totalEncryptedLength = -1; } return totalEncryptedLength; }
From source file:eu.europa.ec.markt.dss.validation102853.SignedDocumentValidator.java
/** * Guess the document format and return an appropriate document * * @param document The instance of DSSDocument to be validated * @return returns the specific instance of SignedDocumentValidator in terms of the document type */// w w w . java 2s . c o m public static SignedDocumentValidator fromDocument(final DSSDocument document) throws IOException { InputStream input = null; try { if (document.getName() != null && document.getName().toLowerCase().endsWith(".xml")) { try { return new XMLDocumentValidator(document); } catch (ParserConfigurationException e) { throw new IOException("Not a valid XML", e); } catch (SAXException e) { throw new IOException("Not a valid XML", e); } } input = new BufferedInputStream(document.openStream()); input.mark(5); byte[] preamble = new byte[5]; int read = input.read(preamble); input.reset(); if (read < 5) { throw new RuntimeException("Not a signed document"); } String preambleString = new String(preamble); byte[] xmlPreamble = new byte[] { '<', '?', 'x', 'm', 'l' }; byte[] xmlUtf8 = new byte[] { -17, -69, -65, '<', '?' }; if (Arrays.equals(preamble, xmlPreamble) || Arrays.equals(preamble, xmlUtf8)) { try { return new XMLDocumentValidator(document); } catch (ParserConfigurationException e) { throw new IOException("Not a valid XML", e); } catch (SAXException e) { throw new IOException("Not a valid XML", e); } } else if (preambleString.equals("%PDF-")) { return new PDFDocumentValidator(document); } else if (preamble[0] == 'P' && preamble[1] == 'K') { try { input.close(); } catch (IOException e) { } input = null; return getInstanceForAsics(document); } else if (preambleString.getBytes()[0] == 0x30) { try { return new CMSDocumentValidator(document); } catch (CMSException e) { throw new IOException("Not a valid CAdES file", e); } } else { throw new RuntimeException("Document format not recognized/handled"); } } finally { if (input != null) { try { input.close(); } catch (IOException e) { } } } }
From source file:com.smartsheet.api.internal.http.HttpEntitySnapshot.java
/** * this ctor creates a snapshot of the original entity (which requires its stream either support reset or it must be * entirely consumed and replaced with an exact copy) *///from www. j a v a 2 s . c o m public HttpEntitySnapshot(HttpEntity original) throws IOException { final String contentType = original.getContentType(); final InputStream contentStream = original.getContent(); final long contentLength = original.getContentLength(); super.setContentLength(contentLength); super.setContentType(contentType); if (contentType != null && contentType.startsWith(JSON_MIME_TYPE)) { // we need to read and then reset (if possible) the original entity's content stream (or replace it with an exact copy) // if contentLength > Integer.MAX_VALUE we have MUCH bigger problems than long->int rollover boolean sourceSupportsMark = contentStream.markSupported(); if (sourceSupportsMark) { // here we can read up to a limited contents contentArray = new byte[MAX_SNAPSHOT_SIZE]; contentStream.mark(MAX_SNAPSHOT_SIZE + 1); int bytesRead = contentStream.read(contentArray, 0, MAX_SNAPSHOT_SIZE); contentStream.reset(); // trim content array to actual size if (bytesRead < MAX_SNAPSHOT_SIZE) { contentArray = Arrays.copyOf(contentArray, bytesRead); } } else { // here we must read everything and then repackage the byte[] into an input stream to replace the original byte[] fullContentArray; try { fullContentArray = StreamUtil.readBytesFromStream(contentStream, StreamUtil.ONE_MB); } finally { contentStream.close(); } // having consumed the content into memory we must now replace the original stream (so it can be read by subsequent code) original.setContent(new ByteArrayInputStream(fullContentArray)); // and we need a copy for potential logging purposes contentArray = Arrays.copyOf(fullContentArray, Math.min(MAX_SNAPSHOT_SIZE, fullContentArray.length)); // we see a lot of Content-Length:-1 from certain responses - no point in logging those if (contentLength != -1 && fullContentArray.length != contentLength) { LoggerFactory.getLogger(HttpEntitySnapshot.class).info( "actual content-length {} doesn't match" + " declared content-length {}", fullContentArray.length, contentLength); } } } else { contentArray = String.format("**contentType '%s' not logged**", contentType).getBytes(); } }
From source file:com.amazonaws.services.glacier.transfer.ArchiveTransferManager.java
private UploadResult uploadInMultipleParts(final String accountId, final String vaultName, final String archiveDescription, final File file, ProgressListener progressListener) { long partSize = calculatePartSize(file.length()); String partSizeString = Long.toString(partSize); publishProgress(progressListener, ProgressEventType.TRANSFER_PREPARING_EVENT); String uploadId = null;/*w ww.ja v a 2 s .c o m*/ try { InitiateMultipartUploadResult initiateResult = glacier .initiateMultipartUpload(new InitiateMultipartUploadRequest().withAccountId(accountId) .withArchiveDescription(archiveDescription).withVaultName(vaultName) .withPartSize(partSizeString)); uploadId = initiateResult.getUploadId(); } catch (Throwable t) { publishProgress(progressListener, ProgressEventType.TRANSFER_FAILED_EVENT); throw failure(t); } publishProgress(progressListener, ProgressEventType.TRANSFER_STARTED_EVENT); try { List<byte[]> binaryChecksums = new LinkedList<byte[]>(); long currentPosition = 0; while (currentPosition < file.length()) { long length = partSize; if (currentPosition + partSize > file.length()) { length = file.length() - currentPosition; } Exception failedException = null; boolean completed = false; int tries = 0; while (!completed && tries < 5) { tries++; InputStream inputSubStream = newInputSubstream(file, currentPosition, length); inputSubStream.mark(-1); String checksum = TreeHashGenerator.calculateTreeHash(inputSubStream); byte[] binaryChecksum = BinaryUtils.fromHex(checksum); inputSubStream.reset(); UploadMultipartPartRequest req = new UploadMultipartPartRequest().withAccountId(accountId) .withChecksum(checksum).withBody(inputSubStream) .withRange("bytes " + currentPosition + "-" + (currentPosition + length - 1) + "/*") .withUploadId(uploadId).withVaultName(vaultName) .withGeneralProgressListener(progressListener); try { glacier.uploadMultipartPart(req); completed = true; binaryChecksums.add(binaryChecksum); } catch (Exception e) { failedException = e; } finally { closeQuietly(inputSubStream, log); } } if (!completed && failedException != null) throw failedException; currentPosition += partSize; } String checksum = TreeHashGenerator.calculateTreeHash(binaryChecksums); String archiveSize = Long.toString(file.length()); CompleteMultipartUploadResult completeMultipartUploadResult = glacier.completeMultipartUpload( new CompleteMultipartUploadRequest().withAccountId(accountId).withArchiveSize(archiveSize) .withVaultName(vaultName).withChecksum(checksum).withUploadId(uploadId)); String artifactId = completeMultipartUploadResult.getArchiveId(); publishProgress(progressListener, ProgressEventType.TRANSFER_COMPLETED_EVENT); return new UploadResult(artifactId); } catch (Throwable t) { publishProgress(progressListener, ProgressEventType.TRANSFER_FAILED_EVENT); glacier.abortMultipartUpload(new AbortMultipartUploadRequest(accountId, vaultName, uploadId)); throw failure(t, "Unable to finish the upload"); } }
From source file:eu.medsea.mimeutil.detector.OpendesktopMimeDetector.java
private Collection lookupMimeTypesForMagicData(InputStream in) { int offset = 0; int len = getMaxExtents(); byte[] data = new byte[len]; // Mark the input stream in.mark(len); try {//from w ww . ja va 2s . c o m // Since an InputStream might return only some data (not all // requested), we have to read in a loop until // either EOF is reached or the desired number of bytes have been // read. int restBytesToRead = len; while (restBytesToRead > 0) { int bytesRead = in.read(data, offset, restBytesToRead); if (bytesRead < 0) break; // EOF offset += bytesRead; restBytesToRead -= bytesRead; } } catch (IOException ioe) { throw new MimeException(ioe); } finally { try { // Reset the input stream to where it was marked. in.reset(); } catch (Exception e) { throw new MimeException(e); } } return lookupMagicData(data); }
From source file:com.microsoft.azure.storage.core.Utility.java
/** * Reads data from an input stream and writes it to an output stream, calculates the length of the data written, and * optionally calculates the MD5 hash for the data. * //from w w w . j a va2s . c o m * @param sourceStream * An <code>InputStream</code> object that represents the input stream to use as the source. * @param outStream * An <code>OutputStream</code> object that represents the output stream to use as the destination. * @param writeLength * The number of bytes to read from the stream. * @param rewindSourceStream * <code>true</code> if the input stream should be rewound <strong>before</strong> it is read; otherwise, * <code>false</code> * @param calculateMD5 * <code>true</code> if an MD5 hash will be calculated; otherwise, <code>false</code>. * @param opContext * An {@link OperationContext} object that represents the context for the current operation. This object * is used to track requests to the storage service, and to provide additional runtime information about * the operation. * @param options * A {@link RequestOptions} object that specifies any additional options for the request. Namely, the * maximum execution time. * @param request * Used by download resume to set currentRequestByteCount on the request. Otherwise, null is always used. * @return A {@link StreamMd5AndLength} object that contains the output stream length, and optionally the MD5 hash. * * @throws IOException * If an I/O error occurs. * @throws StorageException * If a storage service error occurred. */ public static StreamMd5AndLength writeToOutputStream(final InputStream sourceStream, final OutputStream outStream, long writeLength, final boolean rewindSourceStream, final boolean calculateMD5, OperationContext opContext, final RequestOptions options, final Boolean shouldFlush, StorageRequest<?, ?, Integer> request) throws IOException, StorageException { if (rewindSourceStream && sourceStream.markSupported()) { sourceStream.reset(); sourceStream.mark(Constants.MAX_MARK_LENGTH); } final StreamMd5AndLength retVal = new StreamMd5AndLength(); if (calculateMD5) { try { retVal.setDigest(MessageDigest.getInstance("MD5")); } catch (final NoSuchAlgorithmException e) { // This wont happen, throw fatal. throw Utility.generateNewUnexpectedStorageException(e); } } if (writeLength < 0) { writeLength = Long.MAX_VALUE; } final byte[] retrievedBuff = new byte[Constants.BUFFER_COPY_LENGTH]; int nextCopy = (int) Math.min(retrievedBuff.length, writeLength); int count = sourceStream.read(retrievedBuff, 0, nextCopy); while (nextCopy > 0 && count != -1) { // if maximum execution time would be exceeded if (Utility.validateMaxExecutionTimeout(options.getOperationExpiryTimeInMs())) { // throw an exception TimeoutException timeoutException = new TimeoutException(SR.MAXIMUM_EXECUTION_TIMEOUT_EXCEPTION); throw Utility.initIOException(timeoutException); } if (outStream != null) { outStream.write(retrievedBuff, 0, count); } if (calculateMD5) { retVal.getDigest().update(retrievedBuff, 0, count); } retVal.setLength(retVal.getLength() + count); retVal.setCurrentOperationByteCount(retVal.getCurrentOperationByteCount() + count); if (request != null) { request.setCurrentRequestByteCount(request.getCurrentRequestByteCount() + count); } nextCopy = (int) Math.min(retrievedBuff.length, writeLength - retVal.getLength()); count = sourceStream.read(retrievedBuff, 0, nextCopy); } if (outStream != null && shouldFlush) { outStream.flush(); } return retVal; }
From source file:org.alfresco.rm.rest.api.impl.FilePlanComponentsApiUtils.java
/** * Write content to file//from w ww. jav a 2 s. c o m * * @param nodeRef the node to write the content to * @param fileName the name of the file (used for guessing the file's mimetype) * @param stream the input stream to write * @param guessEncoding whether to guess stream encoding */ public void writeContent(NodeRef nodeRef, String fileName, InputStream stream, boolean guessEncoding) { try { ContentWriter writer = contentService.getWriter(nodeRef, ContentModel.PROP_CONTENT, true); String mimeType = mimetypeService.guessMimetype(fileName); if ((mimeType != null) && (!mimeType.equals(MimetypeMap.MIMETYPE_BINARY))) { // quick/weak guess based on file extension writer.setMimetype(mimeType); } else { // stronger guess based on file stream writer.guessMimetype(fileName); } InputStream is = null; if (guessEncoding) { is = new BufferedInputStream(stream); is.mark(1024); writer.setEncoding(guessEncoding(is, mimeType, false)); try { is.reset(); } catch (IOException ioe) { if (LOGGER.isWarnEnabled()) { LOGGER.warn("Failed to reset stream after trying to guess encoding: " + ioe.getMessage()); } } } else { is = stream; } writer.putContent(is); } catch (ContentQuotaException cqe) { throw new InsufficientStorageException(); } catch (ContentLimitViolationException clv) { throw new RequestEntityTooLargeException(clv.getMessage()); } catch (ContentIOException cioe) { if (cioe.getCause() instanceof NodeLockedException) { throw (NodeLockedException) cioe.getCause(); } throw cioe; } }
From source file:org.codelibs.fess.crawler.extractor.impl.TikaExtractor.java
@Override public ExtractData getText(final InputStream inputStream, final Map<String, String> params) { if (inputStream == null) { throw new CrawlerSystemException("The inputstream is null."); }//from w w w .j a v a2 s.c om final File tempFile; final boolean isByteStream = inputStream instanceof ByteArrayInputStream; if (isByteStream) { inputStream.mark(0); tempFile = null; } else { try { tempFile = File.createTempFile("tikaExtractor-", ".out"); } catch (final IOException e) { throw new ExtractException("Could not create a temp file.", e); } } try { final PrintStream originalOutStream = System.out; final ByteArrayOutputStream outStream = new ByteArrayOutputStream(); System.setOut(new PrintStream(outStream, true)); final PrintStream originalErrStream = System.err; final ByteArrayOutputStream errStream = new ByteArrayOutputStream(); System.setErr(new PrintStream(errStream, true)); try { final String resourceName = params == null ? null : params.get(TikaMetadataKeys.RESOURCE_NAME_KEY); final String contentType = params == null ? null : params.get(HttpHeaders.CONTENT_TYPE); String contentEncoding = params == null ? null : params.get(HttpHeaders.CONTENT_ENCODING); // password for pdf String pdfPassword = params == null ? null : params.get(ExtractData.PDF_PASSWORD); if (pdfPassword == null && params != null) { pdfPassword = getPdfPassword(params.get(ExtractData.URL), resourceName); } final Metadata metadata = createMetadata(resourceName, contentType, contentEncoding, pdfPassword); final Parser parser = new DetectParser(); final ParseContext parseContext = new ParseContext(); parseContext.set(Parser.class, parser); String content = getContent(writer -> { InputStream in = null; try { if (!isByteStream) { try (OutputStream out = new FileOutputStream(tempFile)) { CopyUtil.copy(inputStream, out); } in = new FileInputStream(tempFile); } else { in = inputStream; } parser.parse(in, new BodyContentHandler(writer), metadata, parseContext); } finally { IOUtils.closeQuietly(in); } }, contentEncoding); if (StringUtil.isBlank(content)) { if (resourceName != null) { if (logger.isDebugEnabled()) { logger.debug("retry without a resource name: {}", resourceName); } final Metadata metadata2 = createMetadata(null, contentType, contentEncoding, pdfPassword); content = getContent(writer -> { InputStream in = null; try { if (isByteStream) { inputStream.reset(); in = inputStream; } else { in = new FileInputStream(tempFile); } parser.parse(in, new BodyContentHandler(writer), metadata2, parseContext); } finally { IOUtils.closeQuietly(in); } }, contentEncoding); } if (StringUtil.isBlank(content) && contentType != null) { if (logger.isDebugEnabled()) { logger.debug("retry without a content type: {}", contentType); } final Metadata metadata3 = createMetadata(null, null, contentEncoding, pdfPassword); content = getContent(writer -> { InputStream in = null; try { if (isByteStream) { inputStream.reset(); in = inputStream; } else { in = new FileInputStream(tempFile); } parser.parse(in, new BodyContentHandler(writer), metadata3, parseContext); } finally { IOUtils.closeQuietly(in); } }, contentEncoding); } if (readAsTextIfFailed && StringUtil.isBlank(content)) { if (logger.isDebugEnabled()) { logger.debug("read the content as a text."); } if (contentEncoding == null) { contentEncoding = Constants.UTF_8; } final String enc = contentEncoding; content = getContent(writer -> { BufferedReader br = null; try { if (isByteStream) { inputStream.reset(); br = new BufferedReader(new InputStreamReader(inputStream, enc)); } else { br = new BufferedReader( new InputStreamReader(new FileInputStream(tempFile), enc)); } String line; while ((line = br.readLine()) != null) { writer.write(line); } } catch (final Exception e) { logger.warn( "Could not read " + (tempFile != null ? tempFile.getAbsolutePath() : "a byte stream"), e); } finally { IOUtils.closeQuietly(br); } }, contentEncoding); } } final ExtractData extractData = new ExtractData(content); final String[] names = metadata.names(); Arrays.sort(names); for (final String name : names) { extractData.putValues(name, metadata.getValues(name)); } if (logger.isDebugEnabled()) { logger.debug("Result: metadata: {}", metadata); } return extractData; } catch (final TikaException e) { if (e.getMessage().indexOf("bomb") >= 0) { throw e; } final Throwable cause = e.getCause(); if (cause instanceof SAXException) { final Extractor xmlExtractor = crawlerContainer.getComponent("xmlExtractor"); if (xmlExtractor != null) { InputStream in = null; try { if (isByteStream) { inputStream.reset(); in = inputStream; } else { in = new FileInputStream(tempFile); } return xmlExtractor.getText(in, params); } finally { IOUtils.closeQuietly(in); } } } throw e; } finally { if (originalOutStream != null) { System.setOut(originalOutStream); } if (originalErrStream != null) { System.setErr(originalErrStream); } try { if (logger.isInfoEnabled()) { final byte[] bs = outStream.toByteArray(); if (bs.length != 0) { logger.info(new String(bs, outputEncoding)); } } if (logger.isWarnEnabled()) { final byte[] bs = errStream.toByteArray(); if (bs.length != 0) { logger.warn(new String(bs, outputEncoding)); } } } catch (final Exception e) { // NOP } } } catch (final Exception e) { throw new ExtractException("Could not extract a content.", e); } finally { if (tempFile != null && !tempFile.delete()) { logger.warn("Failed to delete " + tempFile.getAbsolutePath()); } } }
From source file:com.microsoft.azure.storage.core.Utility.java
/** * /*from w ww . j a va 2 s . c o m*/ * Determines the size of an input stream, and optionally calculates the MD5 hash for the stream. * * @param sourceStream * A <code>InputStream</code> object that represents the stream to measure. * @param writeLength * The number of bytes to read from the stream. * @param abandonLength * The number of bytes to read before the analysis is abandoned. Set this value to <code>-1</code> to * force the entire stream to be read. This parameter is provided to support upload thresholds. * @param rewindSourceStream * <code>true</code> if the stream should be rewound after it is read; otherwise, <code>false</code>. * @param calculateMD5 * <code>true</code> if an MD5 hash will be calculated; otherwise, <code>false</code>. * * @return A {@link StreamMd5AndLength} object that contains the stream length, and optionally the MD5 hash. * * @throws IOException * If an I/O error occurs. * @throws StorageException * If a storage service error occurred. */ public static StreamMd5AndLength analyzeStream(final InputStream sourceStream, long writeLength, long abandonLength, final boolean rewindSourceStream, final boolean calculateMD5) throws IOException, StorageException { if (abandonLength < 0) { abandonLength = Long.MAX_VALUE; } if (rewindSourceStream) { if (!sourceStream.markSupported()) { throw new IllegalArgumentException(SR.INPUT_STREAM_SHOULD_BE_MARKABLE); } sourceStream.mark(Constants.MAX_MARK_LENGTH); } MessageDigest digest = null; if (calculateMD5) { try { digest = MessageDigest.getInstance("MD5"); } catch (final NoSuchAlgorithmException e) { // This wont happen, throw fatal. throw Utility.generateNewUnexpectedStorageException(e); } } if (writeLength < 0) { writeLength = Long.MAX_VALUE; } final StreamMd5AndLength retVal = new StreamMd5AndLength(); int count = -1; final byte[] retrievedBuff = new byte[Constants.BUFFER_COPY_LENGTH]; int nextCopy = (int) Math.min(retrievedBuff.length, writeLength - retVal.getLength()); count = sourceStream.read(retrievedBuff, 0, nextCopy); while (nextCopy > 0 && count != -1) { if (calculateMD5) { digest.update(retrievedBuff, 0, count); } retVal.setLength(retVal.getLength() + count); if (retVal.getLength() > abandonLength) { // Abandon operation retVal.setLength(-1); retVal.setMd5(null); break; } nextCopy = (int) Math.min(retrievedBuff.length, writeLength - retVal.getLength()); count = sourceStream.read(retrievedBuff, 0, nextCopy); } if (retVal.getLength() != -1 && calculateMD5) { retVal.setMd5(Base64.encode(digest.digest())); } if (retVal.getLength() != -1 && writeLength > 0) { retVal.setLength(Math.min(retVal.getLength(), writeLength)); } if (rewindSourceStream) { sourceStream.reset(); sourceStream.mark(Constants.MAX_MARK_LENGTH); } return retVal; }