List of usage examples for org.apache.commons.io.output DeferredFileOutputStream isInMemory
public boolean isInMemory()
From source file:mitm.common.util.MiscIOUtils.java
/** * Creates an InputStream of the content of {@link DeferredFileOutputStream}. The output is closed before reading the * content. This is required when the {@link DeferredFileOutputStream} uses a file to make sure the file is * completely written to disk.//from w ww .j a va 2s . c o m * * Note: If {@link DeferredFileOutputStream} uses a file, the file will be automatically deleted when #close is * called for the InputStream. The caller is responsible for calling close. */ @SuppressWarnings("resource") public static InputStream toInputStream(DeferredFileOutputStream output) throws IOException { Check.notNull(output, "output"); IOUtils.closeQuietly(output); return output.isInMemory() ? new ByteArrayInputStream(output.getData()) : new BufferedInputStream(new DeleteFileOnCloseFileInputStream(output.getFile())); }
From source file:de.mendelson.comm.as2.message.AS2MessageParser.java
/**Decrypts the data of a message with all given certificates etc * @param info MessageInfo, the encryption algorith will be stored in the encryption type of this info * @param rawMessageData encrypted data, will be decrypted * @param contentType contentType of the data * @param privateKey receivers private key * @param certificate receivers certificate */// w w w . ja v a2 s . c om public byte[] decryptData(AS2Message message, byte[] data, String contentType, PrivateKey privateKeyReceiver, X509Certificate certificateReceiver, String receiverCryptAlias) throws Exception { AS2MessageInfo info = (AS2MessageInfo) message.getAS2Info(); MimeBodyPart encryptedBody = new MimeBodyPart(); encryptedBody.setHeader("content-type", contentType); encryptedBody.setDataHandler(new DataHandler(new ByteArrayDataSource(data, contentType))); RecipientId recipientId = new JceKeyTransRecipientId(certificateReceiver); SMIMEEnveloped enveloped = new SMIMEEnveloped(encryptedBody); BCCryptoHelper helper = new BCCryptoHelper(); String algorithm = helper.convertOIDToAlgorithmName(enveloped.getEncryptionAlgOID()); if (algorithm.equals(BCCryptoHelper.ALGORITHM_3DES)) { info.setEncryptionType(AS2Message.ENCRYPTION_3DES); } else if (algorithm.equals(BCCryptoHelper.ALGORITHM_DES)) { info.setEncryptionType(AS2Message.ENCRYPTION_DES); } else if (algorithm.equals(BCCryptoHelper.ALGORITHM_RC2)) { info.setEncryptionType(AS2Message.ENCRYPTION_RC2_UNKNOWN); } else if (algorithm.equals(BCCryptoHelper.ALGORITHM_AES_128)) { info.setEncryptionType(AS2Message.ENCRYPTION_AES_128); } else if (algorithm.equals(BCCryptoHelper.ALGORITHM_AES_192)) { info.setEncryptionType(AS2Message.ENCRYPTION_AES_192); } else if (algorithm.equals(BCCryptoHelper.ALGORITHM_AES_256)) { info.setEncryptionType(AS2Message.ENCRYPTION_AES_256); } else if (algorithm.equals(BCCryptoHelper.ALGORITHM_RC4)) { info.setEncryptionType(AS2Message.ENCRYPTION_RC4_UNKNOWN); } else { info.setEncryptionType(AS2Message.ENCRYPTION_UNKNOWN_ALGORITHM); } RecipientInformationStore recipients = enveloped.getRecipientInfos(); enveloped = null; encryptedBody = null; RecipientInformation recipient = recipients.get(recipientId); if (recipient == null) { //give some details about the required and used cert for the decryption Collection recipientList = recipients.getRecipients(); Iterator iterator = recipientList.iterator(); while (iterator.hasNext()) { RecipientInformation recipientInfo = (RecipientInformation) iterator.next(); if (this.logger != null) { this.logger.log(Level.SEVERE, this.rb.getResourceString("decryption.inforequired", new Object[] { info.getMessageId(), recipientInfo.getRID() }), info); } } if (this.logger != null) { this.logger.log(Level.SEVERE, this.rb.getResourceString("decryption.infoassigned", new Object[] { info.getMessageId(), receiverCryptAlias, recipientId }), info); } throw new AS2Exception(AS2Exception.AUTHENTIFICATION_ERROR, "Error decrypting the message: Recipient certificate does not match.", message); } //Streamed decryption. Its also possible to use in memory decryption using getContent but that uses //far more memory. InputStream contentStream = recipient .getContentStream(new JceKeyTransEnvelopedRecipient(privateKeyReceiver).setProvider("BC")) .getContentStream(); //InputStream contentStream = recipient.getContentStream(privateKeyReceiver, "BC").getContentStream(); //threshold set to 20 MB: if the data is less then 20MB perform the operaion in memory else stream to disk DeferredFileOutputStream decryptedOutput = new DeferredFileOutputStream(20 * 1024 * 1024, "as2decryptdata_", ".mem", null); this.copyStreams(contentStream, decryptedOutput); decryptedOutput.flush(); decryptedOutput.close(); contentStream.close(); byte[] decryptedData = null; //size of the data was < than the threshold if (decryptedOutput.isInMemory()) { decryptedData = decryptedOutput.getData(); } else { //data has been written to a temp file: reread and return ByteArrayOutputStream memOut = new ByteArrayOutputStream(); decryptedOutput.writeTo(memOut); memOut.flush(); memOut.close(); //finally delete the temp file boolean deleted = decryptedOutput.getFile().delete(); decryptedData = memOut.toByteArray(); } if (this.logger != null) { this.logger.log(Level.INFO, this.rb.getResourceString("decryption.done.alias", new Object[] { info.getMessageId(), receiverCryptAlias, this.rbMessage.getResourceString("encryption." + info.getEncryptionType()) }), info); } return (decryptedData); }
From source file:hudson.scm.CvsTagsParamDefinition.java
@Exported public ListBoxModel getSymbolicNames() { ListBoxModel model = new ListBoxModel(); CvsChangeSet changeSet = null;/*from w ww. ja v a 2s . co m*/ RlogCommand statusCommand = new RlogCommand(); statusCommand.setHeaderOnly(true); statusCommand.setModule(moduleName); statusCommand.setRecursive(true); try { final File tempRlogSpill = File.createTempFile("cvs", "status ); ` "); final DeferredFileOutputStream outputStream = new DeferredFileOutputStream(100 * 1024, tempRlogSpill); final PrintStream logStream = new PrintStream(outputStream, true, getCvsDescriptor().getChangelogEncoding()); final OutputStream errorOutputStream = new OutputStream() { final StringBuffer buffer = new StringBuffer(); @Override public void write(int b) throws IOException { if ((int) ("\n".getBytes()[0]) == b) { flush(); } else { buffer.append(new String(new byte[] { (byte) b })); } } @Override public void flush() throws IOException { logger.info(buffer.toString()); buffer.delete(0, buffer.length()); super.flush(); } public void close() throws IOException { flush(); super.close(); } }; final PrintStream errorPrintStream = new PrintStream(errorOutputStream); Client cvsClient = getCvsClient(cvsRoot, passwordRequired, password); cvsClient.getEventManager().addCVSListener(new BasicListener(logStream, errorPrintStream)); cvsClient.executeCommand(statusCommand, getGlobalOptions(cvsRoot)); logStream.close(); errorPrintStream.flush(); errorPrintStream.close(); CvsLog parser = new CvsLog() { @Override public Reader read() throws IOException { if (outputStream.isInMemory()) return new InputStreamReader(new ByteArrayInputStream(outputStream.getData()), getCvsDescriptor().getChangelogEncoding()); else return new InputStreamReader(new FileInputStream(outputStream.getFile()), getCvsDescriptor().getChangelogEncoding()); } @Override public void dispose() { tempRlogSpill.delete(); } }; changeSet = parser.mapCvsLog(cvsRoot, new CvsRepositoryLocation.HeadRepositoryLocation()); } catch (IOException ex) { model.add(new ListBoxModel.Option("Could not load symbolic names - " + ex.getLocalizedMessage())); return model; } catch (CommandAbortedException ex) { model.add(new ListBoxModel.Option("Could not load symbolic names - " + ex.getLocalizedMessage())); return model; } catch (CommandException ex) { model.add(new ListBoxModel.Option("Could not load symbolic names - " + ex.getLocalizedMessage())); return model; } catch (AuthenticationException ex) { model.add(new ListBoxModel.Option("Could not load symbolic names - " + ex.getLocalizedMessage())); return model; } model.add(new ListBoxModel.Option("Head", "HEAD")); for (String branchName : changeSet.getBranchNames()) { model.add(new ListBoxModel.Option(branchName + " (Branch)", branchName)); } for (String tagName : changeSet.getTagNames()) { model.add(new ListBoxModel.Option(tagName + " (Tag)", tagName)); } return model; }
From source file:hudson.scm.AbstractCvs.java
/** * Gets the output for the CVS <tt>rlog</tt> command for the given module * between the specified dates.//from w ww . java 2 s.co m * * @param repository * the repository to connect to for running rlog against * @param module * the module to check for changes against * @param listener * where to log any error messages to * @param startTime * don't list any changes before this time * @param endTime * don't list any changes after this time * @return the output of rlog with no modifications * @throws IOException * on underlying communication failure */ private CvsLog getRemoteLogForModule(final CvsRepository repository, final CvsRepositoryItem item, final CvsModule module, final Date startTime, final Date endTime, final EnvVars envVars, final TaskListener listener) throws IOException { final Client cvsClient = getCvsClient(repository, envVars, listener); RlogCommand rlogCommand = new RlogCommand(); // we have to synchronize since we're dealing with DateFormat.format() synchronized (DATE_FORMATTER) { final String lastBuildDate = DATE_FORMATTER.format(startTime); final String endDate = DATE_FORMATTER.format(endTime); rlogCommand.setDateFilter(lastBuildDate + "<" + endDate); } // tell CVS which module we're logging rlogCommand.setModule(envVars.expand(module.getRemoteName())); // ignore headers for files that aren't in the current change-set rlogCommand.setSuppressHeader(true); // create an output stream to send the output from CVS command to - we // can then parse it from here final File tmpRlogSpill = File.createTempFile("cvs", "rlog"); final DeferredFileOutputStream outputStream = new DeferredFileOutputStream(100 * 1024, tmpRlogSpill); final PrintStream logStream = new PrintStream(outputStream, true, getDescriptor().getChangelogEncoding()); // set a listener with our output stream that we parse the log from final CVSListener basicListener = new BasicListener(logStream, listener.getLogger()); cvsClient.getEventManager().addCVSListener(basicListener); // log the command to the current run/polling log listener.getLogger().println("cvs " + rlogCommand.getCVSCommand()); // send the command to be run, we can't continue of the task fails try { if (!cvsClient.executeCommand(rlogCommand, getGlobalOptions(repository, envVars))) { throw new RuntimeException("Error while trying to run CVS rlog"); } } catch (CommandAbortedException e) { throw new RuntimeException("CVS rlog command aborted", e); } catch (CommandException e) { throw new RuntimeException("CVS rlog command failed", e); } catch (AuthenticationException e) { throw new RuntimeException("CVS authentication failure while running rlog command", e); } finally { try { cvsClient.getConnection().close(); } catch (IOException ex) { listener.getLogger().println("Could not close client connection: " + ex.getMessage()); } } // flush the output so we have it all available for parsing logStream.close(); // return the contents of the stream as the output of the command return new CvsLog() { @Override public Reader read() throws IOException { // note that master and slave can have different platform encoding if (outputStream.isInMemory()) return new InputStreamReader(new ByteArrayInputStream(outputStream.getData()), getDescriptor().getChangelogEncoding()); else return new InputStreamReader(new FileInputStream(outputStream.getFile()), getDescriptor().getChangelogEncoding()); } @Override public void dispose() { tmpRlogSpill.delete(); } }; }
From source file:de.mendelson.comm.as2.message.AS2MessageCreation.java
/**Encrypts a byte array and returns it*/ private void encryptDataToMessage(AS2Message message, String receiverCryptAlias, int encryptionType, Partner receiver) throws Exception { AS2MessageInfo info = (AS2MessageInfo) message.getAS2Info(); BCCryptoHelper cryptoHelper = new BCCryptoHelper(); X509Certificate certificate = this.encryptionCertManager.getX509Certificate(receiverCryptAlias); CMSEnvelopedDataStreamGenerator dataGenerator = new CMSEnvelopedDataStreamGenerator(); dataGenerator.addKeyTransRecipient(certificate); DeferredFileOutputStream encryptedOutput = new DeferredFileOutputStream(1024 * 1024, "as2encryptdata_", ".mem", null); OutputStream out = null;// w w w . j a v a 2 s. c o m if (encryptionType == AS2Message.ENCRYPTION_3DES) { out = dataGenerator.open(encryptedOutput, CMSEnvelopedDataGenerator.DES_EDE3_CBC, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_DES) { out = dataGenerator.open(encryptedOutput, cryptoHelper.convertAlgorithmNameToOID(BCCryptoHelper.ALGORITHM_DES), 56, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_RC2_40) { out = dataGenerator.open(encryptedOutput, CMSEnvelopedDataGenerator.RC2_CBC, 40, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_RC2_64) { out = dataGenerator.open(encryptedOutput, CMSEnvelopedDataGenerator.RC2_CBC, 64, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_RC2_128) { out = dataGenerator.open(encryptedOutput, CMSEnvelopedDataGenerator.RC2_CBC, 128, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_RC2_196) { out = dataGenerator.open(encryptedOutput, CMSEnvelopedDataGenerator.RC2_CBC, 196, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_AES_128) { out = dataGenerator.open(encryptedOutput, CMSEnvelopedDataGenerator.AES128_CBC, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_AES_192) { out = dataGenerator.open(encryptedOutput, CMSEnvelopedDataGenerator.AES192_CBC, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_AES_256) { out = dataGenerator.open(encryptedOutput, CMSEnvelopedDataGenerator.AES256_CBC, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_RC4_40) { out = dataGenerator.open(encryptedOutput, cryptoHelper.convertAlgorithmNameToOID(BCCryptoHelper.ALGORITHM_RC4), 40, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_RC4_56) { out = dataGenerator.open(encryptedOutput, cryptoHelper.convertAlgorithmNameToOID(BCCryptoHelper.ALGORITHM_RC4), 56, "BC"); } else if (encryptionType == AS2Message.ENCRYPTION_RC4_128) { out = dataGenerator.open(encryptedOutput, cryptoHelper.convertAlgorithmNameToOID(BCCryptoHelper.ALGORITHM_RC4), 128, "BC"); } if (out == null) { throw new Exception("Internal failure: unsupported encryption type " + encryptionType); } out.write(message.getDecryptedRawData()); out.close(); encryptedOutput.close(); //size of the data was < than the threshold if (encryptedOutput.isInMemory()) { message.setRawData(encryptedOutput.getData()); } else { //data has been written to a temp file: reread and return ByteArrayOutputStream memOut = new ByteArrayOutputStream(); encryptedOutput.writeTo(memOut); memOut.flush(); memOut.close(); //finally delete the temp file boolean deleted = encryptedOutput.getFile().delete(); message.setRawData(memOut.toByteArray()); } if (this.logger != null) { String cryptAlias = this.encryptionCertManager .getAliasByFingerprint(receiver.getCryptFingerprintSHA1()); this.logger.log(Level.INFO, this.rb.getResourceString("message.encrypted", new Object[] { info.getMessageId(), cryptAlias, this.rbMessage.getResourceString("encryption." + receiver.getEncryptionType()) }), info); } }
From source file:org.apache.phoenix.iterate.SpoolingResultIterator.java
/** * Create a result iterator by iterating through the results of a scan, spooling them to disk once * a threshold has been reached. The scanner passed in is closed prior to returning. * @param scanner the results of a table scan * @param mm memory manager tracking memory usage across threads. * @param thresholdBytes the requested threshold. Will be dialed down if memory usage (as determined by * the memory manager) is exceeded./* www. j av a 2 s . c o m*/ * @throws SQLException */ SpoolingResultIterator(SpoolingMetricsHolder sMetrics, MemoryMetricsHolder mMetrics, ResultIterator scanner, MemoryManager mm, final int thresholdBytes, final long maxSpoolToDisk, final String spoolDirectory) throws SQLException { this.spoolMetrics = sMetrics; this.memoryMetrics = mMetrics; boolean success = false; long startTime = System.currentTimeMillis(); final MemoryChunk chunk = mm.allocate(0, thresholdBytes); long waitTime = System.currentTimeMillis() - startTime; GLOBAL_MEMORY_WAIT_TIME.update(waitTime); memoryMetrics.getMemoryWaitTimeMetric().change(waitTime); DeferredFileOutputStream spoolTo = null; try { // Can't be bigger than int, since it's the max of the above allocation int size = (int) chunk.getSize(); spoolTo = new DeferredFileOutputStream(size, "ResultSpooler", ".bin", new File(spoolDirectory)) { @Override protected void thresholdReached() throws IOException { try { super.thresholdReached(); } finally { chunk.close(); } } }; DataOutputStream out = new DataOutputStream(spoolTo); final long maxBytesAllowed = maxSpoolToDisk == -1 ? Long.MAX_VALUE : thresholdBytes + maxSpoolToDisk; long bytesWritten = 0L; for (Tuple result = scanner.next(); result != null; result = scanner.next()) { int length = TupleUtil.write(result, out); bytesWritten += length; if (bytesWritten > maxBytesAllowed) { throw new SpoolTooBigToDiskException("result too big, max allowed(bytes): " + maxBytesAllowed); } } if (spoolTo.isInMemory()) { byte[] data = spoolTo.getData(); chunk.resize(data.length); spoolFrom = new InMemoryResultIterator(data, chunk); GLOBAL_MEMORY_CHUNK_BYTES.update(data.length); memoryMetrics.getMemoryChunkSizeMetric().change(data.length); } else { long sizeOfSpoolFile = spoolTo.getFile().length(); GLOBAL_SPOOL_FILE_SIZE.update(sizeOfSpoolFile); GLOBAL_SPOOL_FILE_COUNTER.increment(); spoolMetrics.getNumSpoolFileMetric().increment(); spoolMetrics.getSpoolFileSizeMetric().change(sizeOfSpoolFile); spoolFrom = new OnDiskResultIterator(spoolTo.getFile()); if (spoolTo.getFile() != null) { spoolTo.getFile().deleteOnExit(); } } success = true; } catch (IOException e) { throw ServerUtil.parseServerException(e); } finally { try { scanner.close(); } finally { try { if (spoolTo != null) { if (!success && spoolTo.getFile() != null) { spoolTo.getFile().delete(); } spoolTo.close(); } } catch (IOException ignored) { // ignore close error } finally { if (!success) { chunk.close(); } } } } }
From source file:org.codelibs.fess.crawler.client.http.HcHttpClient.java
protected ResponseData processHttpMethod(final String url, final HttpUriRequest httpRequest) { try {/*from ww w . j ava 2s . c o m*/ processRobotsTxt(url); } catch (final CrawlingAccessException e) { if (logger.isInfoEnabled()) { final StringBuilder buf = new StringBuilder(100); buf.append(e.getMessage()); if (e.getCause() != null) { buf.append(e.getCause().getMessage()); } logger.info(buf.toString()); } else if (logger.isDebugEnabled()) { logger.debug("Crawling Access Exception at " + url, e); } } // request header for (final Header header : requestHeaderList) { httpRequest.addHeader(header); } ResponseData responseData = new ResponseData(); HttpEntity httpEntity = null; try { // get a content final HttpResponse response = executeHttpClient(httpRequest); httpEntity = response.getEntity(); final int httpStatusCode = response.getStatusLine().getStatusCode(); // redirect if (isRedirectHttpStatus(httpStatusCode)) { final Header locationHeader = response.getFirstHeader("location"); if (locationHeader == null) { logger.warn("Invalid redirect location at " + url); } else { responseData = new ResponseData(); responseData.setRedirectLocation(locationHeader.getValue()); return responseData; } } String contentType = null; final Header contentTypeHeader = response.getFirstHeader("Content-Type"); if (contentTypeHeader != null) { contentType = contentTypeHeader.getValue(); final int idx = contentType.indexOf(';'); if (idx > 0) { contentType = contentType.substring(0, idx); if (APPLICATION_OCTET_STREAM.equals(contentType)) { contentType = null; } } } long contentLength = 0; String contentEncoding = Constants.UTF_8; if (httpEntity == null) { responseData.setResponseBody(new byte[0]); if (contentType == null) { contentType = defaultMimeType; } } else { final InputStream responseBodyStream = httpEntity.getContent(); final File outputFile = File.createTempFile("crawler-HcHttpClient-", ".out"); DeferredFileOutputStream dfos = null; try { try { dfos = new DeferredFileOutputStream((int) maxCachedContentSize, outputFile); CopyUtil.copy(responseBodyStream, dfos); dfos.flush(); } finally { IOUtils.closeQuietly(dfos); } } catch (final Exception e) { if (!outputFile.delete()) { logger.warn("Could not delete " + outputFile.getAbsolutePath()); } throw e; } if (dfos.isInMemory()) { responseData.setResponseBody(dfos.getData()); contentLength = dfos.getData().length; if (!outputFile.delete()) { logger.warn("Could not delete " + outputFile.getAbsolutePath()); } if (contentType == null) { try (InputStream is = new ByteArrayInputStream(dfos.getData())) { contentType = mimeTypeHelper.getContentType(is, url); } catch (Exception e) { logger.debug("Failed to detect mime-type.", e); contentType = defaultMimeType; } } } else { responseData.setResponseBody(outputFile, true); contentLength = outputFile.length(); if (contentType == null) { try (InputStream is = new FileInputStream(outputFile)) { contentType = mimeTypeHelper.getContentType(is, url); } catch (Exception e) { logger.debug("Failed to detect mime-type.", e); contentType = defaultMimeType; } } } final Header contentEncodingHeader = httpEntity.getContentEncoding(); if (contentEncodingHeader != null) { contentEncoding = contentEncodingHeader.getValue(); } } // check file size if (contentLengthHelper != null) { final long maxLength = contentLengthHelper.getMaxLength(contentType); if (contentLength > maxLength) { throw new MaxLengthExceededException("The content length (" + contentLength + " byte) is over " + maxLength + " byte. The url is " + url); } } responseData.setUrl(url); responseData.setCharSet(contentEncoding); if (httpRequest instanceof HttpHead) { responseData.setMethod(Constants.HEAD_METHOD); } else { responseData.setMethod(Constants.GET_METHOD); } responseData.setHttpStatusCode(httpStatusCode); for (final Header header : response.getAllHeaders()) { responseData.addMetaData(header.getName(), header.getValue()); } responseData.setMimeType(contentType); final Header contentLengthHeader = response.getFirstHeader("Content-Length"); if (contentLengthHeader == null) { responseData.setContentLength(contentLength); } else { final String value = contentLengthHeader.getValue(); try { responseData.setContentLength(Long.parseLong(value)); } catch (final Exception e) { responseData.setContentLength(contentLength); } } checkMaxContentLength(responseData); final Header lastModifiedHeader = response.getFirstHeader("Last-Modified"); if (lastModifiedHeader != null) { final String value = lastModifiedHeader.getValue(); if (StringUtil.isNotBlank(value)) { final Date d = parseLastModified(value); if (d != null) { responseData.setLastModified(d); } } } return responseData; } catch (final UnknownHostException e) { closeResources(httpRequest, responseData); throw new CrawlingAccessException("Unknown host(" + e.getMessage() + "): " + url, e); } catch (final NoRouteToHostException e) { closeResources(httpRequest, responseData); throw new CrawlingAccessException("No route to host(" + e.getMessage() + "): " + url, e); } catch (final ConnectException e) { closeResources(httpRequest, responseData); throw new CrawlingAccessException("Connection time out(" + e.getMessage() + "): " + url, e); } catch (final SocketException e) { closeResources(httpRequest, responseData); throw new CrawlingAccessException("Socket exception(" + e.getMessage() + "): " + url, e); } catch (final IOException e) { closeResources(httpRequest, responseData); throw new CrawlingAccessException("I/O exception(" + e.getMessage() + "): " + url, e); } catch (final CrawlerSystemException e) { closeResources(httpRequest, responseData); throw e; } catch (final Exception e) { closeResources(httpRequest, responseData); throw new CrawlerSystemException("Failed to access " + url, e); } finally { EntityUtils.consumeQuietly(httpEntity); } }
From source file:org.codelibs.fess.crawler.extractor.impl.TikaExtractor.java
protected InputStream getContentStream(DeferredFileOutputStream dfos) throws IOException { if (dfos.isInMemory()) { return new ByteArrayInputStream(dfos.getData()); } else {/*from w w w . j a v a 2 s .c o m*/ return new BufferedInputStream(new FileInputStream(dfos.getFile())); } }
From source file:org.codelibs.robot.client.http.HcHttpClient.java
protected ResponseData processHttpMethod(final String url, final HttpUriRequest httpRequest) { try {/* ww w . ja v a 2 s .c om*/ processRobotsTxt(url); } catch (final RobotCrawlAccessException e) { if (logger.isInfoEnabled()) { final StringBuilder buf = new StringBuilder(); buf.append(e.getMessage()); if (e.getCause() != null) { buf.append(e.getCause().getMessage()); } logger.info(buf.toString()); } else if (logger.isDebugEnabled()) { logger.debug("Crawling Access Exception at " + url, e); } } // request header for (final Header header : requestHeaderList) { httpRequest.addHeader(header); } ResponseData responseData = null; InputStream inputStream = null; HttpEntity httpEntity = null; try { // get a content final HttpResponse response = executeHttpClient(httpRequest); httpEntity = response.getEntity(); final int httpStatusCode = response.getStatusLine().getStatusCode(); // redirect if (isRedirectHttpStatus(httpStatusCode)) { final Header locationHeader = response.getFirstHeader("location"); if (locationHeader == null) { logger.warn("Invalid redirect location at " + url); } else { responseData = new ResponseData(); responseData.setRedirectLocation(locationHeader.getValue()); return responseData; } } long contentLength = 0; String contentEncoding = Constants.UTF_8; if (httpEntity == null) { inputStream = new ByteArrayInputStream(new byte[0]); } else { final InputStream responseBodyStream = httpEntity.getContent(); final File outputFile = File.createTempFile("s2robot-HcHttpClient-", ".out"); DeferredFileOutputStream dfos = null; try { try { dfos = new DeferredFileOutputStream(responseBodyInMemoryThresholdSize, outputFile); CopyUtil.copy(responseBodyStream, dfos); dfos.flush(); } finally { IOUtils.closeQuietly(dfos); } } catch (final Exception e) { if (!outputFile.delete()) { logger.warn("Could not delete " + outputFile.getAbsolutePath()); } throw e; } if (dfos.isInMemory()) { inputStream = new ByteArrayInputStream(dfos.getData()); contentLength = dfos.getData().length; if (!outputFile.delete()) { logger.warn("Could not delete " + outputFile.getAbsolutePath()); } } else { inputStream = new TemporaryFileInputStream(outputFile); contentLength = outputFile.length(); } final Header contentEncodingHeader = httpEntity.getContentEncoding(); if (contentEncodingHeader != null) { contentEncoding = contentEncodingHeader.getValue(); } } String contentType = null; final Header contentTypeHeader = response.getFirstHeader("Content-Type"); if (contentTypeHeader != null) { contentType = contentTypeHeader.getValue(); final int idx = contentType.indexOf(';'); if (idx > 0) { contentType = contentType.substring(0, idx); } } // check file size if (contentLengthHelper != null) { final long maxLength = contentLengthHelper.getMaxLength(contentType); if (contentLength > maxLength) { throw new MaxLengthExceededException("The content length (" + contentLength + " byte) is over " + maxLength + " byte. The url is " + url); } } responseData = new ResponseData(); responseData.setUrl(url); responseData.setCharSet(contentEncoding); if (httpRequest instanceof HttpHead) { responseData.setMethod(Constants.HEAD_METHOD); } else { responseData.setMethod(Constants.GET_METHOD); } responseData.setResponseBody(inputStream); responseData.setHttpStatusCode(httpStatusCode); for (final Header header : response.getAllHeaders()) { responseData.addMetaData(header.getName(), header.getValue()); } if (contentType == null) { responseData.setMimeType(defaultMimeType); } else { responseData.setMimeType(contentType); } final Header contentLengthHeader = response.getFirstHeader("Content-Length"); if (contentLengthHeader == null) { responseData.setContentLength(contentLength); } else { final String value = contentLengthHeader.getValue(); try { responseData.setContentLength(Long.parseLong(value)); } catch (final Exception e) { responseData.setContentLength(contentLength); } } final Header lastModifiedHeader = response.getFirstHeader("Last-Modified"); if (lastModifiedHeader != null) { final String value = lastModifiedHeader.getValue(); if (StringUtil.isNotBlank(value)) { final Date d = parseLastModified(value); if (d != null) { responseData.setLastModified(d); } } } return responseData; } catch (final UnknownHostException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("Unknown host(" + e.getMessage() + "): " + url, e); } catch (final NoRouteToHostException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("No route to host(" + e.getMessage() + "): " + url, e); } catch (final ConnectException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("Connection time out(" + e.getMessage() + "): " + url, e); } catch (final SocketException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("Socket exception(" + e.getMessage() + "): " + url, e); } catch (final IOException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("I/O exception(" + e.getMessage() + "): " + url, e); } catch (final RobotSystemException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw e; } catch (final Exception e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotSystemException("Failed to access " + url, e); } finally { EntityUtils.consumeQuietly(httpEntity); } }
From source file:org.geoserver.wms.topojson.TopologyBuilder.java
@Override public RawMap build(WMSMapContent mapContent) throws IOException { Map<String, TopoGeom.GeometryColleciton> layers = new HashMap<>(); for (String layer : this.layers.keySet()) { Collection<TopoGeom> collection = this.layers.get(layer); GeometryColleciton layerCollection = new TopoGeom.GeometryColleciton(collection); layers.put(layer, layerCollection); }/* w w w . j a v a2s . co m*/ List<LineString> arcs = this.arcs; this.arcs = null; this.layers = null; Topology topology = new Topology(screenToWorld, arcs, layers); final int threshold = 8096; DeferredFileOutputStream out = new DeferredFileOutputStream(threshold, "topology", ".topojson", null); TopoJSONEncoder encoder = new TopoJSONEncoder(); Writer writer = new OutputStreamWriter(out, Charsets.UTF_8); encoder.encode(topology, writer); writer.flush(); writer.close(); out.close(); long length; RawMap map; if (out.isInMemory()) { byte[] data = out.getData(); length = data.length; map = new RawMap(mapContent, data, MIME_TYPE); } else { File f = out.getFile(); length = f.length(); map = new DeferredFileOutputStreamWebMap(mapContent, out, MIME_TYPE); } map.setResponseHeader("Content-Length", String.valueOf(length)); return map; }