List of usage examples for org.apache.http.util EntityUtils consumeQuietly
public static void consumeQuietly(HttpEntity httpEntity)
From source file:org.codelibs.robot.client.http.HcHttpClient.java
protected void processRobotsTxt(final String url) { if (StringUtil.isBlank(url)) { throw new RobotSystemException("url is null or empty."); }/*from ww w . j a v a2 s .com*/ if (robotsTxtHelper == null || !robotsTxtHelper.isEnabled()) { // not support robots.txt return; } // robot context final S2RobotContext robotContext = CrawlingParameterUtil.getRobotContext(); if (robotContext == null) { // wrong state return; } final int idx = url.indexOf('/', url.indexOf("://") + 3); String hostUrl; if (idx >= 0) { hostUrl = url.substring(0, idx); } else { hostUrl = url; } final String robotTxtUrl = hostUrl + "/robots.txt"; // check url if (robotContext.getRobotTxtUrlSet().contains(robotTxtUrl)) { if (logger.isDebugEnabled()) { logger.debug(robotTxtUrl + " is already visited."); } return; } if (logger.isInfoEnabled()) { logger.info("Checking URL: " + robotTxtUrl); } // add url to a set robotContext.getRobotTxtUrlSet().add(robotTxtUrl); final HttpGet httpGet = new HttpGet(robotTxtUrl); // request header for (final Header header : requestHeaderList) { httpGet.addHeader(header); } HttpEntity httpEntity = null; try { // get a content final HttpResponse response = executeHttpClient(httpGet); httpEntity = response.getEntity(); final int httpStatusCode = response.getStatusLine().getStatusCode(); if (httpStatusCode == 200) { // check file size final Header contentLengthHeader = response.getFirstHeader("Content-Length"); if (contentLengthHeader != null) { final String value = contentLengthHeader.getValue(); final long contentLength = Long.parseLong(value); if (contentLengthHelper != null) { final long maxLength = contentLengthHelper.getMaxLength("text/plain"); if (contentLength > maxLength) { throw new MaxLengthExceededException("The content length (" + contentLength + " byte) is over " + maxLength + " byte. The url is " + robotTxtUrl); } } } if (httpEntity != null) { final RobotsTxt robotsTxt = robotsTxtHelper.parse(httpEntity.getContent()); if (robotsTxt != null) { final String[] sitemaps = robotsTxt.getSitemaps(); if (sitemaps.length > 0) { robotContext.addSitemaps(sitemaps); } final RobotsTxt.Directive directive = robotsTxt.getMatchedDirective(userAgent); if (directive != null) { if (useRobotsTxtDisallows) { for (String urlPattern : directive.getDisallows()) { if (StringUtil.isNotBlank(urlPattern)) { urlPattern = convertRobotsTxtPathPattern(urlPattern); robotContext.getUrlFilter().addExclude(hostUrl + urlPattern); } } } if (useRobotsTxtAllows) { for (String urlPattern : directive.getAllows()) { if (StringUtil.isNotBlank(urlPattern)) { urlPattern = convertRobotsTxtPathPattern(urlPattern); robotContext.getUrlFilter().addInclude(hostUrl + urlPattern); } } } } } } } } catch (final RobotSystemException e) { httpGet.abort(); throw e; } catch (final Exception e) { httpGet.abort(); throw new RobotCrawlAccessException("Could not process " + robotTxtUrl + ". ", e); } finally { EntityUtils.consumeQuietly(httpEntity); } }
From source file:org.codelibs.robot.client.http.HcHttpClient.java
protected ResponseData processHttpMethod(final String url, final HttpUriRequest httpRequest) { try {/* ww w. j av a 2s .c o m*/ processRobotsTxt(url); } catch (final RobotCrawlAccessException e) { if (logger.isInfoEnabled()) { final StringBuilder buf = new StringBuilder(); buf.append(e.getMessage()); if (e.getCause() != null) { buf.append(e.getCause().getMessage()); } logger.info(buf.toString()); } else if (logger.isDebugEnabled()) { logger.debug("Crawling Access Exception at " + url, e); } } // request header for (final Header header : requestHeaderList) { httpRequest.addHeader(header); } ResponseData responseData = null; InputStream inputStream = null; HttpEntity httpEntity = null; try { // get a content final HttpResponse response = executeHttpClient(httpRequest); httpEntity = response.getEntity(); final int httpStatusCode = response.getStatusLine().getStatusCode(); // redirect if (isRedirectHttpStatus(httpStatusCode)) { final Header locationHeader = response.getFirstHeader("location"); if (locationHeader == null) { logger.warn("Invalid redirect location at " + url); } else { responseData = new ResponseData(); responseData.setRedirectLocation(locationHeader.getValue()); return responseData; } } long contentLength = 0; String contentEncoding = Constants.UTF_8; if (httpEntity == null) { inputStream = new ByteArrayInputStream(new byte[0]); } else { final InputStream responseBodyStream = httpEntity.getContent(); final File outputFile = File.createTempFile("s2robot-HcHttpClient-", ".out"); DeferredFileOutputStream dfos = null; try { try { dfos = new DeferredFileOutputStream(responseBodyInMemoryThresholdSize, outputFile); CopyUtil.copy(responseBodyStream, dfos); dfos.flush(); } finally { IOUtils.closeQuietly(dfos); } } catch (final Exception e) { if (!outputFile.delete()) { logger.warn("Could not delete " + outputFile.getAbsolutePath()); } throw e; } if (dfos.isInMemory()) { inputStream = new ByteArrayInputStream(dfos.getData()); contentLength = dfos.getData().length; if (!outputFile.delete()) { logger.warn("Could not delete " + outputFile.getAbsolutePath()); } } else { inputStream = new TemporaryFileInputStream(outputFile); contentLength = outputFile.length(); } final Header contentEncodingHeader = httpEntity.getContentEncoding(); if (contentEncodingHeader != null) { contentEncoding = contentEncodingHeader.getValue(); } } String contentType = null; final Header contentTypeHeader = response.getFirstHeader("Content-Type"); if (contentTypeHeader != null) { contentType = contentTypeHeader.getValue(); final int idx = contentType.indexOf(';'); if (idx > 0) { contentType = contentType.substring(0, idx); } } // check file size if (contentLengthHelper != null) { final long maxLength = contentLengthHelper.getMaxLength(contentType); if (contentLength > maxLength) { throw new MaxLengthExceededException("The content length (" + contentLength + " byte) is over " + maxLength + " byte. The url is " + url); } } responseData = new ResponseData(); responseData.setUrl(url); responseData.setCharSet(contentEncoding); if (httpRequest instanceof HttpHead) { responseData.setMethod(Constants.HEAD_METHOD); } else { responseData.setMethod(Constants.GET_METHOD); } responseData.setResponseBody(inputStream); responseData.setHttpStatusCode(httpStatusCode); for (final Header header : response.getAllHeaders()) { responseData.addMetaData(header.getName(), header.getValue()); } if (contentType == null) { responseData.setMimeType(defaultMimeType); } else { responseData.setMimeType(contentType); } final Header contentLengthHeader = response.getFirstHeader("Content-Length"); if (contentLengthHeader == null) { responseData.setContentLength(contentLength); } else { final String value = contentLengthHeader.getValue(); try { responseData.setContentLength(Long.parseLong(value)); } catch (final Exception e) { responseData.setContentLength(contentLength); } } final Header lastModifiedHeader = response.getFirstHeader("Last-Modified"); if (lastModifiedHeader != null) { final String value = lastModifiedHeader.getValue(); if (StringUtil.isNotBlank(value)) { final Date d = parseLastModified(value); if (d != null) { responseData.setLastModified(d); } } } return responseData; } catch (final UnknownHostException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("Unknown host(" + e.getMessage() + "): " + url, e); } catch (final NoRouteToHostException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("No route to host(" + e.getMessage() + "): " + url, e); } catch (final ConnectException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("Connection time out(" + e.getMessage() + "): " + url, e); } catch (final SocketException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("Socket exception(" + e.getMessage() + "): " + url, e); } catch (final IOException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotCrawlAccessException("I/O exception(" + e.getMessage() + "): " + url, e); } catch (final RobotSystemException e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw e; } catch (final Exception e) { httpRequest.abort(); IOUtils.closeQuietly(inputStream); throw new RobotSystemException("Failed to access " + url, e); } finally { EntityUtils.consumeQuietly(httpEntity); } }
From source file:org.commonjava.util.jhttpc.INTERNAL.util.HttpUtils.java
public static void cleanupResources(CloseableHttpClient client, Set<WeakReference<HttpRequest>> requests, Set<WeakReference<CloseableHttpResponse>> responses) { if (responses != null) { for (WeakReference<CloseableHttpResponse> ref : responses) { if (ref == null) { continue; }// www .jav a 2s .c o m CloseableHttpResponse response = ref.get(); if (response == null) { continue; } if (response.getEntity() != null) { EntityUtils.consumeQuietly(response.getEntity()); closeQuietly(response); } } } if (requests != null) { for (WeakReference<HttpRequest> ref : requests) { if (ref == null) { continue; } HttpRequest request = ref.get(); if (request == null) { continue; } if (request instanceof AbstractExecutionAwareRequest) { ((AbstractExecutionAwareRequest) request).reset(); } } } if (client != null) { closeQuietly(client); } }
From source file:org.kaaproject.kaa.sandbox.AbstractSandboxBuilder.java
private void downloadFile(HttpClient httpClient, HttpContext context, URL sourceUrl, File targetFile) throws Exception { LOG.debug("Download {} to file {}", sourceUrl.toString(), targetFile.getAbsolutePath()); HttpEntity entity = null;//from w w w. j av a2 s. c o m try { HttpGet httpGet = new HttpGet(sourceUrl.toURI()); HttpResponse response = httpClient.execute(httpGet, context); entity = response.getEntity(); long length = entity.getContentLength(); InputStream in = new BufferedInputStream(entity.getContent()); OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile)); copyLarge(in, out, new byte[DEFAULT_BUFFER_SIZE], length); IOUtils.closeQuietly(in); IOUtils.closeQuietly(out); } finally { EntityUtils.consumeQuietly(entity); } }
From source file:org.opennms.core.web.HttpClientWrapper.java
/** * Safely clean up after a response.//from w ww . java 2s. c o m */ public void close(final CloseableHttpResponse response) { if (response != null) { EntityUtils.consumeQuietly(response.getEntity()); IOUtils.closeQuietly(response); } }
From source file:org.sead.nds.repository.C3PRPubRequestFacade.java
public void sendStatus(String stage, String message) { String c3prServer = props.getProperty("c3pr.address"); try {//w w w . ja va 2s . c o m String statusUrl = c3prServer + "api/researchobjects/" + URLEncoder.encode(RO_ID, "UTF-8") + "/status"; log.debug("Posting status to: " + statusUrl); HttpPost postStatus = new HttpPost(statusUrl); postStatus.addHeader("accept", MediaType.APPLICATION_JSON); String statusString = "{\"reporter\":\"" + Repository.getID() + "\", \"stage\":\"" + stage + "\", \"message\":\"" + message + "\"}"; StringEntity status = new StringEntity(statusString); log.trace("Status: " + statusString); postStatus.addHeader("content-type", MediaType.APPLICATION_JSON); postStatus.setEntity(status); CloseableHttpResponse response = client.execute(postStatus, getLocalContext()); if (response.getStatusLine().getStatusCode() == 200) { log.debug("Status Successfully posted"); } else { log.warn("Failed to post status, response code: " + response.getStatusLine().getStatusCode()); } // Must consume entity to allow connection to be released // If this line is not here, the third try to send status will // result in a // org.apache.http.conn.ConnectionPoolTimeoutException: Timeout // waiting for connection from pool // (or a blocked call/hung program if timeouts weren't set EntityUtils.consumeQuietly(response.getEntity()); } catch (UnsupportedEncodingException e) { // TODO Auto-generated catch block log.error("Error posting status.", e); e.printStackTrace(); } catch (ClientProtocolException e) { // TODO Auto-generated catch block log.error("Error posting status.", e); e.printStackTrace(); } catch (IOException e) { log.error("Error posting status.", e); e.printStackTrace(); } catch (Exception e) { log.error("Odd Error posting status.", e); e.printStackTrace(); } if (echoToConsole) { System.out.println("*********************Status Message******************************"); System.out.println("Reporter: " + Repository.getID() + ", Stage: " + stage); System.out.println("Message Text: " + message); System.out.println("*****************************************************************"); } }
From source file:org.wso2.carbon.analytics.api.internal.client.AnalyticsAPIHttpClient.java
private String getResponseString(HttpResponse httpResponse) throws AnalyticsServiceException { BufferedReader br = null;// w w w . j a va 2 s.co m try { br = new BufferedReader(new InputStreamReader(httpResponse.getEntity().getContent())); String readLine; String response = ""; while (((readLine = br.readLine()) != null)) { response += readLine; } return response; } catch (IOException e) { throw new AnalyticsServiceException( "Error while reading the response from the remote service. " + e.getMessage(), e); } finally { EntityUtils.consumeQuietly(httpResponse.getEntity()); if (br != null) { try { br.close(); } catch (IOException e) { log.warn("Error while closing the connection! " + e.getMessage()); } } } }
From source file:org.wso2.carbon.analytics.api.internal.client.AnalyticsAPIHttpClient.java
public AnalyticsSchema getTableSchema(int tenantId, String username, String tableName, boolean securityEnabled) throws AnalyticsServiceException { URIBuilder builder = new URIBuilder(); builder.setScheme(protocol).setHost(hostname).setPort(port) .setPath(AnalyticsAPIConstants.SCHEMA_PROCESSOR_SERVICE_URI) .addParameter(AnalyticsAPIConstants.OPERATION, AnalyticsAPIConstants.GET_SCHEMA_OPERATION) .addParameter(AnalyticsAPIConstants.TABLE_NAME_PARAM, tableName) .addParameter(AnalyticsAPIConstants.ENABLE_SECURITY_PARAM, String.valueOf(securityEnabled)); if (!securityEnabled) { builder.addParameter(AnalyticsAPIConstants.TENANT_ID_PARAM, String.valueOf(tenantId)); } else {//w w w . j ava2s . com builder.addParameter(AnalyticsAPIConstants.USERNAME_PARAM, username); } try { HttpGet getMethod = new HttpGet(builder.build().toString()); getMethod.addHeader(AnalyticsAPIConstants.SESSION_ID, sessionId); HttpResponse httpResponse = httpClient.execute(getMethod); if (httpResponse.getStatusLine().getStatusCode() != HttpServletResponse.SC_OK) { String response = getResponseString(httpResponse); throw new AnalyticsServiceException("Unable to get the schema for the table - " + tableName + " for tenant id : " + tenantId + ". " + response); } else { Object analyticsSchemaObject = GenericUtils .deserializeObject(httpResponse.getEntity().getContent()); EntityUtils.consumeQuietly(httpResponse.getEntity()); if (analyticsSchemaObject != null && analyticsSchemaObject instanceof AnalyticsSchema) { return (AnalyticsSchema) analyticsSchemaObject; } else { throw new AnalyticsServiceException( getUnexpectedResponseReturnedErrorMsg("getting the table schema", tableName, "analytics schema object ", analyticsSchemaObject)); } } } catch (URISyntaxException e) { throw new AnalyticsServiceAuthenticationException("Malformed URL provided. " + e.getMessage(), e); } catch (IOException e) { throw new AnalyticsServiceAuthenticationException( "Error while connecting to the remote service. " + e.getMessage(), e); } }
From source file:org.wso2.carbon.analytics.api.internal.client.AnalyticsAPIHttpClient.java
@SuppressWarnings("unchecked") public List<String> listTables(int tenantId, String username, boolean securityEnabled) throws AnalyticsServiceException { URIBuilder builder = new URIBuilder(); builder.setScheme(protocol).setHost(hostname).setPort(port) .setPath(AnalyticsAPIConstants.TABLE_PROCESSOR_SERVICE_URI) .addParameter(AnalyticsAPIConstants.OPERATION, AnalyticsAPIConstants.LIST_TABLES_OPERATION) .addParameter(AnalyticsAPIConstants.ENABLE_SECURITY_PARAM, String.valueOf(securityEnabled)); if (!securityEnabled) { builder.addParameter(AnalyticsAPIConstants.TENANT_ID_PARAM, String.valueOf(tenantId)); } else {/*from w w w . j a v a 2s . co m*/ builder.addParameter(AnalyticsAPIConstants.USERNAME_PARAM, username); } try { HttpGet getMethod = new HttpGet(builder.build().toString()); getMethod.addHeader(AnalyticsAPIConstants.SESSION_ID, sessionId); HttpResponse httpResponse = httpClient.execute(getMethod); if (httpResponse.getStatusLine().getStatusCode() != HttpServletResponse.SC_OK) { String response = getResponseString(httpResponse); throw new AnalyticsServiceException( "Unable to get the list of tables for tenant id : " + tenantId + ". " + response); } else { Object listOfTablesObj = GenericUtils.deserializeObject(httpResponse.getEntity().getContent()); EntityUtils.consumeQuietly(httpResponse.getEntity()); if (listOfTablesObj != null && listOfTablesObj instanceof List) { return (List<String>) listOfTablesObj; } else { throw new AnalyticsServiceException(getUnexpectedResponseReturnedErrorMsg( "getting list of tables", null, "list of tables", listOfTablesObj)); } } } catch (URISyntaxException e) { throw new AnalyticsServiceAuthenticationException("Malformed URL provided. " + e.getMessage(), e); } catch (IOException e) { throw new AnalyticsServiceAuthenticationException( "Error while connecting to the remote service. " + e.getMessage(), e); } }
From source file:org.wso2.carbon.analytics.api.internal.client.AnalyticsAPIHttpClient.java
@SuppressWarnings("unchecked") public void putRecords(String username, List<Record> records, boolean securityEnabled) throws AnalyticsServiceException { URIBuilder builder = new URIBuilder(); builder.setScheme(protocol).setHost(hostname).setPort(port) .setPath(AnalyticsAPIConstants.RECORD_PROCESSOR_SERVICE_URI) .addParameter(AnalyticsAPIConstants.OPERATION, AnalyticsAPIConstants.PUT_RECORD_OPERATION) .addParameter(AnalyticsAPIConstants.ENABLE_SECURITY_PARAM, String.valueOf(securityEnabled)); if (securityEnabled) { builder.addParameter(AnalyticsAPIConstants.USERNAME_PARAM, username); }//from w w w .j a v a 2 s . co m try { HttpPost postMethod = new HttpPost(builder.build().toString()); postMethod.addHeader(AnalyticsAPIConstants.SESSION_ID, sessionId); postMethod.setEntity(new ByteArrayEntity(GenericUtils.serializeObject(records))); HttpResponse httpResponse = httpClient.execute(postMethod); if (httpResponse.getStatusLine().getStatusCode() != HttpServletResponse.SC_OK) { String response = getResponseString(httpResponse); throw new AnalyticsServiceException("Unable to put the records. " + response); } else { Object recordIdsObj = GenericUtils.deserializeObject(httpResponse.getEntity().getContent()); EntityUtils.consumeQuietly(httpResponse.getEntity()); if (recordIdsObj != null && recordIdsObj instanceof List) { List<String> recordIds = (List<String>) recordIdsObj; int index = 0; for (Record record : records) { record.setId(recordIds.get(index)); index++; } } else { throw new AnalyticsServiceException(getUnexpectedResponseReturnedErrorMsg("putting the records", null, "list of strings", recordIdsObj)); } } } catch (URISyntaxException e) { throw new AnalyticsServiceAuthenticationException("Malformed URL provided. " + e.getMessage(), e); } catch (IOException e) { throw new AnalyticsServiceAuthenticationException( "Error while connecting to the remote service. " + e.getMessage(), e); } }