List of usage examples for org.apache.http.client.methods HttpRequestBase setURI
public void setURI(final URI uri)
From source file:io.apiman.test.common.util.TestPlanRunner.java
/** * Runs a single REST test.//from w ww.j av a 2 s .c om * * @param restTest */ private void runTest(RestTest restTest) throws Error { try { String requestPath = TestUtil.doPropertyReplacement(restTest.getRequestPath()); URI uri = getUri(requestPath); HttpRequestBase request = null; if (restTest.getRequestMethod().equalsIgnoreCase("GET")) { //$NON-NLS-1$ request = new HttpGet(); } else if (restTest.getRequestMethod().equalsIgnoreCase("POST")) { //$NON-NLS-1$ request = new HttpPost(); HttpEntity entity = new StringEntity(restTest.getRequestPayload()); ((HttpPost) request).setEntity(entity); } else if (restTest.getRequestMethod().equalsIgnoreCase("PUT")) { //$NON-NLS-1$ request = new HttpPut(); HttpEntity entity = new StringEntity(restTest.getRequestPayload()); ((HttpPut) request).setEntity(entity); } else if (restTest.getRequestMethod().equalsIgnoreCase("DELETE")) { //$NON-NLS-1$ request = new HttpDelete(); } if (request == null) { Assert.fail("Unsupported method in REST Test: " + restTest.getRequestMethod()); //$NON-NLS-1$ } request.setURI(uri); Map<String, String> requestHeaders = restTest.getRequestHeaders(); for (Entry<String, String> entry : requestHeaders.entrySet()) { request.setHeader(entry.getKey(), entry.getValue()); } // Set up basic auth String authorization = createBasicAuthorization(restTest.getUsername(), restTest.getPassword()); if (authorization != null) { request.setHeader("Authorization", authorization); //$NON-NLS-1$ } HttpResponse response = client.execute(request); assertResponse(restTest, response); } catch (Error e) { logPlain("[ERROR] " + e.getMessage()); //$NON-NLS-1$ throw e; } catch (Exception e) { throw new Error(e); } }
From source file:groovyx.net.http.HTTPBuilder.java
/** * All <code>request</code> methods delegate to this method. *//*from w w w. j a va 2 s. com*/ protected Object doRequest(final RequestConfigDelegate delegate) throws ClientProtocolException, IOException { final HttpRequestBase reqMethod = delegate.getRequest(); Object contentType = delegate.getContentType(); if (this.autoAcceptHeader) { String acceptContentTypes = contentType.toString(); if (contentType instanceof ContentType) acceptContentTypes = ((ContentType) contentType).getAcceptHeader(); reqMethod.setHeader("Accept", acceptContentTypes); } reqMethod.setURI(delegate.getUri().toURI()); if (reqMethod.getURI() == null) throw new IllegalStateException("Request URI cannot be null"); log.debug(reqMethod.getMethod() + " " + reqMethod.getURI()); // set any request headers from the delegate Map<?, ?> headers = delegate.getHeaders(); for (Object key : headers.keySet()) { Object val = headers.get(key); if (key == null) continue; if (val == null) reqMethod.removeHeaders(key.toString()); else reqMethod.setHeader(key.toString(), val.toString()); } HttpResponseDecorator resp = new HttpResponseDecorator(client.execute(reqMethod, delegate.getContext()), delegate.getContext(), null); try { int status = resp.getStatusLine().getStatusCode(); Closure responseClosure = delegate.findResponseHandler(status); log.debug("Response code: " + status + "; found handler: " + responseClosure); Object[] closureArgs = null; switch (responseClosure.getMaximumNumberOfParameters()) { case 1: closureArgs = new Object[] { resp }; break; case 2: // parse the response entity if the response handler expects it: HttpEntity entity = resp.getEntity(); try { if (entity == null || entity.getContentLength() == 0) closureArgs = new Object[] { resp, null }; else closureArgs = new Object[] { resp, parseResponse(resp, contentType) }; } catch (Exception ex) { Header h = entity.getContentType(); String respContentType = h != null ? h.getValue() : null; log.warn("Error parsing '" + respContentType + "' response", ex); throw new ResponseParseException(resp, ex); } break; default: throw new IllegalArgumentException("Response closure must accept one or two parameters"); } Object returnVal = responseClosure.call(closureArgs); log.trace("response handler result: " + returnVal); return returnVal; } finally { HttpEntity entity = resp.getEntity(); if (entity != null) entity.consumeContent(); } }
From source file:com.amazon.s3.http.AmazonHttpClient.java
/** * Internal method to execute the HTTP method given. * /*from ww w .j av a 2 s . c om*/ * @see AmazonHttpClient#execute(Request, HttpResponseHandler, * HttpResponseHandler) * @see AmazonHttpClient#execute(Request, HttpResponseHandler, * HttpResponseHandler, ExecutionContext) */ private <T extends Object> T executeHelper(Request<?> request, HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler, HttpResponseHandler<AmazonServiceException> errorResponseHandler, ExecutionContext executionContext) throws AmazonClientException, AmazonServiceException { /* * Depending on which response handler we end up choosing to handle the * HTTP response, it might require us to leave the underlying HTTP * connection open, depending on whether or not it reads the complete * HTTP response stream from the HTTP connection, or if delays reading * any of the content until after a response is returned to the caller. */ boolean leaveHttpConnectionOpen = false; AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics(); /* * add the service endpoint to the logs. You can infer service name from * service endpoint */ awsRequestMetrics.addProperty(Field.ServiceName.name(), request.getServiceName()); awsRequestMetrics.addProperty(Field.ServiceEndpoint.name(), request.getEndpoint()); // Apply whatever request options we know how to handle, such as // user-agent. applyRequestData(request); int retryCount = 0; URI redirectedURI = null; HttpEntity entity = null; AmazonServiceException exception = null; // Make a copy of the original request params and headers so that we can // permute it in this loop and start over with the original every time. Map<String, String> originalParameters = new HashMap<String, String>(); originalParameters.putAll(request.getParameters()); Map<String, String> originalHeaders = new HashMap<String, String>(); originalHeaders.putAll(request.getHeaders()); while (true) { awsRequestMetrics.setCounter(Field.AttemptCount.name(), retryCount + 1); if (retryCount > 0) { request.setParameters(originalParameters); request.setHeaders(originalHeaders); } HttpRequestBase httpRequest = null; org.apache.http.HttpResponse response = null; try { // Sign the request if a signer was provided if (executionContext.getSigner() != null && executionContext.getCredentials() != null) { awsRequestMetrics.startEvent(Field.RequestSigningTime.name()); executionContext.getSigner().sign(request, executionContext.getCredentials()); awsRequestMetrics.endEvent(Field.RequestSigningTime.name()); } Log.d(TAG, "Sending Request: " + request.toString()); httpRequest = httpRequestFactory.createHttpRequest(request, config, entity, executionContext); if (httpRequest instanceof HttpEntityEnclosingRequest) { entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity(); } if (redirectedURI != null) { httpRequest.setURI(redirectedURI); } if (retryCount > 0) { awsRequestMetrics.startEvent(Field.RetryPauseTime.name()); pauseExponentially(retryCount, exception, executionContext.getCustomBackoffStrategy()); awsRequestMetrics.endEvent(Field.RetryPauseTime.name()); } if (entity != null) { InputStream content = entity.getContent(); if (retryCount > 0) { if (content.markSupported()) { content.reset(); content.mark(-1); } } else { if (content.markSupported()) { content.mark(-1); } } } exception = null; awsRequestMetrics.startEvent(Field.HttpRequestTime.name()); response = httpClient.execute(httpRequest); awsRequestMetrics.endEvent(Field.HttpRequestTime.name()); if (isRequestSuccessful(response)) { awsRequestMetrics.addProperty(Field.StatusCode.name(), response.getStatusLine().getStatusCode()); /* * If we get back any 2xx status code, then we know we * should treat the service call as successful. */ leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen(); return handleResponse(request, responseHandler, httpRequest, response, executionContext); } else if (isTemporaryRedirect(response)) { /* * S3 sends 307 Temporary Redirects if you try to delete an * EU bucket from the US endpoint. If we get a 307, we'll * point the HTTP method to the redirected location, and let * the next retry deliver the request to the right location. */ Header[] locationHeaders = response.getHeaders("location"); String redirectedLocation = locationHeaders[0].getValue(); Log.d(TAG, "Redirecting to: " + redirectedLocation); redirectedURI = URI.create(redirectedLocation); httpRequest.setURI(redirectedURI); awsRequestMetrics.addProperty(Field.StatusCode.name(), response.getStatusLine().getStatusCode()); awsRequestMetrics.addProperty(Field.RedirectLocation.name(), redirectedLocation); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null); } else { leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen(); exception = handleErrorResponse(request, errorResponseHandler, httpRequest, response); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), exception.getRequestId()); awsRequestMetrics.addProperty(Field.AWSErrorCode.name(), exception.getErrorCode()); awsRequestMetrics.addProperty(Field.StatusCode.name(), exception.getStatusCode()); if (!shouldRetry(httpRequest, exception, retryCount)) { throw exception; } resetRequestAfterError(request, exception); } } catch (IOException ioe) { Log.i(TAG, "Unable to execute HTTP request: " + ioe.getMessage(), ioe); awsRequestMetrics.addProperty(Field.Exception.name(), ioe.toString()); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null); if (!shouldRetry(httpRequest, ioe, retryCount)) { throw new AmazonClientException("Unable to execute HTTP request: " + ioe.getMessage(), ioe); } resetRequestAfterError(request, ioe); } finally { retryCount++; /* * Some response handlers need to manually manage the HTTP * connection and will take care of releasing the connection on * their own, but if this response handler doesn't need the * connection left open, we go ahead and release the it to free * up resources. */ if (!leaveHttpConnectionOpen) { try { response.getEntity().getContent().close(); } catch (Throwable t) { } } } } /* end while (true) */ }
From source file:com.gargoylesoftware.htmlunit.HttpWebConnection.java
/** * Creates an <tt>HttpMethod</tt> instance according to the specified parameters. * @param webRequest the request/*from w w w. java 2s .co m*/ * @param httpClientBuilder the httpClientBuilder that will be configured * @return the <tt>HttpMethod</tt> instance constructed according to the specified parameters * @throws IOException * @throws URISyntaxException */ @SuppressWarnings("deprecation") private HttpUriRequest makeHttpMethod(final WebRequest webRequest, final HttpClientBuilder httpClientBuilder) throws IOException, URISyntaxException { final String charset = webRequest.getCharset(); // Make sure that the URL is fully encoded. IE actually sends some Unicode chars in request // URLs; because of this we allow some Unicode chars in URLs. However, at this point we're // handing things over the HttpClient, and HttpClient will blow up if we leave these Unicode // chars in the URL. final URL url = UrlUtils.encodeUrl(webRequest.getUrl(), false, charset); // URIUtils.createURI is deprecated but as of httpclient-4.2.1, URIBuilder doesn't work here as it encodes path // what shouldn't happen here URI uri = URIUtils.createURI(url.getProtocol(), url.getHost(), url.getPort(), url.getPath(), escapeQuery(url.getQuery()), null); if (getVirtualHost() != null) { uri = URI.create(getVirtualHost()); } final HttpRequestBase httpMethod = buildHttpMethod(webRequest.getHttpMethod(), uri); setProxy(httpMethod, webRequest); if (!(httpMethod instanceof HttpEntityEnclosingRequest)) { // this is the case for GET as well as TRACE, DELETE, OPTIONS and HEAD if (!webRequest.getRequestParameters().isEmpty()) { final List<NameValuePair> pairs = webRequest.getRequestParameters(); final org.apache.http.NameValuePair[] httpClientPairs = NameValuePair.toHttpClient(pairs); final String query = URLEncodedUtils.format(Arrays.asList(httpClientPairs), charset); uri = URIUtils.createURI(url.getProtocol(), url.getHost(), url.getPort(), url.getPath(), query, null); httpMethod.setURI(uri); } } else { // POST as well as PUT and PATCH final HttpEntityEnclosingRequest method = (HttpEntityEnclosingRequest) httpMethod; if (webRequest.getEncodingType() == FormEncodingType.URL_ENCODED && method instanceof HttpPost) { final HttpPost postMethod = (HttpPost) method; if (webRequest.getRequestBody() == null) { final List<NameValuePair> pairs = webRequest.getRequestParameters(); final org.apache.http.NameValuePair[] httpClientPairs = NameValuePair.toHttpClient(pairs); final String query = URLEncodedUtils.format(Arrays.asList(httpClientPairs), charset); final StringEntity urlEncodedEntity = new StringEntity(query, charset); urlEncodedEntity.setContentType(URLEncodedUtils.CONTENT_TYPE); postMethod.setEntity(urlEncodedEntity); } else { final String body = StringUtils.defaultString(webRequest.getRequestBody()); final StringEntity urlEncodedEntity = new StringEntity(body, charset); urlEncodedEntity.setContentType(URLEncodedUtils.CONTENT_TYPE); postMethod.setEntity(urlEncodedEntity); } } else if (FormEncodingType.MULTIPART == webRequest.getEncodingType()) { final Charset c = getCharset(charset, webRequest.getRequestParameters()); final MultipartEntityBuilder builder = MultipartEntityBuilder.create().setLaxMode(); builder.setCharset(c); for (final NameValuePair pair : webRequest.getRequestParameters()) { if (pair instanceof KeyDataPair) { buildFilePart((KeyDataPair) pair, builder); } else { builder.addTextBody(pair.getName(), pair.getValue(), ContentType.create("text/plain", charset)); } } method.setEntity(builder.build()); } else { // for instance a PUT or PATCH request final String body = webRequest.getRequestBody(); if (body != null) { method.setEntity(new StringEntity(body, charset)); } } } configureHttpProcessorBuilder(httpClientBuilder, webRequest); // Tell the client where to get its credentials from // (it may have changed on the webClient since last call to getHttpClientFor(...)) final CredentialsProvider credentialsProvider = webClient_.getCredentialsProvider(); // if the used url contains credentials, we have to add this final Credentials requestUrlCredentials = webRequest.getUrlCredentials(); if (null != requestUrlCredentials && webClient_.getBrowserVersion().hasFeature(URL_AUTH_CREDENTIALS)) { final URL requestUrl = webRequest.getUrl(); final AuthScope authScope = new AuthScope(requestUrl.getHost(), requestUrl.getPort()); // updating our client to keep the credentials for the next request credentialsProvider.setCredentials(authScope, requestUrlCredentials); httpContext_.removeAttribute(HttpClientContext.TARGET_AUTH_STATE); } // if someone has set credentials to this request, we have to add this final Credentials requestCredentials = webRequest.getCredentials(); if (null != requestCredentials) { final URL requestUrl = webRequest.getUrl(); final AuthScope authScope = new AuthScope(requestUrl.getHost(), requestUrl.getPort()); // updating our client to keep the credentials for the next request credentialsProvider.setCredentials(authScope, requestCredentials); httpContext_.removeAttribute(HttpClientContext.TARGET_AUTH_STATE); } httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); httpContext_.removeAttribute(HttpClientContext.CREDS_PROVIDER); return httpMethod; }
From source file:com.basho.riak.client.util.ClientHelper.java
/** * Perform and HTTP request and return the resulting response using the * internal HttpClient./* ww w . j a va 2s .c om*/ * * @param bucket * Bucket of the object receiving the request. * @param key * Key of the object receiving the request or null if the request * is for a bucket. * @param httpMethod * The HTTP request to perform; must not be null. * @param meta * Extra HTTP headers to attach to the request. Query parameters * are ignored; they should have already been used to construct * <code>httpMethod</code> and query parameters. * @param streamResponse * If true, the connection will NOT be released. Use * HttpResponse.getHttpMethod().getResponseBodyAsStream() to get * the response stream; HttpResponse.getBody() will return null. * * @return The HTTP response returned by Riak from executing * <code>httpMethod</code>. * * @throws RiakIORuntimeException * If an error occurs during communication with the Riak server * (i.e. HttpClient threw an IOException) */ HttpResponse executeMethod(String bucket, String key, HttpRequestBase httpMethod, RequestMeta meta, boolean streamResponse) { if (meta != null) { Map<String, String> headers = meta.getHeaders(); for (String header : headers.keySet()) { httpMethod.addHeader(header, headers.get(header)); } Map<String, String> queryParams = meta.getQueryParamMap(); if (!queryParams.isEmpty()) { URI originalURI = httpMethod.getURI(); List<NameValuePair> currentQuery = URLEncodedUtils.parse(originalURI, CharsetUtils.UTF_8.name()); List<NameValuePair> newQuery = new LinkedList<NameValuePair>(currentQuery); for (Map.Entry<String, String> qp : queryParams.entrySet()) { newQuery.add(new BasicNameValuePair(qp.getKey(), qp.getValue())); } // For this, HC4.1 authors, I hate you URI newURI; try { newURI = URIUtils.createURI(originalURI.getScheme(), originalURI.getHost(), originalURI.getPort(), originalURI.getPath(), URLEncodedUtils.format(newQuery, "UTF-8"), null); } catch (URISyntaxException e) { throw new RiakIORuntimeException(e); } httpMethod.setURI(newURI); } } HttpEntity entity; try { org.apache.http.HttpResponse response = httpClient.execute(httpMethod); int status = 0; if (response.getStatusLine() != null) { status = response.getStatusLine().getStatusCode(); } Map<String, String> headers = ClientUtils.asHeaderMap(response.getAllHeaders()); byte[] body = null; InputStream stream = null; entity = response.getEntity(); if (streamResponse) { stream = entity.getContent(); } else { if (null != entity) { body = EntityUtils.toByteArray(entity); } } if (!streamResponse) { EntityUtils.consume(entity); } return new DefaultHttpResponse(bucket, key, status, headers, body, stream, response, httpMethod); } catch (IOException e) { httpMethod.abort(); return toss(new RiakIORuntimeException(e)); } }
From source file:com.heaptrip.util.http.bixo.fetcher.SimpleHttpFetcher.java
private FetchedResult doRequest(HttpRequestBase request, String url, List<TupleTwo<?, ?>> data, List<TupleTwo<?, ?>> headers) throws BaseFetchException { LOGGER.trace("Fetching " + url); HttpResponse response;/* w w w .j a v a2 s . c o m*/ long readStartTime; HttpHeaders headerMap = new HttpHeaders(); String redirectedUrl = null; String newBaseUrl = null; int numRedirects = 0; boolean needAbort = true; String contentType = ""; String hostAddress = null; // Create a local instance of cookie store, and bind to local context // Without this we get killed w/lots of threads, due to sync() on single // cookie store. HttpContext localContext = new BasicHttpContext(); CookieStore cookieStore = new BasicCookieStore(); localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore); try { URI uri = new URI(url); request.setURI(uri); request.setHeader("Host", uri.getHost()); if (headers != null) { for (TupleTwo<?, ?> t : headers) { request.setHeader(t.getKey().toString(), t.getValue().toString()); } } //collect post data if available if (request instanceof HttpPost && data != null) { List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(1); for (TupleTwo<?, ?> e : data) { nameValuePairs.add(new BasicNameValuePair(URLEncoder.encode(e.getKey().toString(), "utf-8"), URLEncoder.encode(e.getValue().toString(), "utf-8"))); } ((HttpPost) (request)).setEntity(new UrlEncodedFormEntity(nameValuePairs)); } readStartTime = System.currentTimeMillis(); response = _httpClient.execute(request, localContext); Header[] responseHeaders = response.getAllHeaders(); for (Header header : responseHeaders) { headerMap.add(header.getName(), header.getValue()); } int httpStatus = response.getStatusLine().getStatusCode(); if ((httpStatus < 200) || (httpStatus >= 300)) { // We can't just check against SC_OK, as some wackos return 201, 202, // etc throw new HttpFetchException(url, "Error fetching " + url + " due to http status code " + httpStatus, httpStatus, headerMap); } redirectedUrl = extractRedirectedUrl(url, localContext); URI permRedirectUri = (URI) localContext.getAttribute(PERM_REDIRECT_CONTEXT_KEY); if (permRedirectUri != null) { newBaseUrl = permRedirectUri.toURL().toExternalForm(); } Integer redirects = (Integer) localContext.getAttribute(REDIRECT_COUNT_CONTEXT_KEY); if (redirects != null) { numRedirects = redirects.intValue(); } hostAddress = (String) (localContext.getAttribute(HOST_ADDRESS)); if (hostAddress == null) { throw new UrlFetchException(url, "Host address not saved in context"); } Header cth = response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE); if (cth != null) { contentType = cth.getValue(); } needAbort = false; } catch (ClientProtocolException e) { // Oleg guarantees that no abort is needed in the case of an IOException // (which is is a subclass of) needAbort = false; // If the root case was a "too many redirects" error, we want to map this // to a specific // exception that contains the final redirect. if (e.getCause() instanceof MyRedirectException) { MyRedirectException mre = (MyRedirectException) e.getCause(); String redirectUrl = url; try { redirectUrl = mre.getUri().toURL().toExternalForm(); } catch (MalformedURLException e2) { LOGGER.warn("Invalid URI saved during redirect handling: " + mre.getUri()); } throw new RedirectFetchException(url, redirectUrl, mre.getReason()); } else if (e.getCause() instanceof RedirectException) { throw new RedirectFetchException(url, extractRedirectedUrl(url, localContext), RedirectExceptionReason.TOO_MANY_REDIRECTS); } else { throw new IOFetchException(url, e); } } catch (IOException e) { // Oleg guarantees that no abort is needed in the case of an IOException needAbort = false; if (e instanceof ConnectionPoolTimeoutException) { // Should never happen, so let's dump some info about the connection // pool. ThreadSafeClientConnManager cm = (ThreadSafeClientConnManager) _httpClient.getConnectionManager(); int numConnections = cm.getConnectionsInPool(); cm.closeIdleConnections(0, TimeUnit.MILLISECONDS); LOGGER.error(String.format( "Got ConnectionPoolTimeoutException: %d connections before, %d after idle close", numConnections, cm.getConnectionsInPool())); } throw new IOFetchException(url, e); } catch (URISyntaxException e) { throw new UrlFetchException(url, e.getMessage()); } catch (IllegalStateException e) { throw new UrlFetchException(url, e.getMessage()); } catch (BaseFetchException e) { throw e; } catch (Exception e) { // Map anything else to a generic IOFetchException // TODO KKr - create generic fetch exception throw new IOFetchException(url, new IOException(e)); } finally { safeAbort(needAbort, request); } // Figure out how much data we want to try to fetch. int targetLength = _fetcherPolicy.getMaxContentSize(); boolean truncated = false; String contentLengthStr = headerMap.getFirst(HttpHeaderNames.CONTENT_LENGTH); if (contentLengthStr != null) { try { int contentLength = Integer.parseInt(contentLengthStr); if (contentLength > targetLength) { truncated = true; } else { targetLength = contentLength; } } catch (NumberFormatException e) { // Ignore (and log) invalid content length values. LOGGER.warn("Invalid content length in header: " + contentLengthStr); } } // Now finally read in response body, up to targetLength bytes. // Note that entity might be null, for zero length responses. byte[] content = new byte[0]; long readRate = 0; HttpEntity entity = response.getEntity(); needAbort = true; if (entity != null) { InputStream in = null; try { in = entity.getContent(); byte[] buffer = new byte[BUFFER_SIZE]; int bytesRead = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(DEFAULT_BYTEARRAY_SIZE); int readRequests = 0; int minResponseRate = _fetcherPolicy.getMinResponseRate(); // TODO KKr - we need to monitor the rate while reading a // single block. Look at HttpClient // metrics support for how to do this. Once we fix this, fix // the test to read a smaller (< 20K) // chuck of data. while ((totalRead < targetLength) && ((bytesRead = in.read(buffer, 0, Math.min(buffer.length, targetLength - totalRead))) != -1)) { readRequests += 1; totalRead += bytesRead; out.write(buffer, 0, bytesRead); // Assume read time is at least one millisecond, to avoid DBZ // exception. long totalReadTime = Math.max(1, System.currentTimeMillis() - readStartTime); readRate = (totalRead * 1000L) / totalReadTime; // Don't bail on the first read cycle, as we can get a hiccup starting // out. // Also don't bail if we've read everything we need. if ((readRequests > 1) && (totalRead < targetLength) && (readRate < minResponseRate)) { throw new AbortedFetchException(url, "Slow response rate of " + readRate + " bytes/sec", AbortedFetchReason.SLOW_RESPONSE_RATE); } // Check to see if we got interrupted. if (Thread.interrupted()) { throw new AbortedFetchException(url, AbortedFetchReason.INTERRUPTED); } } content = out.toByteArray(); needAbort = truncated || (in.available() > 0); } catch (IOException e) { // We don't need to abort if there's an IOException throw new IOFetchException(url, e); } finally { safeAbort(needAbort, request); safeClose(in); } } return new FetchedResult(url, redirectedUrl, System.currentTimeMillis(), headerMap, content, contentType, (int) readRate, newBaseUrl, numRedirects, hostAddress); }
From source file:com.basho.riak.client.http.util.ClientHelper.java
/** * Perform and HTTP request and return the resulting response using the * internal HttpClient.//from w ww . ja v a2 s. c om * * @param bucket * Bucket of the object receiving the request. * @param key * Key of the object receiving the request or null if the request * is for a bucket. * @param httpMethod * The HTTP request to perform; must not be null. * @param meta * Extra HTTP headers to attach to the request. Query parameters * are ignored; they should have already been used to construct * <code>httpMethod</code> and query parameters. * @param streamResponse * If true, the connection will NOT be released. Use * HttpResponse.getHttpMethod().getResponseBodyAsStream() to get * the response stream; HttpResponse.getBody() will return null. * * @return The HTTP response returned by Riak from executing * <code>httpMethod</code>. * * @throws RiakIORuntimeException * If an error occurs during communication with the Riak server * (i.e. HttpClient threw an IOException) */ HttpResponse executeMethod(String bucket, String key, HttpRequestBase httpMethod, RequestMeta meta, boolean streamResponse) { if (meta != null) { Map<String, String> headers = meta.getHeaders(); for (String header : headers.keySet()) { httpMethod.addHeader(header, headers.get(header)); } Map<String, String> queryParams = meta.getQueryParamMap(); if (!queryParams.isEmpty()) { URI originalURI = httpMethod.getURI(); List<NameValuePair> currentQuery = URLEncodedUtils.parse(originalURI, CharsetUtils.UTF_8.name()); List<NameValuePair> newQuery = new LinkedList<NameValuePair>(currentQuery); for (Map.Entry<String, String> qp : queryParams.entrySet()) { newQuery.add(new BasicNameValuePair(qp.getKey(), qp.getValue())); } // For this, HC4.1 authors, I hate you URI newURI; try { newURI = new URIBuilder(originalURI).setQuery(URLEncodedUtils.format(newQuery, "UTF-8")) .build(); } catch (URISyntaxException e) { e.printStackTrace(); throw new RiakIORuntimeException(e); } httpMethod.setURI(newURI); } } HttpEntity entity = null; try { org.apache.http.HttpResponse response = httpClient.execute(httpMethod); int status = 0; if (response.getStatusLine() != null) { status = response.getStatusLine().getStatusCode(); } Map<String, String> headers = ClientUtils.asHeaderMap(response.getAllHeaders()); byte[] body = null; InputStream stream = null; entity = response.getEntity(); if (streamResponse) { stream = entity.getContent(); } else { if (null != entity) { body = EntityUtils.toByteArray(entity); } } key = extractKeyFromResponseIfItWasNotAlreadyProvided(key, response); return new DefaultHttpResponse(bucket, key, status, headers, body, stream, response, httpMethod); } catch (IOException e) { httpMethod.abort(); return toss(new RiakIORuntimeException(e)); } finally { if (!streamResponse && entity != null) { try { EntityUtils.consume(entity); } catch (IOException e) { // NO-OP } } } }
From source file:bixo.fetcher.SimpleHttpFetcher.java
private FetchedResult doRequest(HttpRequestBase request, String url, List<Tuple2<?, ?>> data, List<Tuple2<?, ?>> headers) throws BaseFetchException { LOGGER.trace("Fetching " + url); HttpResponse response;/* w w w . j a v a2 s .c o m*/ long readStartTime; HttpHeaders headerMap = new HttpHeaders(); String redirectedUrl = null; String newBaseUrl = null; int numRedirects = 0; boolean needAbort = true; String contentType = ""; String hostAddress = null; // Create a local instance of cookie store, and bind to local context // Without this we get killed w/lots of threads, due to sync() on single // cookie store. HttpContext localContext = new BasicHttpContext(); CookieStore cookieStore = new BasicCookieStore(); localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore); try { URI uri = new URI(url); request.setURI(uri); request.setHeader("Host", uri.getHost()); if (headers != null) { for (Tuple2<?, ?> t : headers) { request.setHeader(t.getKey().toString(), t.getValue().toString()); } } //collect post data if available if (request instanceof HttpPost && data != null) { List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(1); for (Tuple2<?, ?> e : data) { nameValuePairs.add(new BasicNameValuePair(URLEncoder.encode(e.getKey().toString(), "utf-8"), URLEncoder.encode(e.getValue().toString(), "utf-8"))); } ((HttpPost) (request)).setEntity(new UrlEncodedFormEntity(nameValuePairs)); } readStartTime = System.currentTimeMillis(); response = _httpClient.execute(request, localContext); Header[] responseHeaders = response.getAllHeaders(); for (Header header : responseHeaders) { headerMap.add(header.getName(), header.getValue()); } int httpStatus = response.getStatusLine().getStatusCode(); if ((httpStatus < 200) || (httpStatus >= 300)) { // We can't just check against SC_OK, as some wackos return 201, 202, // etc throw new HttpFetchException(url, "Error fetching " + url + " due to http status code " + httpStatus, httpStatus, headerMap); } redirectedUrl = extractRedirectedUrl(url, localContext); URI permRedirectUri = (URI) localContext.getAttribute(PERM_REDIRECT_CONTEXT_KEY); if (permRedirectUri != null) { newBaseUrl = permRedirectUri.toURL().toExternalForm(); } Integer redirects = (Integer) localContext.getAttribute(REDIRECT_COUNT_CONTEXT_KEY); if (redirects != null) { numRedirects = redirects.intValue(); } hostAddress = (String) (localContext.getAttribute(HOST_ADDRESS)); if (hostAddress == null) { throw new UrlFetchException(url, "Host address not saved in context"); } Header cth = response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE); if (cth != null) { contentType = cth.getValue(); } needAbort = false; } catch (ClientProtocolException e) { // Oleg guarantees that no abort is needed in the case of an IOException // (which is is a subclass of) needAbort = false; // If the root case was a "too many redirects" error, we want to map this // to a specific // exception that contains the final redirect. if (e.getCause() instanceof MyRedirectException) { MyRedirectException mre = (MyRedirectException) e.getCause(); String redirectUrl = url; try { redirectUrl = mre.getUri().toURL().toExternalForm(); } catch (MalformedURLException e2) { LOGGER.warn("Invalid URI saved during redirect handling: " + mre.getUri()); } throw new RedirectFetchException(url, redirectUrl, mre.getReason()); } else if (e.getCause() instanceof RedirectException) { throw new RedirectFetchException(url, extractRedirectedUrl(url, localContext), RedirectExceptionReason.TOO_MANY_REDIRECTS); } else { throw new IOFetchException(url, e); } } catch (IOException e) { // Oleg guarantees that no abort is needed in the case of an IOException needAbort = false; if (e instanceof ConnectionPoolTimeoutException) { // Should never happen, so let's dump some info about the connection // pool. ThreadSafeClientConnManager cm = (ThreadSafeClientConnManager) _httpClient.getConnectionManager(); int numConnections = cm.getConnectionsInPool(); cm.closeIdleConnections(0, TimeUnit.MILLISECONDS); LOGGER.error(String.format( "Got ConnectionPoolTimeoutException: %d connections before, %d after idle close", numConnections, cm.getConnectionsInPool())); } throw new IOFetchException(url, e); } catch (URISyntaxException e) { throw new UrlFetchException(url, e.getMessage()); } catch (IllegalStateException e) { throw new UrlFetchException(url, e.getMessage()); } catch (BaseFetchException e) { throw e; } catch (Exception e) { // Map anything else to a generic IOFetchException // TODO KKr - create generic fetch exception throw new IOFetchException(url, new IOException(e)); } finally { safeAbort(needAbort, request); } // Figure out how much data we want to try to fetch. int targetLength = _fetcherPolicy.getMaxContentSize(); boolean truncated = false; String contentLengthStr = headerMap.getFirst(HttpHeaderNames.CONTENT_LENGTH); if (contentLengthStr != null) { try { int contentLength = Integer.parseInt(contentLengthStr); if (contentLength > targetLength) { truncated = true; } else { targetLength = contentLength; } } catch (NumberFormatException e) { // Ignore (and log) invalid content length values. LOGGER.warn("Invalid content length in header: " + contentLengthStr); } } // Now finally read in response body, up to targetLength bytes. // Note that entity might be null, for zero length responses. byte[] content = new byte[0]; long readRate = 0; HttpEntity entity = response.getEntity(); needAbort = true; if (entity != null) { InputStream in = null; try { in = entity.getContent(); byte[] buffer = new byte[BUFFER_SIZE]; int bytesRead = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(DEFAULT_BYTEARRAY_SIZE); int readRequests = 0; int minResponseRate = _fetcherPolicy.getMinResponseRate(); // TODO KKr - we need to monitor the rate while reading a // single block. Look at HttpClient // metrics support for how to do this. Once we fix this, fix // the test to read a smaller (< 20K) // chuck of data. while ((totalRead < targetLength) && ((bytesRead = in.read(buffer, 0, Math.min(buffer.length, targetLength - totalRead))) != -1)) { readRequests += 1; totalRead += bytesRead; out.write(buffer, 0, bytesRead); // Assume read time is at least one millisecond, to avoid DBZ // exception. long totalReadTime = Math.max(1, System.currentTimeMillis() - readStartTime); readRate = (totalRead * 1000L) / totalReadTime; // Don't bail on the first read cycle, as we can get a hiccup starting // out. // Also don't bail if we've read everything we need. if ((readRequests > 1) && (totalRead < targetLength) && (readRate < minResponseRate)) { throw new AbortedFetchException(url, "Slow response rate of " + readRate + " bytes/sec", AbortedFetchReason.SLOW_RESPONSE_RATE); } // Check to see if we got interrupted. if (Thread.interrupted()) { throw new AbortedFetchException(url, AbortedFetchReason.INTERRUPTED); } } content = out.toByteArray(); needAbort = truncated || (in.available() > 0); } catch (IOException e) { // We don't need to abort if there's an IOException throw new IOFetchException(url, e); } finally { safeAbort(needAbort, request); safeClose(in); } } return new FetchedResult(url, redirectedUrl, System.currentTimeMillis(), headerMap, content, contentType, (int) readRate, newBaseUrl, numRedirects, hostAddress); }
From source file:pt.lunacloud.http.AmazonHttpClient.java
/** * Internal method to execute the HTTP method given. * * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler) * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler, ExecutionContext) *//* ww w .j a va 2 s . c o m*/ private <T extends Object> T executeHelper(Request<?> request, HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler, HttpResponseHandler<LunacloudServiceException> errorResponseHandler, ExecutionContext executionContext) throws LunacloudClientException, LunacloudServiceException { /* * Depending on which response handler we end up choosing to handle the * HTTP response, it might require us to leave the underlying HTTP * connection open, depending on whether or not it reads the complete * HTTP response stream from the HTTP connection, or if delays reading * any of the content until after a response is returned to the caller. */ boolean leaveHttpConnectionOpen = false; AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics(); /* add the service endpoint to the logs. You can infer service name from service endpoint */ awsRequestMetrics.addProperty(Field.ServiceName.name(), request.getServiceName()); awsRequestMetrics.addProperty(Field.ServiceEndpoint.name(), request.getEndpoint()); // Apply whatever request options we know how to handle, such as user-agent. applyRequestData(request); int retryCount = 0; URI redirectedURI = null; HttpEntity entity = null; LunacloudServiceException exception = null; // Make a copy of the original request params and headers so that we can // permute it in this loop and start over with the original every time. Map<String, String> originalParameters = new HashMap<String, String>(); originalParameters.putAll(request.getParameters()); Map<String, String> originalHeaders = new HashMap<String, String>(); originalHeaders.putAll(request.getHeaders()); while (true) { awsRequestMetrics.setCounter(Field.AttemptCount.name(), retryCount + 1); if (retryCount > 0) { request.setParameters(originalParameters); request.setHeaders(originalHeaders); } HttpRequestBase httpRequest = null; org.apache.http.HttpResponse response = null; try { // Sign the request if a signer was provided if (executionContext.getSigner() != null && executionContext.getCredentials() != null) { awsRequestMetrics.startEvent(Field.RequestSigningTime.name()); executionContext.getSigner().sign(request, executionContext.getCredentials()); awsRequestMetrics.endEvent(Field.RequestSigningTime.name()); } if (requestLog.isDebugEnabled()) { requestLog.debug("Sending Request: " + request.toString()); } httpRequest = httpRequestFactory.createHttpRequest(request, config, entity, executionContext); if (httpRequest instanceof HttpEntityEnclosingRequest) { entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity(); } if (redirectedURI != null) { httpRequest.setURI(redirectedURI); } if (retryCount > 0) { awsRequestMetrics.startEvent(Field.RetryPauseTime.name()); pauseExponentially(retryCount, exception, executionContext.getCustomBackoffStrategy()); awsRequestMetrics.endEvent(Field.RetryPauseTime.name()); } if (entity != null) { InputStream content = entity.getContent(); if (retryCount > 0) { if (content.markSupported()) { content.reset(); content.mark(-1); } } else { if (content.markSupported()) { content.mark(-1); } } } exception = null; awsRequestMetrics.startEvent(Field.HttpRequestTime.name()); response = httpClient.execute(httpRequest); awsRequestMetrics.endEvent(Field.HttpRequestTime.name()); if (isRequestSuccessful(response)) { awsRequestMetrics.addProperty(Field.StatusCode.name(), response.getStatusLine().getStatusCode()); /* * If we get back any 2xx status code, then we know we should * treat the service call as successful. */ leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen(); return handleResponse(request, responseHandler, httpRequest, response, executionContext); } else if (isTemporaryRedirect(response)) { /* * S3 sends 307 Temporary Redirects if you try to delete an * EU bucket from the US endpoint. If we get a 307, we'll * point the HTTP method to the redirected location, and let * the next retry deliver the request to the right location. */ Header[] locationHeaders = response.getHeaders("location"); String redirectedLocation = locationHeaders[0].getValue(); log.debug("Redirecting to: " + redirectedLocation); redirectedURI = URI.create(redirectedLocation); httpRequest.setURI(redirectedURI); awsRequestMetrics.addProperty(Field.StatusCode.name(), response.getStatusLine().getStatusCode()); awsRequestMetrics.addProperty(Field.RedirectLocation.name(), redirectedLocation); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null); } else { leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen(); exception = handleErrorResponse(request, errorResponseHandler, httpRequest, response); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), exception.getRequestId()); awsRequestMetrics.addProperty(Field.AWSErrorCode.name(), exception.getErrorCode()); awsRequestMetrics.addProperty(Field.StatusCode.name(), exception.getStatusCode()); if (!shouldRetry(httpRequest, exception, retryCount)) { throw exception; } resetRequestAfterError(request, exception); } } catch (IOException ioe) { log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe); awsRequestMetrics.addProperty(Field.Exception.name(), ioe.toString()); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null); if (!shouldRetry(httpRequest, ioe, retryCount)) { throw new LunacloudClientException("Unable to execute HTTP request: " + ioe.getMessage(), ioe); } resetRequestAfterError(request, ioe); } finally { retryCount++; /* * Some response handlers need to manually manage the HTTP * connection and will take care of releasing the connection on * their own, but if this response handler doesn't need the * connection left open, we go ahead and release the it to free * up resources. */ if (!leaveHttpConnectionOpen) { try { response.getEntity().getContent().close(); } catch (Throwable t) { } } } } /* end while (true) */ }
From source file:cn.ctyun.amazonaws.http.AmazonHttpClient.java
/** * Internal method to execute the HTTP method given. * * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler) * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler, ExecutionContext) *//*from w w w .j ava 2 s . c o m*/ private <T extends Object> T executeHelper(Request<?> request, HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler, HttpResponseHandler<AmazonServiceException> errorResponseHandler, ExecutionContext executionContext) throws AmazonClientException, AmazonServiceException { /* * Depending on which response handler we end up choosing to handle the * HTTP response, it might require us to leave the underlying HTTP * connection open, depending on whether or not it reads the complete * HTTP response stream from the HTTP connection, or if delays reading * any of the content until after a response is returned to the caller. */ boolean leaveHttpConnectionOpen = false; AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics(); /* add the service endpoint to the logs. You can infer service name from service endpoint */ awsRequestMetrics.addProperty(Field.ServiceName.name(), request.getServiceName()); awsRequestMetrics.addProperty(Field.ServiceEndpoint.name(), request.getEndpoint()); // Apply whatever request options we know how to handle, such as user-agent. setUserAgent(request); int retryCount = 0; URI redirectedURI = null; HttpEntity entity = null; AmazonServiceException exception = null; // Make a copy of the original request params and headers so that we can // permute it in this loop and start over with the original every time. Map<String, String> originalParameters = new HashMap<String, String>(); originalParameters.putAll(request.getParameters()); Map<String, String> originalHeaders = new HashMap<String, String>(); originalHeaders.putAll(request.getHeaders()); while (true) { awsRequestMetrics.setCounter(Field.AttemptCount.name(), retryCount + 1); if (retryCount > 0) { request.setParameters(originalParameters); request.setHeaders(originalHeaders); } HttpRequestBase httpRequest = null; org.apache.http.HttpResponse response = null; try { // Sign the request if a signer was provided if (executionContext.getSigner() != null && executionContext.getCredentials() != null) { awsRequestMetrics.startEvent(Field.RequestSigningTime.name()); executionContext.getSigner().sign(request, executionContext.getCredentials()); awsRequestMetrics.endEvent(Field.RequestSigningTime.name()); } if (requestLog.isDebugEnabled()) { requestLog.debug("Sending Request: " + request.toString()); } httpRequest = httpRequestFactory.createHttpRequest(request, config, entity, executionContext); if (httpRequest instanceof HttpEntityEnclosingRequest) { entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity(); } if (redirectedURI != null) { httpRequest.setURI(redirectedURI); } if (retryCount > 0) { awsRequestMetrics.startEvent(Field.RetryPauseTime.name()); pauseExponentially(retryCount, exception, executionContext.getCustomBackoffStrategy()); awsRequestMetrics.endEvent(Field.RetryPauseTime.name()); } if (entity != null) { InputStream content = entity.getContent(); if (retryCount > 0) { if (content.markSupported()) { content.reset(); content.mark(-1); } } else { if (content.markSupported()) { content.mark(-1); } } } exception = null; awsRequestMetrics.startEvent(Field.HttpRequestTime.name()); response = httpClient.execute(httpRequest); awsRequestMetrics.endEvent(Field.HttpRequestTime.name()); if (isRequestSuccessful(response)) { awsRequestMetrics.addProperty(Field.StatusCode.name(), response.getStatusLine().getStatusCode()); /* * If we get back any 2xx status code, then we know we should * treat the service call as successful. */ leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen(); return handleResponse(request, responseHandler, httpRequest, response, executionContext); } else if (isTemporaryRedirect(response)) { /* * S3 sends 307 Temporary Redirects if you try to delete an * EU bucket from the US endpoint. If we get a 307, we'll * point the HTTP method to the redirected location, and let * the next retry deliver the request to the right location. */ Header[] locationHeaders = response.getHeaders("location"); String redirectedLocation = locationHeaders[0].getValue(); log.debug("Redirecting to: " + redirectedLocation); redirectedURI = URI.create(redirectedLocation); httpRequest.setURI(redirectedURI); awsRequestMetrics.addProperty(Field.StatusCode.name(), response.getStatusLine().getStatusCode()); awsRequestMetrics.addProperty(Field.RedirectLocation.name(), redirectedLocation); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null); } else { leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen(); exception = handleErrorResponse(request, errorResponseHandler, httpRequest, response); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), exception.getRequestId()); awsRequestMetrics.addProperty(Field.AWSErrorCode.name(), exception.getErrorCode()); awsRequestMetrics.addProperty(Field.StatusCode.name(), exception.getStatusCode()); if (!shouldRetry(httpRequest, exception, retryCount)) { throw exception; } resetRequestAfterError(request, exception); } } catch (IOException ioe) { log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe); awsRequestMetrics.addProperty(Field.Exception.name(), ioe.toString()); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null); if (!shouldRetry(httpRequest, ioe, retryCount)) { throw new AmazonClientException("Unable to execute HTTP request: " + ioe.getMessage(), ioe); } resetRequestAfterError(request, ioe); } finally { retryCount++; /* * Some response handlers need to manually manage the HTTP * connection and will take care of releasing the connection on * their own, but if this response handler doesn't need the * connection left open, we go ahead and release the it to free * up resources. */ if (!leaveHttpConnectionOpen) { try { response.getEntity().getContent().close(); } catch (Throwable t) { } } } } /* end while (true) */ }