List of usage examples for org.apache.http.protocol HttpContext getAttribute
Object getAttribute(String str);
From source file:com.adavr.http.Client.java
public Client() { httpClient = new DefaultHttpClient(new BasicClientConnectionManager()); localContext = new BasicHttpContext(); byteStream = new ByteArrayOutputStream(); CookieStore cookieStore = new BasicCookieStore(); localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore); httpClient.addRequestInterceptor(new HttpRequestInterceptor() { @Override/*ww w. ja v a 2 s . com*/ public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { if (!request.containsHeader("Accept-Encoding")) { request.addHeader("Accept-Encoding", "gzip"); } } }); httpClient.addResponseInterceptor(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); Header ceheader = entity.getContentEncoding(); if (ceheader != null) { HeaderElement[] codecs = ceheader.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } }); httpClient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT, DEFAULT_TIMEOUT); httpClient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, DEFAULT_USER_AGENT); HttpRequestRetryHandler myRetryHandler = new HttpRequestRetryHandler() { @Override public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { if (executionCount >= DEFAULT_RETRY) { // Do not retry if over max retry count return false; } System.out.println(exception.getClass().getName()); if (exception instanceof NoHttpResponseException) { // Retry if the server dropped connection on us return true; } if (exception instanceof SSLHandshakeException) { // Do not retry on SSL handshake exception return false; } HttpRequest request = (HttpRequest) context.getAttribute(ExecutionContext.HTTP_REQUEST); boolean idempotent = !(request instanceof HttpEntityEnclosingRequest); if (idempotent) { // Retry if the request is considered idempotent return true; } return false; } }; httpClient.setHttpRequestRetryHandler(myRetryHandler); }
From source file:com.xtremelabs.robolectric.tester.org.apache.http.impl.client.DefaultRequestDirector.java
public HttpResponse execute(HttpHost target, HttpRequest request, HttpContext context) throws HttpException, IOException { HttpRequest orig = request;// w ww . j a v a 2s . co m RequestWrapper origWrapper = wrapRequest(orig); origWrapper.setParams(params); HttpRoute origRoute = determineRoute(target, origWrapper, context); virtualHost = (HttpHost) orig.getParams().getParameter(ClientPNames.VIRTUAL_HOST); RoutedRequest roureq = new RoutedRequest(origWrapper, origRoute); long timeout = ConnManagerParams.getTimeout(params); int execCount = 0; boolean reuse = false; boolean done = false; try { HttpResponse response = null; while (!done) { // In this loop, the RoutedRequest may be replaced by a // followup request and route. The request and route passed // in the method arguments will be replaced. The original // request is still available in 'orig'. RequestWrapper wrapper = roureq.getRequest(); HttpRoute route = roureq.getRoute(); response = null; // See if we have a user token bound to the execution context Object userToken = context.getAttribute(ClientContext.USER_TOKEN); // Allocate connection if needed if (managedConn == null) { ClientConnectionRequest connRequest = connManager.requestConnection(route, userToken); if (orig instanceof AbortableHttpRequest) { ((AbortableHttpRequest) orig).setConnectionRequest(connRequest); } try { managedConn = connRequest.getConnection(timeout, TimeUnit.MILLISECONDS); } catch (InterruptedException interrupted) { InterruptedIOException iox = new InterruptedIOException(); iox.initCause(interrupted); throw iox; } if (HttpConnectionParams.isStaleCheckingEnabled(params)) { // validate connection if (managedConn.isOpen()) { this.log.debug("Stale connection check"); if (managedConn.isStale()) { this.log.debug("Stale connection detected"); managedConn.close(); } } } } if (orig instanceof AbortableHttpRequest) { ((AbortableHttpRequest) orig).setReleaseTrigger(managedConn); } // Reopen connection if needed if (!managedConn.isOpen()) { managedConn.open(route, context, params); } else { managedConn.setSocketTimeout(HttpConnectionParams.getSoTimeout(params)); } try { establishRoute(route, context); } catch (TunnelRefusedException ex) { if (this.log.isDebugEnabled()) { this.log.debug(ex.getMessage()); } response = ex.getResponse(); break; } // Reset headers on the request wrapper wrapper.resetHeaders(); // Re-write request URI if needed rewriteRequestURI(wrapper, route); // Use virtual host if set target = virtualHost; if (target == null) { target = route.getTargetHost(); } HttpHost proxy = route.getProxyHost(); // Populate the execution context context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, target); context.setAttribute(ExecutionContext.HTTP_PROXY_HOST, proxy); context.setAttribute(ExecutionContext.HTTP_CONNECTION, managedConn); context.setAttribute(ClientContext.TARGET_AUTH_STATE, targetAuthState); context.setAttribute(ClientContext.PROXY_AUTH_STATE, proxyAuthState); // Run request protocol interceptors requestExec.preProcess(wrapper, httpProcessor, context); boolean retrying = true; Exception retryReason = null; while (retrying) { // Increment total exec count (with redirects) execCount++; // Increment exec count for this particular request wrapper.incrementExecCount(); if (!wrapper.isRepeatable()) { this.log.debug("Cannot retry non-repeatable request"); if (retryReason != null) { throw new NonRepeatableRequestException("Cannot retry request " + "with a non-repeatable request entity. The cause lists the " + "reason the original request failed: " + retryReason); } else { throw new NonRepeatableRequestException( "Cannot retry request " + "with a non-repeatable request entity."); } } try { if (this.log.isDebugEnabled()) { this.log.debug("Attempt " + execCount + " to execute request"); } response = requestExec.execute(wrapper, managedConn, context); retrying = false; } catch (IOException ex) { this.log.debug("Closing the connection."); managedConn.close(); if (retryHandler.retryRequest(ex, wrapper.getExecCount(), context)) { if (this.log.isInfoEnabled()) { this.log.info("I/O exception (" + ex.getClass().getName() + ") caught when processing request: " + ex.getMessage()); } if (this.log.isDebugEnabled()) { this.log.debug(ex.getMessage(), ex); } this.log.info("Retrying request"); retryReason = ex; } else { throw ex; } // If we have a direct route to the target host // just re-open connection and re-try the request if (!route.isTunnelled()) { this.log.debug("Reopening the direct connection."); managedConn.open(route, context, params); } else { // otherwise give up this.log.debug("Proxied connection. Need to start over."); retrying = false; } } } if (response == null) { // Need to start over continue; } // Run response protocol interceptors response.setParams(params); requestExec.postProcess(response, httpProcessor, context); // The connection is in or can be brought to a re-usable state. reuse = reuseStrategy.keepAlive(response, context); if (reuse) { // Set the idle duration of this connection long duration = keepAliveStrategy.getKeepAliveDuration(response, context); managedConn.setIdleDuration(duration, TimeUnit.MILLISECONDS); if (this.log.isDebugEnabled()) { if (duration >= 0) { this.log.debug("Connection can be kept alive for " + duration + " ms"); } else { this.log.debug("Connection can be kept alive indefinitely"); } } } RoutedRequest followup = handleResponse(roureq, response, context); if (followup == null) { done = true; } else { if (reuse) { // Make sure the response body is fully consumed, if present HttpEntity entity = response.getEntity(); if (entity != null) { entity.consumeContent(); } // entity consumed above is not an auto-release entity, // need to mark the connection re-usable explicitly managedConn.markReusable(); } else { managedConn.close(); } // check if we can use the same connection for the followup if (!followup.getRoute().equals(roureq.getRoute())) { releaseConnection(); } roureq = followup; } if (managedConn != null && userToken == null) { userToken = userTokenHandler.getUserToken(context); context.setAttribute(ClientContext.USER_TOKEN, userToken); if (userToken != null) { managedConn.setState(userToken); } } } // while not done // check for entity, release connection if possible if ((response == null) || (response.getEntity() == null) || !response.getEntity().isStreaming()) { // connection not needed and (assumed to be) in re-usable state if (reuse) managedConn.markReusable(); releaseConnection(); } else { // install an auto-release entity HttpEntity entity = response.getEntity(); entity = new BasicManagedEntity(entity, managedConn, reuse); response.setEntity(entity); } return response; } catch (HttpException ex) { abortConnection(); throw ex; } catch (IOException ex) { abortConnection(); throw ex; } catch (RuntimeException ex) { abortConnection(); throw ex; } }
From source file:org.robolectric.shadows.httpclient.DefaultRequestDirector.java
public HttpResponse execute(HttpHost target, HttpRequest request, HttpContext context) throws HttpException, IOException { HttpRequest orig = request;//from ww w . j a va2 s . co m RequestWrapper origWrapper = wrapRequest(orig); origWrapper.setParams(params); HttpRoute origRoute = determineRoute(target, origWrapper, context); virtualHost = (HttpHost) orig.getParams().getParameter(ClientPNames.VIRTUAL_HOST); RoutedRequest roureq = new RoutedRequest(origWrapper, origRoute); long timeout = ConnManagerParams.getTimeout(params); int execCount = 0; boolean reuse = false; boolean done = false; try { HttpResponse response = null; while (!done) { // In this loop, the RoutedRequest may be replaced by a // followup request and route. The request and route passed // in the method arguments will be replaced. The original // request is still available in 'orig'. RequestWrapper wrapper = roureq.getRequest(); HttpRoute route = roureq.getRoute(); response = null; // See if we have a user token bound to the execution context Object userToken = context.getAttribute(ClientContext.USER_TOKEN); // Allocate connection if needed if (managedConn == null) { ClientConnectionRequest connRequest = connManager.requestConnection(route, userToken); if (orig instanceof AbortableHttpRequest) { ((AbortableHttpRequest) orig).setConnectionRequest(connRequest); } try { managedConn = connRequest.getConnection(timeout, TimeUnit.MILLISECONDS); } catch (InterruptedException interrupted) { InterruptedIOException iox = new InterruptedIOException(); iox.initCause(interrupted); throw iox; } if (HttpConnectionParams.isStaleCheckingEnabled(params)) { // validate connection if (managedConn.isOpen()) { this.log.debug("Stale connection check"); if (managedConn.isStale()) { this.log.debug("Stale connection detected"); managedConn.close(); } } } } if (orig instanceof AbortableHttpRequest) { ((AbortableHttpRequest) orig).setReleaseTrigger(managedConn); } // Reopen connection if needed if (!managedConn.isOpen()) { managedConn.open(route, context, params); } else { managedConn.setSocketTimeout(HttpConnectionParams.getSoTimeout(params)); } try { establishRoute(route, context); } catch (TunnelRefusedException ex) { if (this.log.isDebugEnabled()) { this.log.debug(ex.getMessage()); } response = ex.getResponse(); break; } // Reset headers on the request wrapper wrapper.resetHeaders(); // Re-write request URI if needed rewriteRequestURI(wrapper, route); // Use virtual host if set target = virtualHost; if (target == null) { target = route.getTargetHost(); } HttpHost proxy = route.getProxyHost(); // Populate the execution context context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, target); context.setAttribute(ExecutionContext.HTTP_PROXY_HOST, proxy); context.setAttribute(ExecutionContext.HTTP_CONNECTION, managedConn); context.setAttribute(ClientContext.TARGET_AUTH_STATE, targetAuthState); context.setAttribute(ClientContext.PROXY_AUTH_STATE, proxyAuthState); // Run request protocol interceptors requestExec.preProcess(wrapper, httpProcessor, context); boolean retrying = true; Exception retryReason = null; while (retrying) { // Increment total exec count (with redirects) execCount++; // Increment exec count for this particular request wrapper.incrementExecCount(); if (!wrapper.isRepeatable()) { this.log.debug("Cannot retry non-repeatable request"); if (retryReason != null) { throw new NonRepeatableRequestException("Cannot retry request " + "with a non-repeatable request entity. The cause lists the " + "reason the original request failed: " + retryReason); } else { throw new NonRepeatableRequestException( "Cannot retry request " + "with a non-repeatable request entity."); } } try { if (this.log.isDebugEnabled()) { this.log.debug("Attempt " + execCount + " to execute request"); } response = requestExec.execute(wrapper, managedConn, context); retrying = false; } catch (IOException ex) { this.log.debug("Closing the connection."); managedConn.close(); if (retryHandler.retryRequest(ex, wrapper.getExecCount(), context)) { if (this.log.isInfoEnabled()) { this.log.info("I/O exception (" + ex.getClass().getName() + ") caught when processing request: " + ex.getMessage()); } if (this.log.isDebugEnabled()) { this.log.debug(ex.getMessage(), ex); } this.log.info("Retrying request"); retryReason = ex; } else { throw ex; } // If we have a direct route to the target host // just re-open connection and re-try the request if (!route.isTunnelled()) { this.log.debug("Reopening the direct connection."); managedConn.open(route, context, params); } else { // otherwise give up this.log.debug("Proxied connection. Need to start over."); retrying = false; } } } if (response == null) { // Need to start over continue; } // Run response protocol interceptors response.setParams(params); requestExec.postProcess(response, httpProcessor, context); // The connection is in or can be brought to a re-usable state. reuse = reuseStrategy.keepAlive(response, context); if (reuse) { // Set the idle duration of this connection long duration = keepAliveStrategy.getKeepAliveDuration(response, context); managedConn.setIdleDuration(duration, TimeUnit.MILLISECONDS); if (this.log.isDebugEnabled()) { if (duration >= 0) { this.log.debug("Connection can be kept alive for " + duration + " ms"); } else { this.log.debug("Connection can be kept alive indefinitely"); } } } RoutedRequest followup = handleResponse(roureq, response, context); if (followup == null) { done = true; } else { if (reuse) { // Make sure the response body is fully consumed, if present HttpEntity entity = response.getEntity(); if (entity != null) { entity.consumeContent(); } // entity consumed above is not an auto-release entity, // need to mark the connection re-usable explicitly managedConn.markReusable(); } else { managedConn.close(); } // check if we can use the same connection for the followup if (!followup.getRoute().equals(roureq.getRoute())) { releaseConnection(); } roureq = followup; } if (managedConn != null && userToken == null) { userToken = userTokenHandler.getUserToken(context); context.setAttribute(ClientContext.USER_TOKEN, userToken); if (userToken != null) { managedConn.setState(userToken); } } } // while not done // check for entity, release connection if possible if ((response == null) || (response.getEntity() == null) || !response.getEntity().isStreaming()) { // connection not needed and (assumed to be) in re-usable state if (reuse) managedConn.markReusable(); releaseConnection(); } else { // install an auto-release entity HttpEntity entity = response.getEntity(); entity = new BasicManagedEntity(entity, managedConn, reuse); response.setEntity(entity); } return response; } catch (HttpException | RuntimeException | IOException ex) { abortConnection(); throw ex; } }
From source file:org.vietspider.net.apache.DefaultRequestDirector.java
public HttpResponse execute(HttpHost target, HttpRequest request, HttpContext context) throws HttpException, IOException { HttpRequest orig = request;//from w w w .j a v a2s .c o m RequestWrapper origWrapper = wrapRequest(orig); origWrapper.setParams(params); HttpRoute origRoute = determineRoute(target, origWrapper, context); virtualHost = (HttpHost) orig.getParams().getParameter(ClientPNames.VIRTUAL_HOST); RoutedRequest roureq = new RoutedRequest(origWrapper, origRoute); long timeout = HttpConnectionParams.getConnectionTimeout(params); boolean reuse = false; boolean done = false; try { HttpResponse response = null; while (!done) { // In this loop, the RoutedRequest may be replaced by a // followup request and route. The request and route passed // in the method arguments will be replaced. The original // request is still available in 'orig'. RequestWrapper wrapper = roureq.getRequest(); HttpRoute route = roureq.getRoute(); response = null; // See if we have a user token bound to the execution context Object userToken = context.getAttribute(ClientContext.USER_TOKEN); // Allocate connection if needed if (managedConn == null) { ClientConnectionRequest connRequest = connManager.requestConnection(route, userToken); if (orig instanceof AbortableHttpRequest) { ((AbortableHttpRequest) orig).setConnectionRequest(connRequest); } try { // if(timeout < 1) timeout = 30000; // System.out.println(" =========== chay vao day roi nhe " + timeout ); // System.out.println(connRequest); managedConn = connRequest.getConnection(timeout, TimeUnit.MILLISECONDS); // System.out.println("get thuc chuyen nay"); } catch (InterruptedException interrupted) { InterruptedIOException iox = new InterruptedIOException(); iox.initCause(interrupted); throw iox; } if (HttpConnectionParams.isStaleCheckingEnabled(params)) { // validate connection if (managedConn.isOpen()) { this.log.debug("Stale connection check"); if (managedConn.isStale()) { this.log.debug("Stale connection detected"); managedConn.close(); } } } } if (orig instanceof AbortableHttpRequest) { ((AbortableHttpRequest) orig).setReleaseTrigger(managedConn); } try { tryConnect(roureq, context); } catch (TunnelRefusedException ex) { if (this.log.isDebugEnabled()) { this.log.debug(ex.getMessage()); } response = ex.getResponse(); break; } // Reset headers on the request wrapper wrapper.resetHeaders(); // Re-write request URI if needed rewriteRequestURI(wrapper, route); // Use virtual host if set target = virtualHost; if (target == null) { target = route.getTargetHost(); } HttpHost proxy = route.getProxyHost(); // Populate the execution context context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, target); context.setAttribute(ExecutionContext.HTTP_PROXY_HOST, proxy); context.setAttribute(ExecutionContext.HTTP_CONNECTION, managedConn); context.setAttribute(ClientContext.TARGET_AUTH_STATE, targetAuthState); context.setAttribute(ClientContext.PROXY_AUTH_STATE, proxyAuthState); // Run request protocol interceptors requestExec.preProcess(wrapper, httpProcessor, context); response = tryExecute(roureq, context); if (response == null) { // Need to start over continue; } // Run response protocol interceptors response.setParams(params); requestExec.postProcess(response, httpProcessor, context); // The connection is in or can be brought to a re-usable state. reuse = reuseStrategy.keepAlive(response, context); if (reuse) { // Set the idle duration of this connection long duration = keepAliveStrategy.getKeepAliveDuration(response, context); if (this.log.isDebugEnabled()) { String s; if (duration > 0) { s = "for " + duration + " " + TimeUnit.MILLISECONDS; } else { s = "indefinitely"; } this.log.debug("Connection can be kept alive " + s); } managedConn.setIdleDuration(duration, TimeUnit.MILLISECONDS); } RoutedRequest followup = handleResponse(roureq, response, context); if (followup == null) { done = true; } else { if (reuse) { // Make sure the response body is fully consumed, if present HttpEntity entity = response.getEntity(); EntityUtils.consume(entity); // entity consumed above is not an auto-release entity, // need to mark the connection re-usable explicitly managedConn.markReusable(); } else { managedConn.close(); } // check if we can use the same connection for the followup if (!followup.getRoute().equals(roureq.getRoute())) { releaseConnection(); } roureq = followup; } if (managedConn != null && userToken == null) { userToken = userTokenHandler.getUserToken(context); context.setAttribute(ClientContext.USER_TOKEN, userToken); if (userToken != null) { managedConn.setState(userToken); } } } // while not done // check for entity, release connection if possible if ((response == null) || (response.getEntity() == null) || !response.getEntity().isStreaming()) { // connection not needed and (assumed to be) in re-usable state if (reuse) managedConn.markReusable(); releaseConnection(); } else { // install an auto-release entity HttpEntity entity = response.getEntity(); entity = new BasicManagedEntity(entity, managedConn, reuse); response.setEntity(entity); } return response; } catch (ConnectionShutdownException ex) { InterruptedIOException ioex = new InterruptedIOException("Connection has been shut down"); ioex.initCause(ex); throw ioex; } catch (HttpException ex) { abortConnection(); throw ex; } catch (IOException ex) { abortConnection(); throw ex; } catch (RuntimeException ex) { abortConnection(); throw ex; } }
From source file:com.akop.bach.parser.XboxLiveParser.java
private GameOverviewInfo parseGameOverview(String url) throws IOException, ParserException { String loadUrl = url;/*from ww w.j av a 2s.co m*/ if (PATTERN_GAME_OVERVIEW_REDIRECTING_URL.matcher(url).find()) { // Redirecting URL; figure out where it's redirecting HttpParams p = mHttpClient.getParams(); try { p.setParameter("http.protocol.max-redirects", 1); HttpGet httpget = new HttpGet(url); HttpContext context = new BasicHttpContext(); HttpResponse response = mHttpClient.execute(httpget, context); HttpEntity entity = response.getEntity(); if (entity != null) entity.consumeContent(); HttpUriRequest request = (HttpUriRequest) context.getAttribute(ExecutionContext.HTTP_REQUEST); try { loadUrl = new URI(url).resolve(request.getURI()).toString(); } catch (URISyntaxException e) { if (App.getConfig().logToConsole()) e.printStackTrace(); } if (App.getConfig().logToConsole()) App.logv("Redirection URL determined to be " + loadUrl); } finally { p.setParameter("http.protocol.max-redirects", 0); } } String page = getResponse(loadUrl.concat("?NoSplash=1")); GameOverviewInfo overview = new GameOverviewInfo(); Matcher m; if (!(m = PATTERN_GAME_OVERVIEW_TITLE.matcher(page)).find()) throw new ParserException(mContext, R.string.error_no_details_available); overview.Title = htmlDecode(m.group(1)); if ((m = PATTERN_GAME_OVERVIEW_DESCRIPTION.matcher(page)).find()) overview.Description = htmlDecode(m.group(1)); if ((m = PATTERN_GAME_OVERVIEW_MANUAL.matcher(page)).find()) overview.ManualUrl = m.group(1); if ((m = PATTERN_GAME_OVERVIEW_ESRB.matcher(page)).find()) { overview.EsrbRatingDescription = htmlDecode(m.group(1)); overview.EsrbRatingIconUrl = m.group(2); } if ((m = PATTERN_GAME_OVERVIEW_BANNER.matcher(page)).find()) overview.BannerUrl = m.group(1); m = PATTERN_GAME_OVERVIEW_IMAGE.matcher(page); while (m.find()) overview.Screenshots.add(m.group(1)); return overview; }
From source file:org.sonatype.nexus.httpclient.internal.NexusRedirectStrategy.java
@Override public boolean isRedirected(final HttpRequest request, final HttpResponse response, final HttpContext context) throws ProtocolException { if (super.isRedirected(request, response, context)) { // code below comes from DefaultRedirectStrategy, as method super.getLocationURI cannot be used // since it modifies context state, and would result in false circular reference detection final Header locationHeader = response.getFirstHeader(LOCATION); if (locationHeader == null) { // got a redirect response, but no location header throw new ProtocolException( "Received redirect response " + response.getStatusLine() + " but no location present"); }//from w w w .j ava 2 s .c om // Some complication here as it appears that something may be null, but its not clear what was null final URI sourceUri = ((HttpUriRequest) request).getURI(); final String sourceScheme = schemeOf(sourceUri); final String sourceHost = hostOf(sourceUri); final URI targetUri = createLocationURI(locationHeader.getValue()); final String targetScheme = schemeOf(targetUri); final String targetHost = hostOf(targetUri); // FIXME: sourceUri might be relative while targetUri is always absolute, which can lead to // false-positive logging because the source scheme (null) differs from the target ("http") // One way to fix this is to resolve the relative sourceUri against the original proxy URL, // but this relies on access to the original configuration. // nag about redirection peculiarities, in any case if (!Objects.equals(sourceScheme, targetScheme)) { if (HTTP.equals(targetScheme)) { // security risk: HTTPS > HTTP downgrade, you are not safe as you think! log.debug("Downgrade from HTTPS to HTTP during redirection {} -> {}", sourceUri, targetUri); } else if ("https".equals(targetScheme) && Objects.equals(sourceHost, targetHost)) { // misconfiguration: your repository configured with wrong protocol and causes performance problems? log.debug("Protocol upgrade during redirection on same host {} -> {}", sourceUri, targetUri); } } // this logic below should trigger only for content fetches made by RRS retrieveItem // hence, we do this ONLY if the HttpRequest is "marked" as such request if (Boolean.TRUE == context.getAttribute(CONTENT_RETRIEVAL_MARKER_KEY)) { if (targetUri.getPath().endsWith("/")) { log.debug("Not following redirection to index {} -> {}", sourceUri, targetUri); return false; } } log.debug("Following redirection {} -> {}", sourceUri, targetUri); return true; } return false; }
From source file:org.robolectric.shadows.httpclient.DefaultRequestDirector.java
/** * Creates a tunnel to the target server. * The connection must be established to the (last) proxy. * A CONNECT request for tunnelling through the proxy will * be created and sent, the response received and checked. * This method does <i>not</i> update the connection with * information about the tunnel, that is left to the caller. * * @param route the route to establish * @param context the context for request execution * * @return <code>true</code> if the tunnelled route is secure, * <code>false</code> otherwise. * The implementation here always returns <code>false</code>, * but derived classes may override. * * @throws HttpException in case of a problem * @throws IOException in case of an IO problem *//* ww w . j av a2 s . c om*/ protected boolean createTunnelToTarget(HttpRoute route, HttpContext context) throws HttpException, IOException { HttpHost proxy = route.getProxyHost(); HttpHost target = route.getTargetHost(); HttpResponse response = null; boolean done = false; while (!done) { done = true; if (!this.managedConn.isOpen()) { this.managedConn.open(route, context, this.params); } HttpRequest connect = createConnectRequest(route, context); connect.setParams(this.params); // Populate the execution context context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, target); context.setAttribute(ExecutionContext.HTTP_PROXY_HOST, proxy); context.setAttribute(ExecutionContext.HTTP_CONNECTION, managedConn); context.setAttribute(ClientContext.TARGET_AUTH_STATE, targetAuthState); context.setAttribute(ClientContext.PROXY_AUTH_STATE, proxyAuthState); context.setAttribute(ExecutionContext.HTTP_REQUEST, connect); this.requestExec.preProcess(connect, this.httpProcessor, context); response = this.requestExec.execute(connect, this.managedConn, context); response.setParams(this.params); this.requestExec.postProcess(response, this.httpProcessor, context); int status = response.getStatusLine().getStatusCode(); if (status < 200) { throw new HttpException("Unexpected response to CONNECT request: " + response.getStatusLine()); } CredentialsProvider credsProvider = (CredentialsProvider) context .getAttribute(ClientContext.CREDS_PROVIDER); if (credsProvider != null && HttpClientParams.isAuthenticating(params)) { if (this.proxyAuthHandler.isAuthenticationRequested(response, context)) { this.log.debug("Proxy requested authentication"); Map<String, Header> challenges = this.proxyAuthHandler.getChallenges(response, context); try { processChallenges(challenges, this.proxyAuthState, this.proxyAuthHandler, response, context); } catch (AuthenticationException ex) { if (this.log.isWarnEnabled()) { this.log.warn("Authentication error: " + ex.getMessage()); break; } } updateAuthState(this.proxyAuthState, proxy, credsProvider); if (this.proxyAuthState.getCredentials() != null) { done = false; // Retry request if (this.reuseStrategy.keepAlive(response, context)) { this.log.debug("Connection kept alive"); // Consume response content HttpEntity entity = response.getEntity(); if (entity != null) { entity.consumeContent(); } } else { this.managedConn.close(); } } } else { // Reset proxy auth scope this.proxyAuthState.setAuthScope(null); } } } int status = response.getStatusLine().getStatusCode(); // can't be null if (status > 299) { // Buffer response content HttpEntity entity = response.getEntity(); if (entity != null) { response.setEntity(new BufferedHttpEntity(entity)); } this.managedConn.close(); throw new TunnelRefusedException("CONNECT refused by proxy: " + response.getStatusLine(), response); } this.managedConn.markReusable(); // How to decide on security of the tunnelled connection? // The socket factory knows only about the segment to the proxy. // Even if that is secure, the hop to the target may be insecure. // Leave it to derived classes, consider insecure by default here. return false; }
From source file:it.staiger.jmeter.protocol.http.sampler.HTTPHC4DynamicFilePost.java
@Override protected HTTPSampleResult sample(URL url, String method, boolean areFollowingRedirect, int frameDepth) { if (log.isDebugEnabled()) { log.debug("Start : sample " + url.toString()); log.debug("method " + method + " followingRedirect " + areFollowingRedirect + " depth " + frameDepth); }/*from ww w . ja v a2 s . c o m*/ HTTPSampleResult res = createSampleResult(url, method); HttpClient httpClient = setupClient(url, res); HttpRequestBase httpRequest = null; try { URI uri = url.toURI(); if (method.equals(HTTPConstants.POST)) { httpRequest = new HttpPost(uri); } else { throw new IllegalArgumentException("Unexpected method: '" + method + "'"); } setupRequest(url, httpRequest, res); // can throw IOException } catch (Exception e) { res.sampleStart(); res.sampleEnd(); errorResult(e, res); return res; } HttpContext localContext = new BasicHttpContext(); res.sampleStart(); final CacheManager cacheManager = getCacheManager(); if (cacheManager != null && HTTPConstants.GET.equalsIgnoreCase(method)) { if (cacheManager.inCache(url)) { return updateSampleResultForResourceInCache(res); } } try { currentRequest = httpRequest; // attach POST data String postBody = sendPostData((HttpPost) httpRequest); res.setQueryString(postBody); // perform the sample HttpResponse httpResponse = executeRequest(httpClient, httpRequest, localContext, url); // parse response and fill the SampleResult with all info parseResponse(httpRequest, localContext, res, httpResponse); // Store any cookies received in the cookie manager: saveConnectionCookies(httpResponse, res.getURL(), getCookieManager()); // Save cache information if (cacheManager != null) { cacheManager.saveDetails(httpResponse, res); } // Follow redirects and download page resources if appropriate: res = resultProcessing(areFollowingRedirect, frameDepth, res); } catch (IOException e) { log.debug("IOException", e); if (res.getEndTime() == 0) { res.sampleEnd(); } // pick up headers if failed to execute the request if (res.getRequestHeaders() != null) { log.debug("Overwriting request old headers: " + res.getRequestHeaders()); } res.setRequestHeaders( getConnectionHeaders((HttpRequest) localContext.getAttribute(ExecutionContext.HTTP_REQUEST))); errorResult(e, res); return res; } catch (RuntimeException e) { log.debug("RuntimeException", e); if (res.getEndTime() == 0) { res.sampleEnd(); } errorResult(e, res); return res; } finally { currentRequest = null; } return res; }
From source file:crawlercommons.fetcher.SimpleHttpFetcher.java
private FetchedResult doRequest(HttpRequestBase request, String url, Payload payload) throws BaseFetchException { LOGGER.trace("Fetching " + url); HttpResponse response;/*from w w w .j ava 2s . c o m*/ long readStartTime; Metadata headerMap = new Metadata(); String redirectedUrl = null; String newBaseUrl = null; int numRedirects = 0; boolean needAbort = true; String contentType = ""; String mimeType = ""; String hostAddress = null; // Create a local instance of cookie store, and bind to local context // Without this we get killed w/lots of threads, due to sync() on single cookie store. HttpContext localContext = new BasicHttpContext(); CookieStore cookieStore = new BasicCookieStore(); localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore); StringBuilder fetchTrace = null; if (LOGGER.isTraceEnabled()) { fetchTrace = new StringBuilder("Fetched url: " + url); } try { request.setURI(new URI(url)); readStartTime = System.currentTimeMillis(); response = _httpClient.execute(request, localContext); Header[] headers = response.getAllHeaders(); for (Header header : headers) { headerMap.add(header.getName(), header.getValue()); } int httpStatus = response.getStatusLine().getStatusCode(); if (LOGGER.isTraceEnabled()) { fetchTrace.append("; status code: " + httpStatus); if (headerMap.get(HttpHeaders.CONTENT_LENGTH) != null) { fetchTrace.append("; Content-Length: " + headerMap.get(HttpHeaders.CONTENT_LENGTH)); } if (headerMap.get(HttpHeaders.LOCATION) != null) { fetchTrace.append("; Location: " + headerMap.get(HttpHeaders.LOCATION)); } } if ((httpStatus < 200) || (httpStatus >= 300)) { // We can't just check against SC_OK, as some wackos return 201, 202, etc throw new HttpFetchException(url, "Error fetching " + url, httpStatus, headerMap); } redirectedUrl = extractRedirectedUrl(url, localContext); URI permRedirectUri = (URI) localContext.getAttribute(PERM_REDIRECT_CONTEXT_KEY); if (permRedirectUri != null) { newBaseUrl = permRedirectUri.toURL().toExternalForm(); } Integer redirects = (Integer) localContext.getAttribute(REDIRECT_COUNT_CONTEXT_KEY); if (redirects != null) { numRedirects = redirects.intValue(); } hostAddress = (String) (localContext.getAttribute(HOST_ADDRESS)); if (hostAddress == null) { throw new UrlFetchException(url, "Host address not saved in context"); } Header cth = response.getFirstHeader(HttpHeaders.CONTENT_TYPE); if (cth != null) { contentType = cth.getValue(); } // Check if we should abort due to mime-type filtering. Note that this will fail if the server // doesn't report a mime-type, but that's how we want it as this configuration is typically // used when only a subset of parsers are installed/enabled, so we don't want the auto-detect // code in Tika to get triggered & try to process an unsupported type. If you want unknown // mime-types from the server to be processed, set "" as one of the valid mime-types in FetcherPolicy. mimeType = getMimeTypeFromContentType(contentType); Set<String> mimeTypes = getValidMimeTypes(); if ((mimeTypes != null) && (mimeTypes.size() > 0)) { if (!mimeTypes.contains(mimeType)) { throw new AbortedFetchException(url, "Invalid mime-type: " + mimeType, AbortedFetchReason.INVALID_MIMETYPE); } } needAbort = false; } catch (ClientProtocolException e) { // Oleg guarantees that no abort is needed in the case of an IOException (which is is a subclass of) needAbort = false; // If the root case was a "too many redirects" error, we want to map this to a specific // exception that contains the final redirect. if (e.getCause() instanceof MyRedirectException) { MyRedirectException mre = (MyRedirectException) e.getCause(); String redirectUrl = url; try { redirectUrl = mre.getUri().toURL().toExternalForm(); } catch (MalformedURLException e2) { LOGGER.warn("Invalid URI saved during redirect handling: " + mre.getUri()); } throw new RedirectFetchException(url, redirectUrl, mre.getReason()); } else if (e.getCause() instanceof RedirectException) { throw new RedirectFetchException(url, extractRedirectedUrl(url, localContext), RedirectExceptionReason.TOO_MANY_REDIRECTS); } else { throw new IOFetchException(url, e); } } catch (IOException e) { // Oleg guarantees that no abort is needed in the case of an IOException needAbort = false; if (e instanceof ConnectionPoolTimeoutException) { // Should never happen, so let's dump some info about the connection pool. ThreadSafeClientConnManager cm = (ThreadSafeClientConnManager) _httpClient.getConnectionManager(); int numConnections = cm.getConnectionsInPool(); cm.closeIdleConnections(0, TimeUnit.MILLISECONDS); LOGGER.error(String.format( "Got ConnectionPoolTimeoutException: %d connections before, %d after idle close", numConnections, cm.getConnectionsInPool())); } throw new IOFetchException(url, e); } catch (URISyntaxException e) { throw new UrlFetchException(url, e.getMessage()); } catch (IllegalStateException e) { throw new UrlFetchException(url, e.getMessage()); } catch (BaseFetchException e) { throw e; } catch (Exception e) { // Map anything else to a generic IOFetchException // TODO KKr - create generic fetch exception throw new IOFetchException(url, new IOException(e)); } finally { safeAbort(needAbort, request); } // Figure out how much data we want to try to fetch. int maxContentSize = getMaxContentSize(mimeType); int targetLength = maxContentSize; boolean truncated = false; String contentLengthStr = headerMap.get(HttpHeaders.CONTENT_LENGTH); if (contentLengthStr != null) { try { int contentLength = Integer.parseInt(contentLengthStr); if (contentLength > targetLength) { truncated = true; } else { targetLength = contentLength; } } catch (NumberFormatException e) { // Ignore (and log) invalid content length values. LOGGER.warn("Invalid content length in header: " + contentLengthStr); } } // Now finally read in response body, up to targetLength bytes. // Note that entity might be null, for zero length responses. byte[] content = new byte[0]; long readRate = 0; HttpEntity entity = response.getEntity(); needAbort = true; if (entity != null) { InputStream in = null; try { in = entity.getContent(); byte[] buffer = new byte[BUFFER_SIZE]; int bytesRead = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(DEFAULT_BYTEARRAY_SIZE); int readRequests = 0; int minResponseRate = getMinResponseRate(); // TODO KKr - we need to monitor the rate while reading a // single block. Look at HttpClient // metrics support for how to do this. Once we fix this, fix // the test to read a smaller (< 20K) // chuck of data. while ((totalRead < targetLength) && ((bytesRead = in.read(buffer, 0, Math.min(buffer.length, targetLength - totalRead))) != -1)) { readRequests += 1; totalRead += bytesRead; out.write(buffer, 0, bytesRead); // Assume read time is at least one millisecond, to avoid DBZ exception. long totalReadTime = Math.max(1, System.currentTimeMillis() - readStartTime); readRate = (totalRead * 1000L) / totalReadTime; // Don't bail on the first read cycle, as we can get a hiccup starting out. // Also don't bail if we've read everything we need. if ((readRequests > 1) && (totalRead < targetLength) && (readRate < minResponseRate)) { throw new AbortedFetchException(url, "Slow response rate of " + readRate + " bytes/sec", AbortedFetchReason.SLOW_RESPONSE_RATE); } // Check to see if we got interrupted. if (Thread.interrupted()) { throw new AbortedFetchException(url, AbortedFetchReason.INTERRUPTED); } } content = out.toByteArray(); needAbort = truncated || (in.available() > 0); } catch (IOException e) { // We don't need to abort if there's an IOException throw new IOFetchException(url, e); } finally { safeAbort(needAbort, request); safeClose(in); } } // Toss truncated image content. if ((truncated) && (!isTextMimeType(mimeType))) { throw new AbortedFetchException(url, "Truncated image", AbortedFetchReason.CONTENT_SIZE); } // Now see if we need to uncompress the content. String contentEncoding = headerMap.get(HttpHeaders.CONTENT_ENCODING); if (contentEncoding != null) { if (LOGGER.isTraceEnabled()) { fetchTrace.append("; Content-Encoding: " + contentEncoding); } // TODO KKr We might want to just decompress a truncated gzip // containing text (since we have a max content size to save us // from any gzip corruption). We might want to break the following // out into a separate method, by the way (if not refactor this // entire monolithic method). // try { if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) { if (truncated) { throw new AbortedFetchException(url, "Truncated compressed data", AbortedFetchReason.CONTENT_SIZE); } else { ExpandedResult expandedResult = EncodingUtils.processGzipEncoded(content, maxContentSize); truncated = expandedResult.isTruncated(); if ((truncated) && (!isTextMimeType(mimeType))) { throw new AbortedFetchException(url, "Truncated decompressed image", AbortedFetchReason.CONTENT_SIZE); } else { content = expandedResult.getExpanded(); if (LOGGER.isTraceEnabled()) { fetchTrace.append("; unzipped to " + content.length + " bytes"); } } // } else if ("deflate".equals(contentEncoding)) { // content = EncodingUtils.processDeflateEncoded(content); // if (LOGGER.isTraceEnabled()) { // fetchTrace.append("; inflated to " + content.length + " bytes"); // } } } } catch (IOException e) { throw new IOFetchException(url, e); } } // Finally dump out the trace msg we've been building. if (LOGGER.isTraceEnabled()) { LOGGER.trace(fetchTrace.toString()); } // TODO KKr - Save truncated flag in FetchedResult/FetchedDatum. return new FetchedResult(url, redirectedUrl, System.currentTimeMillis(), headerMap, content, contentType, (int) readRate, payload, newBaseUrl, numRedirects, hostAddress); }