List of usage examples for org.apache.http.cookie.params CookieSpecParamBean CookieSpecParamBean
public CookieSpecParamBean(final HttpParams params)
From source file:com.heaptrip.util.http.bixo.fetcher.SimpleHttpFetcher.java
private synchronized void init() { if (_httpClient == null) { // Create and initialize HTTP parameters HttpParams params = new BasicHttpParams(); // TODO KKr - w/4.1, switch to new api (ThreadSafeClientConnManager) // cm.setMaxTotalConnections(_maxThreads); // cm.setDefaultMaxPerRoute(Math.max(10, _maxThreads/10)); ConnManagerParams.setMaxTotalConnections(params, _maxThreads); // Set the maximum time we'll wait for a spare connection in the // connection pool. We // shouldn't actually hit this, as we make sure (in FetcherManager) that // the max number // of active requests doesn't exceed the value returned by getMaxThreads() // here.//from w w w . ja v a 2 s .c o m ConnManagerParams.setTimeout(params, CONNECTION_POOL_TIMEOUT); // Set the socket and connection timeout to be something reasonable. HttpConnectionParams.setSoTimeout(params, _socketTimeout); HttpConnectionParams.setConnectionTimeout(params, _connectionTimeout); // Even with stale checking enabled, a connection can "go stale" between // the check and the // next request. So we still need to handle the case of a closed socket // (from the server side), // and disabling this check improves performance. HttpConnectionParams.setStaleCheckingEnabled(params, false); // FUTURE - set this on a per-route (host) basis when we have per-host // policies for // doing partner crawls. We could define a BixoConnPerRoute class that // supports this. ConnPerRouteBean connPerRoute = new ConnPerRouteBean(_fetcherPolicy.getMaxConnectionsPerHost()); ConnManagerParams.setMaxConnectionsPerRoute(params, connPerRoute); HttpProtocolParams.setVersion(params, _httpVersion); HttpProtocolParams.setUserAgent(params, _userAgent.getUserAgentString()); HttpProtocolParams.setContentCharset(params, "UTF-8"); HttpProtocolParams.setHttpElementCharset(params, "UTF-8"); HttpProtocolParams.setUseExpectContinue(params, true); // TODO KKr - set on connection manager params, or client params? CookieSpecParamBean cookieParams = new CookieSpecParamBean(params); cookieParams.setSingleHeader(true); // Create and initialize scheme registry SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); SSLSocketFactory sf = null; if (sf != null) { sf.setHostnameVerifier(new DummyX509HostnameVerifier()); schemeRegistry.register(new Scheme("https", sf, 443)); } else { LOGGER.warn("No valid SSLContext found for https"); } // Use ThreadSafeClientConnManager since more than one thread will be // using the HttpClient. ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(params, schemeRegistry); _httpClient = new DefaultHttpClient(cm, params); _httpClient.setHttpRequestRetryHandler(new MyRequestRetryHandler(_maxRetryCount)); _httpClient.setRedirectHandler(new MyRedirectHandler(_fetcherPolicy.getRedirectMode())); _httpClient.addRequestInterceptor(new MyRequestInterceptor()); params = _httpClient.getParams(); // FUTURE KKr - support authentication HttpClientParams.setAuthenticating(params, false); HttpClientParams.setCookiePolicy(params, CookiePolicy.BEST_MATCH); ClientParamBean clientParams = new ClientParamBean(params); if (_fetcherPolicy.getMaxRedirects() == 0) { clientParams.setHandleRedirects(false); } else { clientParams.setHandleRedirects(true); clientParams.setMaxRedirects(_fetcherPolicy.getMaxRedirects()); } // Set up default headers. This helps us get back from servers what we // want. HashSet<Header> defaultHeaders = new HashSet<Header>(); defaultHeaders .add(new BasicHeader(HttpHeaderNames.ACCEPT_LANGUAGE, _fetcherPolicy.getAcceptLanguage())); defaultHeaders.add(new BasicHeader(HttpHeaderNames.ACCEPT_CHARSET, DEFAULT_ACCEPT_CHARSET)); defaultHeaders.add(new BasicHeader(HttpHeaderNames.ACCEPT, DEFAULT_ACCEPT)); clientParams.setDefaultHeaders(defaultHeaders); } }
From source file:bixo.fetcher.SimpleHttpFetcher.java
private synchronized void init() { if (_httpClient == null) { // Create and initialize HTTP parameters HttpParams params = new BasicHttpParams(); // TODO KKr - w/4.1, switch to new api (ThreadSafeClientConnManager) // cm.setMaxTotalConnections(_maxThreads); // cm.setDefaultMaxPerRoute(Math.max(10, _maxThreads/10)); ConnManagerParams.setMaxTotalConnections(params, _maxThreads); // Set the maximum time we'll wait for a spare connection in the // connection pool. We // shouldn't actually hit this, as we make sure (in FetcherManager) that // the max number // of active requests doesn't exceed the value returned by getMaxThreads() // here.//w w w .ja v a 2 s . c o m ConnManagerParams.setTimeout(params, CONNECTION_POOL_TIMEOUT); // Set the socket and connection timeout to be something reasonable. HttpConnectionParams.setSoTimeout(params, _socketTimeout); HttpConnectionParams.setConnectionTimeout(params, _connectionTimeout); // Even with stale checking enabled, a connection can "go stale" between // the check and the // next request. So we still need to handle the case of a closed socket // (from the server side), // and disabling this check improves performance. HttpConnectionParams.setStaleCheckingEnabled(params, false); // FUTURE - set this on a per-route (host) basis when we have per-host // policies for // doing partner crawls. We could define a BixoConnPerRoute class that // supports this. ConnPerRouteBean connPerRoute = new ConnPerRouteBean(_fetcherPolicy.getMaxConnectionsPerHost()); ConnManagerParams.setMaxConnectionsPerRoute(params, connPerRoute); HttpProtocolParams.setVersion(params, _httpVersion); HttpProtocolParams.setUserAgent(params, _userAgent.getUserAgentString()); HttpProtocolParams.setContentCharset(params, "UTF-8"); HttpProtocolParams.setHttpElementCharset(params, "UTF-8"); HttpProtocolParams.setUseExpectContinue(params, true); // TODO KKr - set on connection manager params, or client params? CookieSpecParamBean cookieParams = new CookieSpecParamBean(params); cookieParams.setSingleHeader(true); // Create and initialize scheme registry SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); SSLSocketFactory sf = null; for (String contextName : SSL_CONTEXT_NAMES) { try { SSLContext sslContext = SSLContext.getInstance(contextName); sslContext.init(null, new TrustManager[] { new DummyX509TrustManager(null) }, null); sf = new SSLSocketFactory(sslContext); break; } catch (NoSuchAlgorithmException e) { LOGGER.debug("SSLContext algorithm not available: " + contextName); } catch (Exception e) { LOGGER.debug("SSLContext can't be initialized: " + contextName, e); } } if (sf != null) { sf.setHostnameVerifier(new DummyX509HostnameVerifier()); schemeRegistry.register(new Scheme("https", sf, 443)); } else { LOGGER.warn("No valid SSLContext found for https"); } // Use ThreadSafeClientConnManager since more than one thread will be // using the HttpClient. ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(params, schemeRegistry); _httpClient = new DefaultHttpClient(cm, params); _httpClient.setHttpRequestRetryHandler(new MyRequestRetryHandler(_maxRetryCount)); _httpClient.setRedirectHandler(new MyRedirectHandler(_fetcherPolicy.getRedirectMode())); _httpClient.addRequestInterceptor(new MyRequestInterceptor()); params = _httpClient.getParams(); // FUTURE KKr - support authentication HttpClientParams.setAuthenticating(params, false); HttpClientParams.setCookiePolicy(params, CookiePolicy.BEST_MATCH); ClientParamBean clientParams = new ClientParamBean(params); if (_fetcherPolicy.getMaxRedirects() == 0) { clientParams.setHandleRedirects(false); } else { clientParams.setHandleRedirects(true); clientParams.setMaxRedirects(_fetcherPolicy.getMaxRedirects()); } // Set up default headers. This helps us get back from servers what we // want. HashSet<Header> defaultHeaders = new HashSet<Header>(); defaultHeaders .add(new BasicHeader(HttpHeaderNames.ACCEPT_LANGUAGE, _fetcherPolicy.getAcceptLanguage())); defaultHeaders.add(new BasicHeader(HttpHeaderNames.ACCEPT_CHARSET, DEFAULT_ACCEPT_CHARSET)); defaultHeaders.add(new BasicHeader(HttpHeaderNames.ACCEPT, DEFAULT_ACCEPT)); clientParams.setDefaultHeaders(defaultHeaders); } }
From source file:crawlercommons.fetcher.SimpleHttpFetcher.java
private synchronized void init() { if (_httpClient == null) { // Create and initialize HTTP parameters HttpParams params = new BasicHttpParams(); // TODO KKr - w/4.1, switch to new api (ThreadSafeClientConnManager) // cm.setMaxTotalConnections(_maxThreads); // cm.setDefaultMaxPerRoute(Math.max(10, _maxThreads/10)); ConnManagerParams.setMaxTotalConnections(params, _maxThreads); // Set the maximum time we'll wait for a spare connection in the connection pool. We // shouldn't actually hit this, as we make sure (in FetcherManager) that the max number // of active requests doesn't exceed the value returned by getMaxThreads() here. ConnManagerParams.setTimeout(params, CONNECTION_POOL_TIMEOUT); // Set the socket and connection timeout to be something reasonable. HttpConnectionParams.setSoTimeout(params, _socketTimeout); HttpConnectionParams.setConnectionTimeout(params, _connectionTimeout); // Even with stale checking enabled, a connection can "go stale" between the check and the // next request. So we still need to handle the case of a closed socket (from the server side), // and disabling this check improves performance. HttpConnectionParams.setStaleCheckingEnabled(params, false); // FUTURE - set this on a per-route (host) basis when we have per-host policies for // doing partner crawls. We could define a BixoConnPerRoute class that supports this. ConnPerRouteBean connPerRoute = new ConnPerRouteBean(getMaxConnectionsPerHost()); ConnManagerParams.setMaxConnectionsPerRoute(params, connPerRoute); HttpProtocolParams.setVersion(params, _httpVersion); HttpProtocolParams.setUserAgent(params, _userAgent.getUserAgentString()); HttpProtocolParams.setContentCharset(params, "UTF-8"); HttpProtocolParams.setHttpElementCharset(params, "UTF-8"); HttpProtocolParams.setUseExpectContinue(params, true); // TODO KKr - set on connection manager params, or client params? CookieSpecParamBean cookieParams = new CookieSpecParamBean(params); cookieParams.setSingleHeader(true); // Create and initialize scheme registry SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); SSLSocketFactory sf = null; for (String contextName : SSL_CONTEXT_NAMES) { try { SSLContext sslContext = SSLContext.getInstance(contextName); sslContext.init(null, new TrustManager[] { new DummyX509TrustManager(null) }, null); sf = new SSLSocketFactory(sslContext); break; } catch (NoSuchAlgorithmException e) { LOGGER.debug("SSLContext algorithm not available: " + contextName); } catch (Exception e) { LOGGER.debug("SSLContext can't be initialized: " + contextName, e); }/*from w w w . jav a 2 s . c o m*/ } if (sf != null) { sf.setHostnameVerifier(new DummyX509HostnameVerifier()); schemeRegistry.register(new Scheme("https", sf, 443)); } else { LOGGER.warn("No valid SSLContext found for https"); } // Use ThreadSafeClientConnManager since more than one thread will be using the HttpClient. ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(params, schemeRegistry); _httpClient = new DefaultHttpClient(cm, params); _httpClient.setHttpRequestRetryHandler(new MyRequestRetryHandler(_maxRetryCount)); _httpClient.setRedirectHandler(new MyRedirectHandler(getRedirectMode())); _httpClient.addRequestInterceptor(new MyRequestInterceptor()); params = _httpClient.getParams(); // FUTURE KKr - support authentication HttpClientParams.setAuthenticating(params, false); HttpClientParams.setCookiePolicy(params, CookiePolicy.BEST_MATCH); ClientParamBean clientParams = new ClientParamBean(params); if (getMaxRedirects() == 0) { clientParams.setHandleRedirects(false); } else { clientParams.setHandleRedirects(true); clientParams.setMaxRedirects(getMaxRedirects()); } // Set up default headers. This helps us get back from servers what we want. HashSet<Header> defaultHeaders = new HashSet<Header>(); defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, getAcceptLanguage())); defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT_CHARSET, DEFAULT_ACCEPT_CHARSET)); defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT_ENCODING, DEFAULT_ACCEPT_ENCODING)); defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT, DEFAULT_ACCEPT)); clientParams.setDefaultHeaders(defaultHeaders); } }
From source file:org.apache.camel.component.http4.HttpComponent.java
protected HttpParams configureHttpParams(Map<String, Object> parameters) throws Exception { HttpParams clientParams = new BasicHttpParams(); AuthParamBean authParamBean = new AuthParamBean(clientParams); IntrospectionSupport.setProperties(authParamBean, parameters, "httpClient."); ClientParamBean clientParamBean = new ClientParamBean(clientParams); IntrospectionSupport.setProperties(clientParamBean, parameters, "httpClient."); ConnConnectionParamBean connConnectionParamBean = new ConnConnectionParamBean(clientParams); IntrospectionSupport.setProperties(connConnectionParamBean, parameters, "httpClient."); ConnManagerParamBean connManagerParamBean = new ConnManagerParamBean(clientParams); IntrospectionSupport.setProperties(connManagerParamBean, parameters, "httpClient."); ConnRouteParamBean connRouteParamBean = new ConnRouteParamBean(clientParams); IntrospectionSupport.setProperties(connRouteParamBean, parameters, "httpClient."); CookieSpecParamBean cookieSpecParamBean = new CookieSpecParamBean(clientParams); IntrospectionSupport.setProperties(cookieSpecParamBean, parameters, "httpClient."); HttpConnectionParamBean httpConnectionParamBean = new HttpConnectionParamBean(clientParams); IntrospectionSupport.setProperties(httpConnectionParamBean, parameters, "httpClient."); HttpProtocolParamBean httpProtocolParamBean = new HttpProtocolParamBean(clientParams); IntrospectionSupport.setProperties(httpProtocolParamBean, parameters, "httpClient."); return clientParams; }