List of usage examples for org.apache.commons.httpclient HostConfiguration getParams
public HostParams getParams()
From source file:org.apache.nutch.protocol.httpclient.proxy.Http.java
/** * Configures the HTTP client/*w w w. j av a 2s . co m*/ */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. //ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); ProtocolSocketFactory factory = new DummySSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(maxThreadsTotal); // executeMethod(HttpMethod) seems to ignore the connection timeout on the // connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); ArrayList<Header> headers = new ArrayList<Header>(); // Set the User Agent in the header //headers.add(new Header("User-Agent", userAgent)); //NUTCH-1941 // prefer English headers.add(new Header("Accept-Language", acceptLanguage)); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")); // accept gzipped content headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); // HTTP proxy server details if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); if (proxyUsername.length() > 0) { AuthScope proxyAuthScope = getAuthScope(this.proxyHost, this.proxyPort, this.proxyRealm); NTCredentials proxyCredentials = new NTCredentials(this.proxyUsername, this.proxyPassword, Http.agentHost, this.proxyRealm); client.getState().setProxyCredentials(proxyAuthScope, proxyCredentials); } } }
From source file:org.apache.nutch.protocol.webdriver.Http.java
/** * Configures the HTTP client/*from w w w.ja v a2 s. c om*/ */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(maxThreadsTotal); // executeMethod(HttpMethod) seems to ignore the connection timeout on // the connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); } ArrayList<Header> headers = new ArrayList<Header>(); // prefer English headers.add(new Header("Accept-Language", "en-us,en-gb,en;q=0.7,*;q=0.3")); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")); // accept gzipped content // headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); }
From source file:org.deri.any23.http.DefaultHTTPClient.java
private void ensureClientInitialized() { if (configuration == null) throw new IllegalStateException("client must be initialized first."); if (client != null) return;/*from ww w . j a va 2 s. c o m*/ client = new HttpClient(manager); HttpConnectionManager connectionManager = client.getHttpConnectionManager(); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(configuration.getDefaultTimeout()); params.setSoTimeout(configuration.getDefaultTimeout()); params.setMaxTotalConnections(configuration.getMaxConnections()); HostConfiguration hostConf = client.getHostConfiguration(); List<Header> headers = new ArrayList<Header>(); headers.add(new Header("User-Agent", configuration.getUserAgent())); if (configuration.getAcceptHeader() != null) { headers.add(new Header("Accept", configuration.getAcceptHeader())); } headers.add(new Header("Accept-Language", "en-us,en-gb,en,*;q=0.3")); headers.add(new Header("Accept-Charset", "utf-8,iso-8859-1;q=0.7,*;q=0.5")); // headers.add(new Header("Accept-Encoding", "x-gzip, gzip")); hostConf.getParams().setParameter("http.default-headers", headers); }
From source file:org.eclipse.smila.connectivity.framework.crawler.web.http.Http.java
/** * Loads HTTP client configuration for this web site. *///from w w w . j a va 2 s. co m private void configureClient() { final HttpConnectionManagerParams params = s_connectionManager.getParams(); if (_timeout != 0) { params.setConnectionTimeout(_timeout); params.setSoTimeout(_timeout); } else { params.setConnectionTimeout(_connectTimeout); params.setSoTimeout(_readTimeout); } params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); final HostConfiguration hostConf = s_client.getHostConfiguration(); final List<Header> headers = new ArrayList<Header>(); // prefer English headers.add(new Header("Accept-Language", "en-us,en-gb,en;q=0.7,*;q=0.3")); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8")); // accept GZIP content headers.add(new Header("Accept-Encoding", "x-gzip, gzip")); final String[] webSiteHeaders = getConf().get(HttpProperties.HEADERS).split(SEMICOLON); for (String header : webSiteHeaders) { final String[] headerInformation = header.split(COLON); if (headerInformation.length > 2) { headers.add(new Header(headerInformation[0].trim(), headerInformation[1].trim())); } } hostConf.getParams().setParameter("http.default-headers", headers); if (_useProxy) { hostConf.setProxy(_proxyHost, _proxyPort); if (_proxyLogin.length() > 0) { final Credentials proxyCreds = new UsernamePasswordCredentials(_proxyLogin, _proxyPassword); s_client.getState().setProxyCredentials(new AuthScope(AuthScope.ANY), proxyCreds); } } final List<Rfc2617Authentication> httpAuthentications = _authentication.getRfc2617Authentications(); for (Rfc2617Authentication auth : httpAuthentications) { s_client.getState().setCredentials( new AuthScope(auth.getHost(), Integer.valueOf(auth.getPort()), auth.getRealm()), new UsernamePasswordCredentials(auth.getLogin(), auth.getPassword())); } final SslCertificateAuthentication sslAuth = _authentication.getSslCertificateAuthentication(); if (sslAuth != null) { try { final URL truststoreURL = new File(sslAuth.getTruststoreUrl()).toURL(); final URL keystoreURL = new File(sslAuth.getKeystoreUrl()).toURL(); final ProtocolSocketFactory sslFactory = new AuthSSLProtocolSocketFactory(keystoreURL, sslAuth.getKeystorePassword(), truststoreURL, sslAuth.getTruststorePassword()); _https = new Protocol(sslAuth.getProtocolName(), sslFactory, Integer.valueOf(sslAuth.getPort())); Protocol.registerProtocol(sslAuth.getProtocolName(), _https); } catch (MalformedURLException exception) { LOG.error("unable to bind https protocol" + exception.toString()); } } }
From source file:org.globus.axis.transport.commons.CommonsHttpConnectionManager.java
public HttpConnection getConnectionWithTimeout(HostConfiguration hostConfiguration, long timeout) { ExtendedHostConfiguration extendedHostConfiguration = new ExtendedHostConfiguration(hostConfiguration, this.hostConfigurationParams); ConnectionPool pool = getConnectionPool(extendedHostConfiguration); ExtendedHttpConnection httpConnection = pool.getPooledConnection(); if (httpConnection == null) { // not in the pool - create a new connection httpConnection = getNewConnection(extendedHostConfiguration); httpConnection.setFromPool(false); } else {//from ww w .ja v a 2s. c o m httpConnection.setFromPool(true); } if (this.staleChecking) { // install our retry handler hostConfiguration.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, httpConnection.getRetryHandler()); } return httpConnection; }
From source file:org.globus.axis.transport.commons.CommonsHttpConnectionManager.java
private ExtendedHttpConnection getNewConnection(HostConfiguration hostConfiguration) { ExtendedHttpConnection httpConnection = new ExtendedHttpConnection(hostConfiguration, this.staleChecking); httpConnection.setHttpConnectionManager(this); HttpConnectionParams connectionParams = httpConnection.getParams(); connectionParams.setDefaults(this.params); if (this.hostConfigurationParams != null) { HostParams hostParams = hostConfiguration.getParams(); for (int i = 0; i < this.hostConfigurationParams.length; i++) { String key = this.hostConfigurationParams[i]; Object value = hostParams.getParameter(key); if (value != null) { connectionParams.setParameter(key, value); }/* www.ja v a 2 s .co m*/ } } if (logger.isDebugEnabled()) { logger.debug("got new connection: " + httpConnection); } return httpConnection; }
From source file:org.globus.axis.transport.commons.HTTPSSender.java
protected HostConfiguration getHostConfiguration(HttpClient client, MessageContext context, URL targetURL) { HostConfiguration config = super.getHostConfiguration(client, context, targetURL); // handle disable chunking option Boolean prop = (Boolean) context.getProperty(HTTPUtils.DISABLE_CHUNKING); if (prop != null) { client.getParams().setParameter(HTTPUtils.DISABLE_CHUNKING, prop); }/*from w ww . ja v a2 s.co m*/ for (int i = 0; i < PARAMS.length; i++) { Object value = context.getProperty(PARAMS[i]); if (value != null) { config.getParams().setParameter(PARAMS[i], value); } } return config; }
From source file:org.globus.axis.transport.commons.tests.CommonsHttpConnectionManagerTest.java
public void testConnectionReuseWithParams() throws Exception { CommonsHttpConnectionManager manager = new CommonsHttpConnectionManager(PARAMS); HostConfiguration h1 = new HostConfiguration(); h1.setHost(address, server1.getLocalPort()); h1.getParams().setParameter("A", "foo"); h1.getParams().setParameter("B", "bar"); h1.getParams().setParameter("C", "fff"); HttpConnection c1 = manager.getConnection(h1); assertTrue(!c1.isOpen());/*from www .ja v a 2s .c o m*/ c1.open(); c1.releaseConnection(); HostConfiguration h2 = new HostConfiguration(); h2.setHost(address, server1.getLocalPort()); h2.getParams().setParameter("A", "foo"); h2.getParams().setParameter("B", "bar"); // still should be reused since C is not checked param h2.getParams().setParameter("C", "ggg"); HttpConnection c2 = manager.getConnection(h2); // connection should have been released // so c2 is c1 assertTrue(h2.equals(h1)); assertTrue(c2.isOpen()); assertTrue(c2 == c1); HttpConnection c3 = manager.getConnection(h2); // new connection becuase it wasn't released assertTrue(c3 != c1); assertTrue(c3 != c2); assertTrue(!c3.isOpen()); c2.releaseConnection(); c3.releaseConnection(); // this one does not have params HostConfiguration h4 = new HostConfiguration(); h4.setHost(address, server1.getLocalPort()); HttpConnection c4 = manager.getConnection(h4); // new connection assertTrue(c4 != c1); assertTrue(c4 != c2); assertTrue(c4 != c3); assertTrue(!c4.isOpen()); c4.open(); c4.releaseConnection(); // this one only has B parameter HostConfiguration h5 = new HostConfiguration(); h5.setHost(address, server1.getLocalPort()); h5.getParams().setParameter("B", "bar"); HttpConnection c5 = manager.getConnection(h5); // also a new connection assertTrue(c5 != c1); assertTrue(c5 != c2); assertTrue(c5 != c3); assertTrue(c5 != c4); assertTrue(!c5.isOpen()); c5.open(); c5.releaseConnection(); // this one only has different B parameter HostConfiguration h6 = new HostConfiguration(); h6.setHost(address, server1.getLocalPort()); h6.getParams().setParameter("A", "fooo"); h6.getParams().setParameter("B", "bar"); HttpConnection c6 = manager.getConnection(h6); assertTrue(c6 != c1); assertTrue(c6 != c2); assertTrue(c6 != c3); assertTrue(c6 != c4); assertTrue(c6 != c5); assertTrue(!c6.isOpen()); c6.open(); c6.releaseConnection(); }
From source file:org.gss_project.gss.web.client.TestClient.java
public static void main(String[] args) { String user = "ebstest@grnet-hq.admin.grnet.gr"; String token = "PcxaZ/4oIqCqIvCYgsUcKr1hAFcsW40G3kcWJSRPJV5GjzoNuo8RsA=="; String host = "pithos.grnet.gr"; String restPath = "/pithos/rest"; String path = "/" + user + "/files/"; String now = DateUtil.formatDate(new Date()); String signature = sign("GET", now, path, token); HttpClient client = new HttpClient(); HostConfiguration hostconfig = new HostConfiguration(); hostconfig.setHost(host);/*from w ww .java 2 s . c om*/ HttpMethod method = new GetMethod(restPath + path); Collection<Header> headers = new ArrayList<Header>(); Header auth = new Header("Authorization", user + " " + signature); headers.add(auth); Header date = new Header("X-GSS-Date", now); headers.add(date); System.out.println(headers.toString()); hostconfig.getParams().setParameter("http.default-headers", headers); try { // Execute the method. int statusCode = client.executeMethod(hostconfig, method); if (statusCode != HttpStatus.SC_OK) System.err.println("Method failed: " + method.getStatusLine()); // Read the response body. byte[] responseBody = method.getResponseBody(); // Deal with the response. // Use caution: ensure correct character encoding and is not binary data System.out.println(new String(responseBody)); } catch (HttpException e) { System.err.println("Fatal protocol violation: " + e.getMessage()); e.printStackTrace(); } catch (IOException e) { System.err.println("Fatal transport error: " + e.getMessage()); e.printStackTrace(); } finally { // Release the connection. method.releaseConnection(); } }
From source file:org.sonatype.nexus.proxy.storage.remote.commonshttpclient.HttpClientProxyUtil.java
public static void applyProxyToHttpClient(HttpClient httpClient, RemoteStorageContext ctx, Logger logger) { httpClient.setHttpConnectionManager(new CustomMultiThreadedHttpConnectionManager()); // getting the timeout from RemoteStorageContext. The value we get depends on per-repo and global settings. // The value will "cascade" from repo level to global level, see imple of it. int timeout = ctx.getRemoteConnectionSettings().getConnectionTimeout(); // getting the connection pool size, using a little trick to allow us "backdoor" to tune it using system // properties, but defaulting it to the same we had before (httpClient defaults) int connectionPoolSize = SystemPropertiesHelper.getInteger(CONNECTION_POOL_SIZE_KEY, MultiThreadedHttpConnectionManager.DEFAULT_MAX_TOTAL_CONNECTIONS); httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(timeout); httpClient.getHttpConnectionManager().getParams().setSoTimeout(timeout); // httpClient.getHttpConnectionManager().getParams().setTcpNoDelay( true ); httpClient.getHttpConnectionManager().getParams().setMaxTotalConnections(connectionPoolSize); // NOTE: connPool is _per_ repo, hence all of those will connect to same host (unless mirrors are used) // so, we are violating intentionally the RFC and we let the whole pool size to chase same host httpClient.getHttpConnectionManager().getParams() .setMaxConnectionsPerHost(HostConfiguration.ANY_HOST_CONFIGURATION, connectionPoolSize); // Setting auth if needed HostConfiguration httpConfiguration = httpClient.getHostConfiguration(); // BASIC and DIGEST auth only RemoteAuthenticationSettings ras = ctx.getRemoteAuthenticationSettings(); boolean isSimpleAuthUsed = false; boolean isNtlmUsed = false; if (ras != null) { List<String> authPrefs = new ArrayList<String>(2); authPrefs.add(AuthPolicy.DIGEST); authPrefs.add(AuthPolicy.BASIC); if (ras instanceof ClientSSLRemoteAuthenticationSettings) { // ClientSSLRemoteAuthenticationSettings cras = (ClientSSLRemoteAuthenticationSettings) ras; // TODO - implement this } else if (ras instanceof NtlmRemoteAuthenticationSettings) { NtlmRemoteAuthenticationSettings nras = (NtlmRemoteAuthenticationSettings) ras; // Using NTLM auth, adding it as first in policies authPrefs.add(0, AuthPolicy.NTLM); logger(logger).info("... authentication setup for NTLM domain '{}'", nras.getNtlmDomain()); httpConfiguration.setHost(nras.getNtlmHost()); httpClient.getState().setCredentials(AuthScope.ANY, new NTCredentials(nras.getUsername(), nras.getPassword(), nras.getNtlmHost(), nras.getNtlmDomain())); isNtlmUsed = true;/*from w w w . j av a 2 s. c om*/ } else if (ras instanceof UsernamePasswordRemoteAuthenticationSettings) { UsernamePasswordRemoteAuthenticationSettings uras = (UsernamePasswordRemoteAuthenticationSettings) ras; // Using Username/Pwd auth, will not add NTLM logger(logger).info("... authentication setup for remote storage with username '{}'", uras.getUsername()); httpClient.getState().setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(uras.getUsername(), uras.getPassword())); isSimpleAuthUsed = true; } httpClient.getParams().setParameter(AuthPolicy.AUTH_SCHEME_PRIORITY, authPrefs); } RemoteProxySettings rps = ctx.getRemoteProxySettings(); boolean isProxyUsed = false; if (rps.isEnabled()) { isProxyUsed = true; logger(logger).info("... proxy setup with host '{}'", rps.getHostname()); httpConfiguration.setProxy(rps.getHostname(), rps.getPort()); // check if we have non-proxy hosts if (rps.getNonProxyHosts() != null && !rps.getNonProxyHosts().isEmpty()) { Set<Pattern> nonProxyHostPatterns = new HashSet<Pattern>(rps.getNonProxyHosts().size()); for (String nonProxyHostRegex : rps.getNonProxyHosts()) { try { nonProxyHostPatterns.add(Pattern.compile(nonProxyHostRegex, Pattern.CASE_INSENSITIVE)); } catch (PatternSyntaxException e) { logger(logger).warn("Invalid non proxy host regex: {}", nonProxyHostRegex, e); } } httpConfiguration.getParams().setParameter( CustomMultiThreadedHttpConnectionManager.NON_PROXY_HOSTS_PATTERNS_KEY, nonProxyHostPatterns); } if (rps.getProxyAuthentication() != null) { ras = rps.getProxyAuthentication(); List<String> authPrefs = new ArrayList<String>(2); authPrefs.add(AuthPolicy.DIGEST); authPrefs.add(AuthPolicy.BASIC); if (ras instanceof ClientSSLRemoteAuthenticationSettings) { // ClientSSLRemoteAuthenticationSettings cras = (ClientSSLRemoteAuthenticationSettings) ras; // TODO - implement this } else if (ras instanceof NtlmRemoteAuthenticationSettings) { NtlmRemoteAuthenticationSettings nras = (NtlmRemoteAuthenticationSettings) ras; // Using NTLM auth, adding it as first in policies authPrefs.add(0, AuthPolicy.NTLM); if (ctx.getRemoteAuthenticationSettings() != null && (ctx .getRemoteAuthenticationSettings() instanceof NtlmRemoteAuthenticationSettings)) { logger(logger).warn("... Apache Commons HttpClient 3.x is unable to use NTLM auth scheme\n" + " for BOTH server side and proxy side authentication!\n" + " You MUST reconfigure server side auth and use BASIC/DIGEST scheme\n" + " if you have to use NTLM proxy, otherwise it will not work!\n" + " *** SERVER SIDE AUTH OVERRIDDEN"); } logger(logger).info("... proxy authentication setup for NTLM domain '{}'", nras.getNtlmDomain()); httpConfiguration.setHost(nras.getNtlmHost()); httpClient.getState().setProxyCredentials(AuthScope.ANY, new NTCredentials(nras.getUsername(), nras.getPassword(), nras.getNtlmHost(), nras.getNtlmDomain())); isNtlmUsed = true; } else if (ras instanceof UsernamePasswordRemoteAuthenticationSettings) { UsernamePasswordRemoteAuthenticationSettings uras = (UsernamePasswordRemoteAuthenticationSettings) ras; // Using Username/Pwd auth, will not add NTLM logger(logger).info("... proxy authentication setup for remote storage with username '{}'", uras.getUsername()); httpClient.getState().setProxyCredentials(AuthScope.ANY, new UsernamePasswordCredentials(uras.getUsername(), uras.getPassword())); } httpClient.getParams().setParameter(AuthPolicy.AUTH_SCHEME_PRIORITY, authPrefs); } } // set preemptive only for simplest scenario: // no proxy and BASIC auth is used if (isSimpleAuthUsed && !isProxyUsed) { logger(logger) .info("... simple scenario: simple authentication used with no proxy in between target and us," + " will use preemptive authentication"); // we have authentication, let's do it preemptive httpClient.getParams().setAuthenticationPreemptive(true); } // mark the fact that NTLM is in use // but ONLY IF IT CHANGED! // Otherwise, doing it always, actually marks the ctx itself as "changed", causing an avalanche of other // consequences, like resetting all the HTTP clients of all remote storages (coz they think there is a change // in proxy or remote connection settings, etc). final Boolean isNtlmUsedOldValue = (Boolean) ctx.getContextObject(NTLM_IS_IN_USE_KEY); if (isNtlmUsedOldValue == null || isNtlmUsedOldValue.booleanValue() != isNtlmUsed) { if (isNtlmUsed) { ctx.putContextObject(NTLM_IS_IN_USE_KEY, Boolean.TRUE); } else { ctx.putContextObject(NTLM_IS_IN_USE_KEY, Boolean.FALSE); } } }