List of usage examples for org.apache.commons.httpclient.params HttpConnectionManagerParams setDefaultMaxConnectionsPerHost
public void setDefaultMaxConnectionsPerHost(int paramInt)
From source file:org.apache.nutch.protocol.httpclient.Http.java
/** * Configures the HTTP client/*from w w w. j a v a 2 s. c om*/ */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. // ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); ProtocolSocketFactory factory = new DummySSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); // -------------------------------------------------------------------------------- // NUTCH-1836: Modification to increase the number of available connections // for multi-threaded crawls. // -------------------------------------------------------------------------------- params.setMaxTotalConnections(conf.getInt("mapreduce.tasktracker.map.tasks.maximum", 5) * conf.getInt("fetcher.threads.fetch", maxThreadsTotal)); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(conf.getInt("fetcher.threads.fetch", maxThreadsTotal)); // executeMethod(HttpMethod) seems to ignore the connection timeout on the // connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); ArrayList<Header> headers = new ArrayList<Header>(); // Note: some header fields (e.g., "User-Agent") are set per GET request if (!acceptLanguage.isEmpty()) { headers.add(new Header("Accept-Language", acceptLanguage)); } if (!acceptCharset.isEmpty()) { headers.add(new Header("Accept-Charset", acceptCharset)); } if (!accept.isEmpty()) { headers.add(new Header("Accept", accept)); } // accept gzipped content headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); // HTTP proxy server details if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); if (proxyUsername.length() > 0) { AuthScope proxyAuthScope = getAuthScope(this.proxyHost, this.proxyPort, this.proxyRealm); NTCredentials proxyCredentials = new NTCredentials(this.proxyUsername, this.proxyPassword, Http.agentHost, this.proxyRealm); client.getState().setProxyCredentials(proxyAuthScope, proxyCredentials); } } }
From source file:org.apache.nutch.protocol.httpclient.proxy.Http.java
/** * Configures the HTTP client/*from ww w.j ava 2s . c om*/ */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. //ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); ProtocolSocketFactory factory = new DummySSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(maxThreadsTotal); // executeMethod(HttpMethod) seems to ignore the connection timeout on the // connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); ArrayList<Header> headers = new ArrayList<Header>(); // Set the User Agent in the header //headers.add(new Header("User-Agent", userAgent)); //NUTCH-1941 // prefer English headers.add(new Header("Accept-Language", acceptLanguage)); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")); // accept gzipped content headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); // HTTP proxy server details if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); if (proxyUsername.length() > 0) { AuthScope proxyAuthScope = getAuthScope(this.proxyHost, this.proxyPort, this.proxyRealm); NTCredentials proxyCredentials = new NTCredentials(this.proxyUsername, this.proxyPassword, Http.agentHost, this.proxyRealm); client.getState().setProxyCredentials(proxyAuthScope, proxyCredentials); } } }
From source file:org.apache.nutch.protocol.webdriver.Http.java
/** * Configures the HTTP client//from ww w .j a va 2 s .c o m */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(maxThreadsTotal); // executeMethod(HttpMethod) seems to ignore the connection timeout on // the connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); } ArrayList<Header> headers = new ArrayList<Header>(); // prefer English headers.add(new Header("Accept-Language", "en-us,en-gb,en;q=0.7,*;q=0.3")); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")); // accept gzipped content // headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); }
From source file:org.apache.servicemix.http.HttpComponent.java
protected void doInit() throws Exception { // Load configuration configuration.setRootDir(context.getWorkspaceRoot()); configuration.setComponentName(context.getComponentName()); configuration.load();/* w ww . j a v a 2 s . c o m*/ // Lookup keystoreManager and authenticationService if (configuration.getKeystoreManager() == null) { try { String name = configuration.getKeystoreManagerName(); Object km = context.getNamingContext().lookup(name); configuration.setKeystoreManager(km); } catch (Throwable e) { // ignore } } if (configuration.getAuthenticationService() == null) { try { String name = configuration.getAuthenticationServiceName(); Object as = context.getNamingContext().lookup(name); configuration.setAuthenticationService(as); } catch (Throwable e) { try { Class cl = Class .forName("org.apache.servicemix.jbi.security.auth.impl.JAASAuthenticationService"); configuration.setAuthenticationService(cl.newInstance()); } catch (Throwable t) { logger.warn("Unable to retrieve or create the authentication service"); } } } // Create client if (client == null) { connectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams params = new HttpConnectionManagerParams(); params.setDefaultMaxConnectionsPerHost(configuration.getMaxConnectionsPerHost()); params.setMaxTotalConnections(configuration.getMaxTotalConnections()); connectionManager.setParams(params); client = new HttpClient(connectionManager); } // Create connectionPool if (connectionPool == null) { connectionPool = createNewJettyClient(); } // Create serverManager if (configuration.isManaged()) { server = new ManagedContextManager(); } else { JettyContextManager jcm = new JettyContextManager(); jcm.setMBeanServer(context.getMBeanServer()); this.server = jcm; } server.setConfiguration(configuration); server.init(); server.start(); if (listener != null) { listener.setMBeanServer(context.getMBeanServer()); listener.setDomainAndContainer(context.getMBeanNames().getJmxDomainName(), containerNameFromObjectName()); } // Default initalization super.doInit(); }
From source file:org.apache.tuscany.sca.binding.ws.axis2.provider.Axis2ReferenceBindingProvider.java
public void start() { configContext = Axis2EngineIntegration.getAxisConfigurationContext(extensionPoints.getServiceDiscovery()); // Apply the configuration from any other policies if (isRampartRequired) { Axis2EngineIntegration.loadRampartModule(configContext); }// w w w. j a va 2 s. c o m for (PolicyProvider pp : this.endpointReference.getPolicyProviders()) { pp.configureBinding(this); } try { Definition definition = wsBinding.getGeneratedWSDLDocument(); QName serviceQName = wsBinding.getService().getQName(); Port port = wsBinding.getPort(); if (port == null) { // service has multiple ports, select one port to use // TODO - it feels like there is much more to this than is // here at the moment as need to match with the service side // assuming that it's available Collection<Port> ports = wsBinding.getService().getPorts().values(); for (Port p : ports) { // look for a SOAP 1.1 port first if (p.getExtensibilityElements().get(0) instanceof SOAPAddress) { port = p; break; } } if (port == null) { // no SOAP 1.1 port available, so look for a SOAP 1.2 port for (Port p : ports) { if (p.getExtensibilityElements().get(0) instanceof SOAP12Address) { port = p; break; } } } } axisClientSideService = Axis2EngineIntegration.createClientSideAxisService(definition, serviceQName, port.getName(), new Options()); HttpClient httpClient = (HttpClient) configContext.getProperty(HTTPConstants.CACHED_HTTP_CLIENT); if (httpClient == null) { MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams connectionManagerParams = new HttpConnectionManagerParams(); connectionManagerParams.setDefaultMaxConnectionsPerHost(2); connectionManagerParams.setTcpNoDelay(true); connectionManagerParams.setStaleCheckingEnabled(true); connectionManagerParams.setLinger(0); connectionManager.setParams(connectionManagerParams); httpClient = new HttpClient(connectionManager); configContext.setThreadPool(new ThreadPool(1, 5)); configContext.setProperty(HTTPConstants.REUSE_HTTP_CLIENT, Boolean.TRUE); configContext.setProperty(HTTPConstants.CACHED_HTTP_CLIENT, httpClient); } serviceClient = new ServiceClient(configContext, axisClientSideService); } catch (AxisFault e) { throw new RuntimeException(e); // TODO: better exception } }
From source file:org.apache.wicket.threadtest.tester.Tester.java
/** * Runs the test./*from ww w. j a v a2 s . co m*/ * * @throws Exception */ public void run() throws Exception { activeThreads = 0; HttpConnectionManagerParams connManagerParams = new HttpConnectionManagerParams(); connManagerParams.setDefaultMaxConnectionsPerHost(numberOfThreads * 2); MultiThreadedHttpConnectionManager manager = new MultiThreadedHttpConnectionManager(); manager.setParams(connManagerParams); Server server = null; GetMethod getMethod = new GetMethod("http://localhost:" + port + "/"); try { getMethod.setFollowRedirects(true); HttpClient httpClient = new HttpClient(params, manager); int code = httpClient.executeMethod(getMethod); if (code != 200) { server = startServer(port); } } catch (Exception e) { server = startServer(port); } finally { getMethod.releaseConnection(); } try { ThreadGroup g = new ThreadGroup("runners"); Thread[] threads = new Thread[numberOfThreads]; HttpClient client = null; for (int i = 0; i < numberOfThreads; i++) { if (multipleSessions) { client = new HttpClient(params, manager); client.getHostConfiguration().setHost(host, port); } else { if (client == null) { client = new HttpClient(params, manager); client.getHostConfiguration().setHost(host, port); } } threads[i] = new Thread(g, new CommandRunner(commands, client, this)); } long start = System.currentTimeMillis(); for (int i = 0; i < numberOfThreads; i++) { activeThreads++; threads[i].start(); } while (activeThreads > 0) { synchronized (this) { wait(); } } long end = System.currentTimeMillis(); long time = end - start; log.info("\n******** finished in " + Duration.milliseconds(time) + " (" + time + " milis)"); } finally { MultiThreadedHttpConnectionManager.shutdownAll(); if (server != null) { server.stop(); } } }
From source file:org.appverse.web.framework.backend.ws.helpers.StubHelper.java
public static void configureEndpoint(String endpointPropertiesFile, String timeoutPropertyName, ServiceClient _serviceClient) {//from w ww . j a v a 2 s.com Properties endpointsProperties = new Properties(); InputStream endPointsInputStream = StubHelper.class.getResourceAsStream(endpointPropertiesFile); try { endpointsProperties.load(endPointsInputStream); } catch (IOException e) { e.printStackTrace(); } String accountTimeoutString = (String) endpointsProperties.get(timeoutPropertyName); try { long accountTimeout = new Long(accountTimeoutString) * 1000; _serviceClient.getOptions().setTimeOutInMilliSeconds(accountTimeout); } catch (NumberFormatException e) { logger.equals("Error login axis account service timeout"); } String endpointProxyEnabled = (String) endpointsProperties.get("endpoint.proxy.enabled"); if (endpointProxyEnabled != null && endpointProxyEnabled.equals("true")) { HttpTransportProperties.ProxyProperties proxyProperties = new HttpTransportProperties.ProxyProperties(); String endpointProxyHost = endpointsProperties.getProperty("endpoint.proxy.host"); proxyProperties.setProxyName(endpointProxyHost); int endpointProxyPort = new Integer(endpointsProperties.getProperty("endpoint.proxy.port")); proxyProperties.setProxyPort(endpointProxyPort); _serviceClient.getOptions().setProperty(HTTPConstants.PROXY, proxyProperties); } if (endpointsProperties.getProperty("endpoint.ignore_SSL_errors") != null && endpointsProperties.getProperty("endpoint.ignore_SSL_errors").equals("true")) { // Create a trust manager that does not validate certificate // chains TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { @Override public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) { } @Override public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) { } @Override public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; } } }; // Install the all-trusting trust manager try { SSLContext sc = SSLContext.getInstance("SSL"); sc.init(null, trustAllCerts, new java.security.SecureRandom()); HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); } catch (Exception e) { } } ConfigurationContext configurationContext = _serviceClient.getServiceContext().getConfigurationContext(); MultiThreadedHttpConnectionManager multiThreadedHttpConnectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams params = new HttpConnectionManagerParams(); params.setDefaultMaxConnectionsPerHost(50); multiThreadedHttpConnectionManager.setParams(params); HttpClient httpClient = new HttpClient(multiThreadedHttpConnectionManager); configurationContext.setProperty(HTTPConstants.CACHED_HTTP_CLIENT, httpClient); }
From source file:org.attribyte.api.http.impl.commons.Commons3Client.java
private HttpConnectionManagerParams fromOptions(ClientOptions options) { HttpConnectionManagerParams connectionParams = new HttpConnectionManagerParams(); if (options != ClientOptions.IMPLEMENTATION_DEFAULT) { connectionParams.setConnectionTimeout(options.connectionTimeoutMillis); connectionParams.setSoTimeout(options.socketTimeoutMillis); connectionParams.setDefaultMaxConnectionsPerHost(options.maxConnectionsPerDestination); connectionParams.setMaxTotalConnections(options.maxConnectionsTotal); connectionParams.setSendBufferSize(options.requestBufferSize); connectionParams.setReceiveBufferSize(options.responseBufferSize); //connectionParams.setLinger(); //connectionParams.setStaleCheckingEnabled(); //connectionParams.setTcpNoDelay(); }// w w w.ja v a 2s . c o m return connectionParams; }
From source file:org.collectionspace.chain.csp.persistence.services.connection.ServicesConnection.java
private void initClient() { if (manager != null) return;/*from w w w.ja va 2 s. c o m*/ synchronized (getClass()) { if (manager != null) return; // We're only connecting to one host, so set the max connections per host to be // the same as the max total connections. HttpConnectionManagerParams params = new HttpConnectionManagerParams(); params.setMaxTotalConnections(MAX_SERVICES_CONNECTIONS); params.setDefaultMaxConnectionsPerHost(MAX_SERVICES_CONNECTIONS); manager = new MultiThreadedHttpConnectionManager(); manager.setParams(params); } }
From source file:org.eclipse.ecf.internal.provider.filetransfer.httpclient.ConnectionManagerHelper.java
private static void initParameters(HttpClient httpClient, HttpConnectionManager cm, boolean cmIsShared, Map options) {/* w ww. j a v a 2 s . co m*/ if (cmIsShared) { long closeIdlePeriod = getLongProperty(ConnectionOptions.PROP_POOL_CLOSE_IDLE_PERIOD, ConnectionOptions.POOL_CLOSE_IDLE_PERIOD_DEFAULT); if (closeIdlePeriod > 0) { Trace.trace(Activator.PLUGIN_ID, "Closing connections which were idle at least " + closeIdlePeriod + " milliseconds."); //$NON-NLS-1$ //$NON-NLS-2$ cm.closeIdleConnections(closeIdlePeriod); } } // HttpClient parameters can be traced independently httpClient.setHttpConnectionManager(cm); int readTimeout = getSocketReadTimeout(options); cm.getParams().setSoTimeout(readTimeout); int connectTimeout = getConnectTimeout(options); cm.getParams().setConnectionTimeout(connectTimeout); if (cmIsShared) { HttpConnectionManagerParams cmParams = cm.getParams(); int maxHostConnections = getIntegerProperty(ConnectionOptions.PROP_MAX_CONNECTIONS_PER_HOST, ConnectionOptions.MAX_CONNECTIONS_PER_HOST_DEFAULT); int maxTotalConnections = getIntegerProperty(ConnectionOptions.PROP_MAX_TOTAL_CONNECTIONS, ConnectionOptions.MAX_TOTAL_CONNECTIONS_DEFAULT); cmParams.setDefaultMaxConnectionsPerHost(maxHostConnections); cmParams.setMaxTotalConnections(maxTotalConnections); long connectionManagerTimeout = getLongProperty(ConnectionOptions.PROP_POOL_CONNECTION_TIMEOUT, ConnectionOptions.POOL_CONNECTION_TIMEOUT_DEFAULT); httpClient.getParams().setConnectionManagerTimeout(connectionManagerTimeout); } }