List of usage examples for org.apache.commons.httpclient.params HttpConnectionManagerParams setMaxTotalConnections
public void setMaxTotalConnections(int paramInt)
From source file:org.apache.nutch.protocol.httpclient.Http.java
/** * Configures the HTTP client/* w w w . j a va 2 s . co m*/ */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. // ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); ProtocolSocketFactory factory = new DummySSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); // -------------------------------------------------------------------------------- // NUTCH-1836: Modification to increase the number of available connections // for multi-threaded crawls. // -------------------------------------------------------------------------------- params.setMaxTotalConnections(conf.getInt("mapreduce.tasktracker.map.tasks.maximum", 5) * conf.getInt("fetcher.threads.fetch", maxThreadsTotal)); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(conf.getInt("fetcher.threads.fetch", maxThreadsTotal)); // executeMethod(HttpMethod) seems to ignore the connection timeout on the // connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); ArrayList<Header> headers = new ArrayList<Header>(); // Note: some header fields (e.g., "User-Agent") are set per GET request if (!acceptLanguage.isEmpty()) { headers.add(new Header("Accept-Language", acceptLanguage)); } if (!acceptCharset.isEmpty()) { headers.add(new Header("Accept-Charset", acceptCharset)); } if (!accept.isEmpty()) { headers.add(new Header("Accept", accept)); } // accept gzipped content headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); // HTTP proxy server details if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); if (proxyUsername.length() > 0) { AuthScope proxyAuthScope = getAuthScope(this.proxyHost, this.proxyPort, this.proxyRealm); NTCredentials proxyCredentials = new NTCredentials(this.proxyUsername, this.proxyPassword, Http.agentHost, this.proxyRealm); client.getState().setProxyCredentials(proxyAuthScope, proxyCredentials); } } }
From source file:org.apache.nutch.protocol.httpclient.HttpBak.java
/** * Configures the HTTP client//from w w w . j a va 2 s . c o m */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. Protocol https = new Protocol("https", new DummySSLProtocolSocketFactory(), 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); // executeMethod(HttpMethod) seems to ignore the connection timeout on the connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); ArrayList headers = new ArrayList(); // Set the User Agent in the header headers.add(new Header("User-Agent", userAgent)); // prefer English headers.add(new Header("Accept-Language", acceptLanguage)); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")); // accept gzipped content headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); // HTTP proxy server details if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); if (proxyUsername.length() > 0) { AuthScope proxyAuthScope = getAuthScope(this.proxyHost, this.proxyPort, this.proxyRealm); NTCredentials proxyCredentials = new NTCredentials(this.proxyUsername, this.proxyPassword, this.agentHost, this.proxyRealm); client.getState().setProxyCredentials(proxyAuthScope, proxyCredentials); } } }
From source file:org.apache.nutch.protocol.httpclient.proxy.Http.java
/** * Configures the HTTP client/*from ww w . j av a 2s . c o m*/ */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. //ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); ProtocolSocketFactory factory = new DummySSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(maxThreadsTotal); // executeMethod(HttpMethod) seems to ignore the connection timeout on the // connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); ArrayList<Header> headers = new ArrayList<Header>(); // Set the User Agent in the header //headers.add(new Header("User-Agent", userAgent)); //NUTCH-1941 // prefer English headers.add(new Header("Accept-Language", acceptLanguage)); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")); // accept gzipped content headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); // HTTP proxy server details if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); if (proxyUsername.length() > 0) { AuthScope proxyAuthScope = getAuthScope(this.proxyHost, this.proxyPort, this.proxyRealm); NTCredentials proxyCredentials = new NTCredentials(this.proxyUsername, this.proxyPassword, Http.agentHost, this.proxyRealm); client.getState().setProxyCredentials(proxyAuthScope, proxyCredentials); } } }
From source file:org.apache.nutch.protocol.webdriver.Http.java
/** * Configures the HTTP client/* w ww . j ava 2 s . co m*/ */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(maxThreadsTotal); // executeMethod(HttpMethod) seems to ignore the connection timeout on // the connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); } ArrayList<Header> headers = new ArrayList<Header>(); // prefer English headers.add(new Header("Accept-Language", "en-us,en-gb,en;q=0.7,*;q=0.3")); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")); // accept gzipped content // headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); }
From source file:org.apache.servicemix.http.HttpComponent.java
protected void doInit() throws Exception { // Load configuration configuration.setRootDir(context.getWorkspaceRoot()); configuration.setComponentName(context.getComponentName()); configuration.load();//from w w w. j a v a 2 s . co m // Lookup keystoreManager and authenticationService if (configuration.getKeystoreManager() == null) { try { String name = configuration.getKeystoreManagerName(); Object km = context.getNamingContext().lookup(name); configuration.setKeystoreManager(km); } catch (Throwable e) { // ignore } } if (configuration.getAuthenticationService() == null) { try { String name = configuration.getAuthenticationServiceName(); Object as = context.getNamingContext().lookup(name); configuration.setAuthenticationService(as); } catch (Throwable e) { try { Class cl = Class .forName("org.apache.servicemix.jbi.security.auth.impl.JAASAuthenticationService"); configuration.setAuthenticationService(cl.newInstance()); } catch (Throwable t) { logger.warn("Unable to retrieve or create the authentication service"); } } } // Create client if (client == null) { connectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams params = new HttpConnectionManagerParams(); params.setDefaultMaxConnectionsPerHost(configuration.getMaxConnectionsPerHost()); params.setMaxTotalConnections(configuration.getMaxTotalConnections()); connectionManager.setParams(params); client = new HttpClient(connectionManager); } // Create connectionPool if (connectionPool == null) { connectionPool = createNewJettyClient(); } // Create serverManager if (configuration.isManaged()) { server = new ManagedContextManager(); } else { JettyContextManager jcm = new JettyContextManager(); jcm.setMBeanServer(context.getMBeanServer()); this.server = jcm; } server.setConfiguration(configuration); server.init(); server.start(); if (listener != null) { listener.setMBeanServer(context.getMBeanServer()); listener.setDomainAndContainer(context.getMBeanNames().getJmxDomainName(), containerNameFromObjectName()); } // Default initalization super.doInit(); }
From source file:org.attribyte.api.http.impl.commons.Commons3Client.java
private HttpConnectionManagerParams fromOptions(ClientOptions options) { HttpConnectionManagerParams connectionParams = new HttpConnectionManagerParams(); if (options != ClientOptions.IMPLEMENTATION_DEFAULT) { connectionParams.setConnectionTimeout(options.connectionTimeoutMillis); connectionParams.setSoTimeout(options.socketTimeoutMillis); connectionParams.setDefaultMaxConnectionsPerHost(options.maxConnectionsPerDestination); connectionParams.setMaxTotalConnections(options.maxConnectionsTotal); connectionParams.setSendBufferSize(options.requestBufferSize); connectionParams.setReceiveBufferSize(options.responseBufferSize); //connectionParams.setLinger(); //connectionParams.setStaleCheckingEnabled(); //connectionParams.setTcpNoDelay(); }//from w w w . j a v a2 s.c o m return connectionParams; }
From source file:org.collectionspace.chain.csp.persistence.services.connection.ServicesConnection.java
private void initClient() { if (manager != null) return;// w w w.j a v a 2 s . c o m synchronized (getClass()) { if (manager != null) return; // We're only connecting to one host, so set the max connections per host to be // the same as the max total connections. HttpConnectionManagerParams params = new HttpConnectionManagerParams(); params.setMaxTotalConnections(MAX_SERVICES_CONNECTIONS); params.setDefaultMaxConnectionsPerHost(MAX_SERVICES_CONNECTIONS); manager = new MultiThreadedHttpConnectionManager(); manager.setParams(params); } }
From source file:org.deri.any23.http.DefaultHTTPClient.java
private void ensureClientInitialized() { if (configuration == null) throw new IllegalStateException("client must be initialized first."); if (client != null) return;/*from ww w.ja v a 2s. c om*/ client = new HttpClient(manager); HttpConnectionManager connectionManager = client.getHttpConnectionManager(); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(configuration.getDefaultTimeout()); params.setSoTimeout(configuration.getDefaultTimeout()); params.setMaxTotalConnections(configuration.getMaxConnections()); HostConfiguration hostConf = client.getHostConfiguration(); List<Header> headers = new ArrayList<Header>(); headers.add(new Header("User-Agent", configuration.getUserAgent())); if (configuration.getAcceptHeader() != null) { headers.add(new Header("Accept", configuration.getAcceptHeader())); } headers.add(new Header("Accept-Language", "en-us,en-gb,en,*;q=0.3")); headers.add(new Header("Accept-Charset", "utf-8,iso-8859-1;q=0.7,*;q=0.5")); // headers.add(new Header("Accept-Encoding", "x-gzip, gzip")); hostConf.getParams().setParameter("http.default-headers", headers); }
From source file:org.eclipse.ecf.internal.provider.filetransfer.httpclient.ConnectionManagerHelper.java
private static void initParameters(HttpClient httpClient, HttpConnectionManager cm, boolean cmIsShared, Map options) {//w ww. jav a 2 s . co m if (cmIsShared) { long closeIdlePeriod = getLongProperty(ConnectionOptions.PROP_POOL_CLOSE_IDLE_PERIOD, ConnectionOptions.POOL_CLOSE_IDLE_PERIOD_DEFAULT); if (closeIdlePeriod > 0) { Trace.trace(Activator.PLUGIN_ID, "Closing connections which were idle at least " + closeIdlePeriod + " milliseconds."); //$NON-NLS-1$ //$NON-NLS-2$ cm.closeIdleConnections(closeIdlePeriod); } } // HttpClient parameters can be traced independently httpClient.setHttpConnectionManager(cm); int readTimeout = getSocketReadTimeout(options); cm.getParams().setSoTimeout(readTimeout); int connectTimeout = getConnectTimeout(options); cm.getParams().setConnectionTimeout(connectTimeout); if (cmIsShared) { HttpConnectionManagerParams cmParams = cm.getParams(); int maxHostConnections = getIntegerProperty(ConnectionOptions.PROP_MAX_CONNECTIONS_PER_HOST, ConnectionOptions.MAX_CONNECTIONS_PER_HOST_DEFAULT); int maxTotalConnections = getIntegerProperty(ConnectionOptions.PROP_MAX_TOTAL_CONNECTIONS, ConnectionOptions.MAX_TOTAL_CONNECTIONS_DEFAULT); cmParams.setDefaultMaxConnectionsPerHost(maxHostConnections); cmParams.setMaxTotalConnections(maxTotalConnections); long connectionManagerTimeout = getLongProperty(ConnectionOptions.PROP_POOL_CONNECTION_TIMEOUT, ConnectionOptions.POOL_CONNECTION_TIMEOUT_DEFAULT); httpClient.getParams().setConnectionManagerTimeout(connectionManagerTimeout); } }
From source file:org.eclipse.ecr.core.storage.sql.RepositoryImpl.java
public RepositoryImpl(RepositoryDescriptor repositoryDescriptor) throws StorageException { this.repositoryDescriptor = repositoryDescriptor; sessions = new CopyOnWriteArrayList<SessionImpl>(); cachePropagator = new InvalidationsPropagator(); eventPropagator = new InvalidationsPropagator(); repositoryEventQueue = new InvalidationsQueue("repo-" + repositoryDescriptor.name); try {/*from w ww . ja v a 2 s .c om*/ schemaManager = Framework.getService(SchemaManager.class); } catch (Exception e) { throw new StorageException(e); } try { eventService = Framework.getService(EventService.class); } catch (Exception e) { throw new StorageException(e); } connectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams params = connectionManager.getParams(); params.setDefaultMaxConnectionsPerHost(20); params.setMaxTotalConnections(20); httpClient = new HttpClient(connectionManager); binaryManager = createBinaryManager(); backend = createBackend(); createServer(); }