List of usage examples for org.apache.http.impl.conn PoolingClientConnectionManager setMaxTotal
public void setMaxTotal(final int max)
From source file:net.opentsdb.search.ElasticSearch.java
/** * Initializes the search plugin, setting up the HTTP client pool and config * options.// www. j av a 2 s. c o m * @param tsdb The TSDB to which we belong * @return null if successful, otherwise it throws an exception * @throws IllegalArgumentException if a config value is invalid * @throws NumberFormatException if a config value is invalid */ @Override public void initialize(final TSDB tsdb) { config = new ESPluginConfig(tsdb.getConfig()); setConfiguration(); // setup a connection pool for reuse PoolingClientConnectionManager http_pool = new PoolingClientConnectionManager(); http_pool.setDefaultMaxPerRoute(config.getInt("tsd.search.elasticsearch.pool.max_per_route")); http_pool.setMaxTotal(config.getInt("tsd.search.elasticsearch.pool.max_total")); httpClient = new FailoverHttpClient(http_pool); // start worker threads indexers = new SearchIndexer[index_threads]; for (int i = 0; i < index_threads; i++) { indexers[i] = new SearchIndexer(); indexers[i].start(); } }
From source file:com.impetus.client.couchdb.CouchDBClientFactory.java
@Override protected Object createPoolOrConnection() { PersistenceUnitMetadata puMetadata = kunderaMetadata.getApplicationMetadata() .getPersistenceUnitMetadata(getPersistenceUnit()); Properties props = puMetadata.getProperties(); String contactNode = null;// w w w .j a v a 2 s .co m String defaultPort = null; String keyspace = null; String poolSize = null; String userName = null; String password = null; String maxConnections = null; if (externalProperties != null) { contactNode = (String) externalProperties.get(PersistenceProperties.KUNDERA_NODES); defaultPort = (String) externalProperties.get(PersistenceProperties.KUNDERA_PORT); keyspace = (String) externalProperties.get(PersistenceProperties.KUNDERA_KEYSPACE); poolSize = (String) externalProperties.get(PersistenceProperties.KUNDERA_POOL_SIZE_MAX_ACTIVE); userName = (String) externalProperties.get(PersistenceProperties.KUNDERA_USERNAME); password = (String) externalProperties.get(PersistenceProperties.KUNDERA_PASSWORD); maxConnections = (String) externalProperties.get(PersistenceProperties.KUNDERA_POOL_SIZE_MAX_TOTAL); } if (contactNode == null) { contactNode = (String) props.get(PersistenceProperties.KUNDERA_NODES); } if (defaultPort == null) { defaultPort = (String) props.get(PersistenceProperties.KUNDERA_PORT); } if (keyspace == null) { keyspace = (String) props.get(PersistenceProperties.KUNDERA_KEYSPACE); } if (poolSize == null) { poolSize = props.getProperty(PersistenceProperties.KUNDERA_POOL_SIZE_MAX_ACTIVE); } if (userName == null) { userName = props.getProperty(PersistenceProperties.KUNDERA_USERNAME); password = props.getProperty(PersistenceProperties.KUNDERA_PASSWORD); } onValidation(contactNode, defaultPort); try { SchemeSocketFactory ssf = null; ssf = PlainSocketFactory.getSocketFactory(); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme(CouchDBConstants.PROTOCOL, Integer.parseInt(defaultPort), ssf)); PoolingClientConnectionManager ccm = new PoolingClientConnectionManager(schemeRegistry); httpClient = new DefaultHttpClient(ccm); httpHost = new HttpHost(contactNode, Integer.parseInt(defaultPort), CouchDBConstants.PROTOCOL); httpClient.getParams().setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, "UTF-8"); // httpclient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT, // props.getSocketTimeout()); // httpclient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, // props.getConnectionTimeout()); if (!StringUtils.isBlank(maxConnections)) { ccm.setMaxTotal(Integer.parseInt(maxConnections)); ccm.setDefaultMaxPerRoute(Integer.parseInt(maxConnections)); } // basic authentication if (userName != null && password != null) { ((AbstractHttpClient) httpClient).getCredentialsProvider().setCredentials( new AuthScope(contactNode, Integer.parseInt(defaultPort)), new UsernamePasswordCredentials(userName, password)); } // request interceptor ((DefaultHttpClient) httpClient).addRequestInterceptor(new HttpRequestInterceptor() { public void process(final HttpRequest request, final HttpContext context) throws IOException { if (logger.isInfoEnabled()) { RequestLine requestLine = request.getRequestLine(); logger.info( ">> " + requestLine.getMethod() + " " + URI.create(requestLine.getUri()).getPath()); } } }); // response interceptor ((DefaultHttpClient) httpClient).addResponseInterceptor(new HttpResponseInterceptor() { public void process(final HttpResponse response, final HttpContext context) throws IOException { if (logger.isInfoEnabled()) logger.info("<< Status: " + response.getStatusLine().getStatusCode()); } }); } catch (Exception e) { logger.error("Error Creating HTTP client, caoused by {}. ", e); throw new IllegalStateException(e); } return httpClient; }
From source file:org.apache.marmotta.platform.core.services.http.HttpClientServiceImpl.java
@PostConstruct protected void initialize() { try {/*from w w w.j a va 2 s.com*/ lock.writeLock().lock(); httpParams = new BasicHttpParams(); String userAgentString = "Apache Marmotta/" + configurationService.getStringConfiguration("kiwi.version") + " (running at " + configurationService.getServerUri() + ")" + " lmf-core/" + configurationService.getStringConfiguration("kiwi.version"); userAgentString = configurationService.getStringConfiguration("core.http.user_agent", userAgentString); httpParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, configurationService.getIntConfiguration("core.http.so_timeout", 60000)); httpParams.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, configurationService.getIntConfiguration("core.http.connection_timeout", 10000)); httpParams.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, true); httpParams.setIntParameter(ClientPNames.MAX_REDIRECTS, 3); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); PoolingClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry); cm.setMaxTotal(configurationService.getIntConfiguration(CoreOptions.HTTP_MAX_CONNECTIONS, 20)); cm.setDefaultMaxPerRoute( configurationService.getIntConfiguration(CoreOptions.HTTP_MAX_CONNECTIONS_PER_ROUTE, 10)); final DefaultHttpClient hc = new DefaultHttpClient(cm, httpParams); hc.setRedirectStrategy(new LMFRedirectStrategy()); hc.setHttpRequestRetryHandler(new LMFHttpRequestRetryHandler()); hc.removeRequestInterceptorByClass(org.apache.http.protocol.RequestUserAgent.class); hc.addRequestInterceptor(new LMFRequestUserAgent(userAgentString)); if (configurationService.getBooleanConfiguration(CoreOptions.HTTP_CLIENT_CACHE_ENABLE, true)) { CacheConfig cacheConfig = new CacheConfig(); // FIXME: Hardcoded constants - is this useful? cacheConfig.setMaxCacheEntries(1000); cacheConfig.setMaxObjectSize(81920); final HttpCacheStorage cacheStore = new MapHttpCacheStorage(httpCache); this.httpClient = new MonitoredHttpClient(new CachingHttpClient(hc, cacheStore, cacheConfig)); } else { this.httpClient = new MonitoredHttpClient(hc); } bytesSent.set(0); bytesReceived.set(0); requestsExecuted.set(0); idleConnectionMonitorThread = new IdleConnectionMonitorThread(httpClient.getConnectionManager()); idleConnectionMonitorThread.start(); StatisticsProvider stats = new StatisticsProvider(cm); statisticsService.registerModule(HttpClientService.class.getSimpleName(), stats); } finally { lock.writeLock().unlock(); } }
From source file:com.amazonservices.mws.client.MwsConnection.java
/** * Get a connection manager to use for this connection. * <p>/* w w w .j a v a 2s. com*/ * Called late in initialization. * <p> * Default implementation uses a shared PoolingClientConnectionManager. * * @return The connection manager to use. */ private ClientConnectionManager getConnectionManager() { synchronized (this.getClass()) { if (sharedCM == null) { PoolingClientConnectionManager cm = new PoolingClientConnectionManager(); cm.setMaxTotal(maxConnections); cm.setDefaultMaxPerRoute(maxConnections); sharedCM = cm; } return sharedCM; } }
From source file:API.amazon.mws.orders.MarketplaceWebServiceOrdersClient.java
/** * Configure HttpClient with set of defaults as well as configuration from * MarketplaceWebServiceProductsConfig instance * //from w w w . ja v a 2 s . co m */ private HttpClient configureHttpClient(String applicationName, String applicationVersion) { // respect a user-provided User-Agent header as-is, but if none is provided // then generate one satisfying the MWS User-Agent requirements if (config.getUserAgent() == null) { config.setUserAgent(quoteAppName(applicationName), quoteAppVersion(applicationVersion), quoteAttributeValue("Java/" + System.getProperty("java.version") + "/" + System.getProperty("java.class.version") + "/" + System.getProperty("java.vendor")), quoteAttributeName("Platform"), quoteAttributeValue("" + System.getProperty("os.name") + "/" + System.getProperty("os.arch") + "/" + System.getProperty("os.version")), quoteAttributeName("MWSClientVersion"), quoteAttributeValue(clientVersion)); } defaultHeaders.add(new BasicHeader("X-Amazon-User-Agent", config.getUserAgent())); /* Set http client parameters */ BasicHttpParams httpParams = new BasicHttpParams(); httpParams.setParameter(CoreProtocolPNames.USER_AGENT, config.getUserAgent()); /* Set connection parameters */ HttpConnectionParams.setConnectionTimeout(httpParams, 50000); HttpConnectionParams.setSoTimeout(httpParams, 50000); HttpConnectionParams.setStaleCheckingEnabled(httpParams, true); HttpConnectionParams.setTcpNoDelay(httpParams, true); /* Set connection manager */ PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setMaxTotal(config.getMaxConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnections()); /* Set http client */ httpClient = new DefaultHttpClient(connectionManager, httpParams); httpContext = new BasicHttpContext(); /* Set proxy if configured */ if (config.isSetProxyHost() && config.isSetProxyPort()) { log.info("Configuring Proxy. Proxy Host: " + config.getProxyHost() + "Proxy Port: " + config.getProxyPort()); final HttpHost hostConfiguration = new HttpHost(config.getProxyHost(), config.getProxyPort(), (usesHttps(config.getServiceURL()) ? "https" : "http")); httpContext = new BasicHttpContext(); httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, hostConfiguration); if (config.isSetProxyUsername() && config.isSetProxyPassword()) { credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); httpContext.setAttribute(ClientContext.CREDS_PROVIDER, credentialsProvider); } } return httpClient; }
From source file:com.ngdata.hbaseindexer.supervisor.IndexerSupervisor.java
private void startIndexer(IndexerDefinition indexerDef) { IndexerHandle handle = null;//from w ww.j a va 2 s . co m SolrServer solr = null; String indexerProcessId = null; try { // If this is an update and the indexer failed to start, it's still in the registry as a // failed process if (indexerProcessIds.containsKey(indexerDef.getName())) { indexerProcessRegistry.unregisterIndexerProcess(indexerProcessIds.remove(indexerDef.getName())); } indexerProcessId = indexerProcessRegistry.registerIndexerProcess(indexerDef.getName(), hostName); indexerProcessIds.put(indexerDef.getName(), indexerProcessId); // Create and register the indexer IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory( indexerDef.getIndexerComponentFactory(), new ByteArrayInputStream(indexerDef.getConfiguration()), indexerDef.getConnectionParams()); IndexerConf indexerConf = factory.createIndexerConf(); ResultToSolrMapper mapper = factory.createMapper(indexerDef.getName()); Sharder sharder = null; SolrInputDocumentWriter solrWriter; PoolingClientConnectionManager connectionManager = null; if (indexerDef.getConnectionType() == null || indexerDef.getConnectionType().equals("solr")) { Map<String, String> connectionParams = indexerDef.getConnectionParams(); String solrMode = SolrConnectionParamUtil.getSolrMode(connectionParams); if (solrMode.equals("cloud")) { solrWriter = new DirectSolrInputDocumentWriter(indexerDef.getName(), createCloudSolrServer(connectionParams)); } else if (solrMode.equals("classic")) { connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(connectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(connectionParams)); httpClient = new DefaultHttpClient(connectionManager); List<SolrServer> solrServers = createHttpSolrServers(connectionParams, httpClient); solrWriter = new DirectSolrClassicInputDocumentWriter(indexerDef.getName(), solrServers); sharder = createSharder(connectionParams, solrServers.size()); } else { throw new RuntimeException( "Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } } else { throw new RuntimeException("Invalid connection type: " + indexerDef.getConnectionType() + ". Only 'solr' is supported"); } Indexer indexer = Indexer.createIndexer(indexerDef.getName(), indexerConf, indexerConf.getTable(), mapper, htablePool, sharder, solrWriter); IndexingEventListener eventListener = new IndexingEventListener(indexer, indexerConf.getTable()); int threads = hbaseConf.getInt("hbaseindexer.indexer.threads", 10); SepConsumer sepConsumer = new SepConsumer(indexerDef.getSubscriptionId(), indexerDef.getSubscriptionTimestamp(), eventListener, threads, hostName, zk, hbaseConf, null); handle = new IndexerHandle(indexerDef, indexer, sepConsumer, solr, connectionManager); handle.start(); indexers.put(indexerDef.getName(), handle); indexerRegistry.register(indexerDef.getName(), indexer); log.info("Started indexer for " + indexerDef.getName()); } catch (Throwable t) { if (t instanceof InterruptedException) { Thread.currentThread().interrupt(); } log.error("Problem starting indexer " + indexerDef.getName(), t); try { if (indexerProcessId != null) { indexerProcessRegistry.setErrorStatus(indexerProcessId, t); } } catch (Exception e) { log.error("Error setting error status on indexer process " + indexerProcessId, e); } if (handle != null) { // stop any listeners that might have been started try { handle.stop(); } catch (Throwable t2) { if (t2 instanceof InterruptedException) { Thread.currentThread().interrupt(); } log.error( "Problem stopping consumers for failed-to-start indexer '" + indexerDef.getName() + "'", t2); } } else { // Might be the handle was not yet created, but the solr connection was Closer.close(solr); } } }