List of usage examples for org.apache.commons.httpclient MultiThreadedHttpConnectionManager MultiThreadedHttpConnectionManager
public MultiThreadedHttpConnectionManager()
From source file:com.google.jstestdriver.requesthandlers.RequestHandlersModule.java
@Provides @Singleton/*www. ja v a 2 s . c o m*/ HttpClient provideHttpClient() { Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory) new EasySSLProtocolSocketFactory(), 443)); MultiThreadedHttpConnectionManager manager = new MultiThreadedHttpConnectionManager(); manager.getParams().setDefaultMaxConnectionsPerHost(20); manager.getParams().setMaxTotalConnections(200); HttpClient client = new HttpClient(manager); client.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES); return client; }
From source file:com.atlassian.jira.rest.client.internal.jersey.JerseyJiraRestClient.java
private static ApacheHttpClientHandler createDefaultClientHander(DefaultApacheHttpClientConfig config) { final HttpClient client = new HttpClient(new MultiThreadedHttpConnectionManager()); return new ApacheHttpClientHandler(client, config); }
From source file:com.googlecode.fightinglayoutbugs.DetectInvalidImageUrls.java
public Collection<LayoutBug> findLayoutBugsIn(@Nonnull WebPage webPage) { try {/* ww w.j a v a2 s .c o m*/ _webPage = webPage; _baseUrl = _webPage.getUrl(); _documentCharset = (String) _webPage.executeJavaScript("return document.characterSet"); _screenshotTaken = false; _checkedCssUrls = new ConcurrentSkipListSet<String>(); _faviconUrl = "/favicon.ico"; _layoutBugs = new ArrayList<LayoutBug>(); _mockBrowser = new MockBrowser( _httpClient == null ? new HttpClient(new MultiThreadedHttpConnectionManager()) : _httpClient); try { // 1. Check the src attribute of all visible <img> elements ... checkVisibleImgElements(); // 2. Check the style attribute of all elements ... checkStyleAttributes(); // 3. Check all <style> elements ... checkStyleElements(); // 4. Check all linked CSS resources ... checkLinkedCss(); // 5. Check favicon ... checkFavicon(); // 6. Wait until all asynchronous checks are finished ... _mockBrowser.waitUntilAllDownloadsAreFinished(); return _layoutBugs; } finally { _mockBrowser.dispose(); } } finally { // Free resources for garbage collection ... _mockBrowser = null; _layoutBugs = null; _faviconUrl = null; _checkedCssUrls = null; _documentCharset = null; _baseUrl = null; _webPage = null; } }
From source file:com.smartitengineering.dao.hbase.autoincrement.AutoIncrementRowIdForLongTest.java
@BeforeClass public static void globalSetup() throws Exception { /*/*from w ww .ja va 2 s . c om*/ * Start HBase and initialize tables */ //-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl System.setProperty("javax.xml.parsers.DocumentBuilderFactory", "com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"); try { TEST_UTIL.startMiniCluster(); } catch (Exception ex) { LOGGER.error(ex.getMessage(), ex); } //Create table for testing InputStream classpathResource = AutoIncrementRowIdForLongTest.class.getClassLoader() .getResourceAsStream("ddl-config-sample1.json"); Collection<HBaseTableConfiguration> configs = ConfigurationJsonParser.getConfigurations(classpathResource); try { new HBaseTableGenerator(configs, TEST_UTIL.getConfiguration(), true).generateTables(); } catch (Exception ex) { LOGGER.error("Could not create table!", ex); Assert.fail(ex.getMessage()); } pool = new HTablePool(TEST_UTIL.getConfiguration(), 200); //Start web app jettyServer = new Server(PORT); final String webapp = "./src/main/webapp/"; if (!new File(webapp).exists()) { throw new IllegalStateException("WebApp file/dir does not exist!"); } Properties properties = new Properties(); properties.setProperty(GuiceUtil.CONTEXT_NAME_PROP, "com.smartitengineering.dao.impl.hbase"); properties.setProperty(GuiceUtil.IGNORE_MISSING_DEP_PROP, Boolean.TRUE.toString()); properties.setProperty(GuiceUtil.MODULES_LIST_PROP, TestModule.class.getName()); GuiceUtil.getInstance(properties).register(); WebAppContext webAppHandler = new WebAppContext(webapp, "/"); jettyServer.setHandler(webAppHandler); jettyServer.setSendDateHeader(true); jettyServer.start(); //Initialize client final MultiThreadedHttpConnectionManager manager = new MultiThreadedHttpConnectionManager(); manager.getParams().setDefaultMaxConnectionsPerHost(THREAD_COUNT); manager.getParams().setMaxTotalConnections(THREAD_COUNT); httpClient = new HttpClient(manager); }
From source file:com.linkedin.pinot.controller.ControllerStarter.java
public void start() { LOGGER.info("Starting Pinot controller"); Utils.logVersions();//ww w . j av a2s . c o m component.getServers().add(Protocol.HTTP, Integer.parseInt(config.getControllerPort())); component.getClients().add(Protocol.FILE); component.getClients().add(Protocol.JAR); final Context applicationContext = component.getContext().createChildContext(); LOGGER.info("Injecting configuration and resource manager to the API context"); applicationContext.getAttributes().put(ControllerConf.class.toString(), config); applicationContext.getAttributes().put(PinotHelixResourceManager.class.toString(), helixResourceManager); MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); connectionManager.getParams().setConnectionTimeout(config.getServerAdminRequestTimeoutSeconds()); applicationContext.getAttributes().put(HttpConnectionManager.class.toString(), connectionManager); applicationContext.getAttributes().put(Executor.class.toString(), executorService); controllerRestApp.setContext(applicationContext); component.getDefaultHost().attach(controllerRestApp); MetricsHelper.initializeMetrics(config.subset("pinot.controller.metrics")); MetricsHelper.registerMetricsRegistry(_metricsRegistry); final ControllerMetrics controllerMetrics = new ControllerMetrics(_metricsRegistry); try { LOGGER.info("Starting Pinot Helix resource manager and connecting to Zookeeper"); helixResourceManager.start(); // Helix resource manager must be started in order to create PinotLLCRealtimeSegmentManager PinotLLCRealtimeSegmentManager.create(helixResourceManager, config); LOGGER.info("Starting Pinot REST API component"); component.start(); LOGGER.info("Starting retention manager"); retentionManager.start(); LOGGER.info("Starting validation manager"); validationManager.start(); LOGGER.info("Starting realtime segment manager"); realtimeSegmentsManager.start(controllerMetrics); PinotLLCRealtimeSegmentManager.getInstance().start(); LOGGER.info("Starting segment status manager"); segmentStatusChecker.start(controllerMetrics); LOGGER.info("Pinot controller ready and listening on port {} for API requests", config.getControllerPort()); LOGGER.info("Controller services available at http://{}:{}/", config.getControllerHost(), config.getControllerPort()); } catch (final Exception e) { LOGGER.error("Caught exception while starting controller", e); Utils.rethrowException(e); throw new AssertionError("Should not reach this"); } controllerMetrics.addCallbackGauge("helix.connected", new Callable<Long>() { @Override public Long call() throws Exception { return helixResourceManager.getHelixZkManager().isConnected() ? 1L : 0L; } }); controllerMetrics.addCallbackGauge("helix.leader", new Callable<Long>() { @Override public Long call() throws Exception { return helixResourceManager.getHelixZkManager().isLeader() ? 1L : 0L; } }); helixResourceManager.getHelixZkManager().addPreConnectCallback(new PreConnectCallback() { @Override public void onPreConnect() { controllerMetrics.addMeteredGlobalValue(ControllerMeter.HELIX_ZOOKEEPER_RECONNECTS, 1L); } }); controllerMetrics.initializeGlobalMeters(); ControllerRestApplication.setControllerMetrics(controllerMetrics); }
From source file:com.gisgraphy.domain.geoloc.service.fulltextsearch.FulltextSearchEngineTest.java
@Test public void testGetUrl() { String urlOfSolrClient = "URLOfSolRclient"; assertTrue(fullTextSearchEngine.isAlive()); FullTextSearchEngine fullTextSearchEngineTest = new FullTextSearchEngine( new MultiThreadedHttpConnectionManager()); IsolrClient mockSolClient = EasyMock.createMock(IsolrClient.class); EasyMock.expect(mockSolClient.getURL()).andReturn(urlOfSolrClient); EasyMock.replay(mockSolClient);/*w ww .java 2 s . c o m*/ fullTextSearchEngineTest.setSolrClient(mockSolClient); assertEquals(urlOfSolrClient, fullTextSearchEngineTest.getURL()); EasyMock.verify(mockSolClient); }
From source file:com.jivesoftware.os.jive.utils.http.client.HttpClientFactoryProvider.java
public HttpClientFactory createHttpClientFactory(final Collection<HttpClientConfiguration> configurations) { return new HttpClientFactory() { @Override/*from w w w .j av a 2 s .c o m*/ public HttpClient createClient(String host, int port) { ApacheHttpClient31BackedHttpClient httpClient = createApacheClient(); HostConfiguration hostConfiguration = new HostConfiguration(); configureSsl(hostConfiguration, host, port, httpClient); configureProxy(hostConfiguration, httpClient); httpClient.setHostConfiguration(hostConfiguration); configureOAuth(httpClient); return httpClient; } private ApacheHttpClient31BackedHttpClient createApacheClient() { HttpClientConfig httpClientConfig = locateConfig(HttpClientConfig.class, HttpClientConfig.newBuilder().build()); HttpConnectionManager connectionManager = createConnectionManager(httpClientConfig); org.apache.commons.httpclient.HttpClient client = new org.apache.commons.httpclient.HttpClient( connectionManager); client.getParams().setParameter(HttpMethodParams.COOKIE_POLICY, CookiePolicy.RFC_2109); client.getParams().setParameter(HttpMethodParams.PROTOCOL_VERSION, HttpVersion.HTTP_1_1); client.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8"); client.getParams().setBooleanParameter(HttpMethodParams.USE_EXPECT_CONTINUE, false); client.getParams().setBooleanParameter(HttpConnectionParams.STALE_CONNECTION_CHECK, true); client.getParams().setParameter(HttpConnectionParams.CONNECTION_TIMEOUT, httpClientConfig.getSocketTimeoutInMillis() > 0 ? httpClientConfig.getSocketTimeoutInMillis() : 0); client.getParams().setParameter(HttpConnectionParams.SO_TIMEOUT, httpClientConfig.getSocketTimeoutInMillis() > 0 ? httpClientConfig.getSocketTimeoutInMillis() : 0); return new ApacheHttpClient31BackedHttpClient(client, httpClientConfig.getCopyOfHeadersForEveryRequest()); } @SuppressWarnings("unchecked") private <T> T locateConfig(Class<? extends T> _class, T defaultConfiguration) { for (HttpClientConfiguration configuration : configurations) { if (_class.isInstance(configuration)) { return (T) configuration; } } return defaultConfiguration; } private boolean hasValidProxyUsernameAndPasswordSettings(HttpClientProxyConfig httpClientProxyConfig) { return StringUtils.isNotBlank(httpClientProxyConfig.getProxyUsername()) && StringUtils.isNotBlank(httpClientProxyConfig.getProxyPassword()); } private HttpConnectionManager createConnectionManager(HttpClientConfig config) { MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); if (config.getMaxConnectionsPerHost() > 0) { connectionManager.getParams() .setDefaultMaxConnectionsPerHost(config.getMaxConnectionsPerHost()); } else { connectionManager.getParams().setDefaultMaxConnectionsPerHost(Integer.MAX_VALUE); } if (config.getMaxConnections() > 0) { connectionManager.getParams().setMaxTotalConnections(config.getMaxConnections()); } return connectionManager; } private void configureOAuth(ApacheHttpClient31BackedHttpClient httpClient) { HttpClientOAuthConfig httpClientOAuthConfig = locateConfig(HttpClientOAuthConfig.class, null); if (httpClientOAuthConfig != null) { String serviceName = httpClientOAuthConfig.getServiceName(); HttpClientConsumerKeyAndSecretProvider consumerKeyAndSecretProvider = httpClientOAuthConfig .getConsumerKeyAndSecretProvider(); String consumerKey = consumerKeyAndSecretProvider.getConsumerKey(serviceName); if (StringUtils.isEmpty(consumerKey)) { throw new RuntimeException( "could create oauth client because consumerKey is null or empty for service:" + serviceName); } String consumerSecret = consumerKeyAndSecretProvider.getConsumerSecret(serviceName); if (StringUtils.isEmpty(consumerSecret)) { throw new RuntimeException( "could create oauth client because consumerSecret is null or empty for service:" + serviceName); } httpClient.setConsumerTokens(consumerKey, consumerSecret); } } private void configureProxy(HostConfiguration hostConfiguration, ApacheHttpClient31BackedHttpClient httpClient) { HttpClientProxyConfig httpClientProxyConfig = locateConfig(HttpClientProxyConfig.class, null); if (httpClientProxyConfig != null) { hostConfiguration.setProxy(httpClientProxyConfig.getProxyHost(), httpClientProxyConfig.getProxyPort()); if (hasValidProxyUsernameAndPasswordSettings(httpClientProxyConfig)) { HttpState state = new HttpState(); state.setProxyCredentials(AuthScope.ANY, new UsernamePasswordCredentials(httpClientProxyConfig.getProxyUsername(), httpClientProxyConfig.getProxyPassword())); httpClient.setState(state); } } } private void configureSsl(HostConfiguration hostConfiguration, String host, int port, ApacheHttpClient31BackedHttpClient httpClient) throws IllegalStateException { HttpClientSSLConfig httpClientSSLConfig = locateConfig(HttpClientSSLConfig.class, null); if (httpClientSSLConfig != null) { Protocol sslProtocol; if (httpClientSSLConfig.getCustomSSLSocketFactory() != null) { sslProtocol = new Protocol(HTTPS_PROTOCOL, new CustomSecureProtocolSocketFactory( httpClientSSLConfig.getCustomSSLSocketFactory()), SSL_PORT); } else { sslProtocol = Protocol.getProtocol(HTTPS_PROTOCOL); } hostConfiguration.setHost(host, port, sslProtocol); httpClient.setUsingSSL(); } else { hostConfiguration.setHost(host, port); } } }; }
From source file:com.fatwire.dta.sscrawler.URLReaderService.java
public void start(final ProgressMonitor monitor) { connectionManager = new MultiThreadedHttpConnectionManager(); connectionManager.getParams().setConnectionTimeout(30000); connectionManager.getParams().setDefaultMaxConnectionsPerHost(1500); connectionManager.getParams().setMaxTotalConnections(30000); final MBeanServer platform = java.lang.management.ManagementFactory.getPlatformMBeanServer(); try {//ww w . j a v a 2 s . c o m platform.registerMBean(new ReaderService(scheduler, connectionManager), new ObjectName("com.fatwire.crawler:name=scheduler")); } catch (final Throwable t) { log.error(t.getMessage(), t); } monitor.beginTask("Crawling on " + hostConfig.toString(), maxPages == Integer.MAX_VALUE ? -1 : maxPages); scheduler.monitor = monitor; for (final QueryString thingToDo : startUrls) { scheduler.schedulePage(thingToDo); } scheduler.waitForlAllTasksToFinish(); try { connectionManager.shutdown(); } catch (final Throwable t) { log.error(t.getMessage(), t); } try { platform.unregisterMBean(new ObjectName("com.fatwire.crawler:name=scheduler")); } catch (final Throwable t) { log.error(t.getMessage(), t); } monitor.done(); }
From source file:com.smartitengineering.util.rest.client.jersey.cache.CustomResolverBasedCacheableClientTest.java
protected Client getClient(ApacheHttpClientConfig clientConfig) { if (CLIENT_CACHE.containsKey(clientConfig)) { return CLIENT_CACHE.get(clientConfig); }//from w w w . j a v a 2 s . c o m clientConfig.getClasses().add(Resources.HeaderWriter.class); final MultiThreadedHttpConnectionManager multiThreadedHttpConnectionManager = new MultiThreadedHttpConnectionManager(); HttpClient httpClient = new HttpClient(multiThreadedHttpConnectionManager); CacheableClientHandler handler = new CacheableClientHandler(httpClient, clientConfig); Client client = CacheableClient.create(clientConfig); CLIENT_CACHE.put(clientConfig, client); HTTP_CLIENTS.put(client, new SimpleEntry<HttpClient, MultiThreadedHttpConnectionManager>(httpClient, multiThreadedHttpConnectionManager)); return client; }
From source file:com.twinsoft.convertigo.engine.util.HttpUtils.java
public static HttpClient makeHttpClient3(boolean usePool) { HttpClient httpClient;// w w w. j a v a 2 s .c o m if (usePool) { MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); int maxTotalConnections = 100; try { maxTotalConnections = new Integer( EnginePropertiesManager.getProperty(PropertyName.HTTP_CLIENT_MAX_TOTAL_CONNECTIONS)) .intValue(); } catch (NumberFormatException e) { Engine.logEngine.warn("Unable to retrieve the max number of connections; defaults to 100."); } int maxConnectionsPerHost = 50; try { maxConnectionsPerHost = new Integer( EnginePropertiesManager.getProperty(PropertyName.HTTP_CLIENT_MAX_CONNECTIONS_PER_HOST)) .intValue(); } catch (NumberFormatException e) { Engine.logEngine .warn("Unable to retrieve the max number of connections per host; defaults to 100."); } HttpConnectionManagerParams httpConnectionManagerParams = new HttpConnectionManagerParams(); httpConnectionManagerParams.setDefaultMaxConnectionsPerHost(maxConnectionsPerHost); httpConnectionManagerParams.setMaxTotalConnections(maxTotalConnections); connectionManager.setParams(httpConnectionManagerParams); httpClient = new HttpClient(connectionManager); } else { httpClient = new HttpClient(); } HttpClientParams httpClientParams = (HttpClientParams) HttpClientParams.getDefaultParams(); httpClientParams.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); /** #741 : belambra wants only one Set-Cookie header */ httpClientParams.setParameter("http.protocol.single-cookie-header", Boolean.TRUE); /** #5066 : httpClient auto retries failed request up to 3 times by default */ httpClientParams.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(0, false)); httpClient.setParams(httpClientParams); return httpClient; }