List of usage examples for org.apache.http.impl.conn.tsccm ThreadSafeClientConnManager ThreadSafeClientConnManager
public ThreadSafeClientConnManager(final SchemeRegistry schreg)
From source file:yin.autoflowcontrol.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); HttpParams params = new BasicHttpParams(); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); paramsBean.setContentCharset("UTF-8"); paramsBean.setUseExpectContinue(false); params.setParameter(CoreProtocolPNames.USER_AGENT, config.getUserAgentString()); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeout()); params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout()); params.setBooleanParameter("http.protocol.handle-redirects", false); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); if (config.isIncludeHttpsPages()) { schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); }/*from w ww. j a v a 2 s . co m*/ connectionManager = new ThreadSafeClientConnManager(schemeRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); httpClient = new DefaultHttpClient(connectionManager, params); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { httpClient.getCredentialsProvider().setCredentials( new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); } httpClient.addResponseInterceptor(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) { HeaderElement[] codecs = contentEncoding.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } }); if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:edu.hust.grid.crawl.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); HttpParams params = new BasicHttpParams(); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); paramsBean.setContentCharset("UTF-8"); paramsBean.setUseExpectContinue(false); params.setParameter(CoreProtocolPNames.USER_AGENT, config.getUserAgentString()); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeout()); params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout()); params.setBooleanParameter("http.protocol.handle-redirects", false); params.setParameter("http.language.Accept-Language", "en-us"); params.setParameter("http.protocol.content-charset", "UTF-8"); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); if (config.isIncludeHttpsPages()) { schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); }// w ww.j a v a2 s . c o m connectionManager = new ThreadSafeClientConnManager(schemeRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); httpClient = new DefaultHttpClient(connectionManager, params); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { httpClient.getCredentialsProvider().setCredentials( new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); httpClient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:16.0) Gecko/20100101 Firefox/16.0"); } httpClient.addResponseInterceptor(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) { HeaderElement[] codecs = contentEncoding.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } }); if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:org.apache.abdera2.common.protocol.BasicClient.java
/** * Default initialization of the Client Connection Manager, * subclasses may overload this to customize the connection * manager configuration/*ww w . j a v a 2s.c o m*/ */ protected ClientConnectionManager initConnectionManager(SchemeRegistry sr) { ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(sr); cm.setDefaultMaxPerRoute(initDefaultMaxConnectionsPerRoute()); cm.setMaxTotal(initDefaultMaxTotalConnections()); return cm; }
From source file:com.mnxfst.testing.client.TSClientPlanResultCollectCallable.java
public TSClientPlanResultCollectCallable(String hostname, int port, String uri) { this.httpHost = new HttpHost(hostname, port); this.getMethod = new HttpGet(uri.toString()); // TODO setting? SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); ThreadSafeClientConnManager threadSafeClientConnectionManager = new ThreadSafeClientConnManager( schemeRegistry);//from w w w .j a v a 2 s. c om threadSafeClientConnectionManager.setMaxTotal(20); threadSafeClientConnectionManager.setDefaultMaxPerRoute(20); this.httpClient = new DefaultHttpClient(threadSafeClientConnectionManager); }
From source file:com.mnxfst.testing.client.TSClientPlanExecCallable.java
public TSClientPlanExecCallable(String hostname, int port, String uri, byte[] testplan) { this.httpHost = new HttpHost(hostname, port); // this.getMethod = new HttpGet(uri.toString()); this.postMethod = new HttpPost(uri.toString()); try {/*from w ww .j a va 2 s.co m*/ String convertedTestplan = new String(testplan, "UTF-8"); postMethod.setEntity(new StringEntity( TSClient.REQUEST_PARAMETER_TESTPLAN + "=" + URLEncoder.encode(convertedTestplan, "UTF-8"))); } catch (UnsupportedEncodingException e) { throw new RuntimeException("Unsupported encoding exception. Error: " + e.getMessage()); } // TODO setting? SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); ThreadSafeClientConnManager threadSafeClientConnectionManager = new ThreadSafeClientConnManager( schemeRegistry); threadSafeClientConnectionManager.setMaxTotal(20); threadSafeClientConnectionManager.setDefaultMaxPerRoute(20); this.httpClient = new DefaultHttpClient(threadSafeClientConnectionManager); }
From source file:com.mockey.ClientExecuteProxy.java
/** * //from w w w. j a v a 2s . co m * @param twistInfo * @param proxyServer * @param realServiceUrl * @param httpMethod * @param request * @return * @throws ClientExecuteProxyException */ public ResponseFromService execute(TwistInfo twistInfo, ProxyServerModel proxyServer, Url realServiceUrl, boolean allowRedirectFollow, RequestFromClient request) throws ClientExecuteProxyException { log.info("Request: " + String.valueOf(realServiceUrl)); // general setup SchemeRegistry supportedSchemes = new SchemeRegistry(); // Register the "http" and "https" protocol schemes, they are // required by the default operator to look up socket factories. supportedSchemes.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); supportedSchemes.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); // prepare parameters HttpParams params = new BasicHttpParams(); HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); HttpProtocolParams.setContentCharset(params, HTTP.ISO_8859_1); HttpProtocolParams.setUseExpectContinue(params, false); ClientConnectionManager ccm = new ThreadSafeClientConnManager(supportedSchemes); DefaultHttpClient httpclient = new DefaultHttpClient(ccm, params); if (!allowRedirectFollow) { // Do NOT allow for 302 REDIRECT httpclient.setRedirectStrategy(new DefaultRedirectStrategy() { public boolean isRedirected(HttpRequest request, HttpResponse response, HttpContext context) { boolean isRedirect = false; try { isRedirect = super.isRedirected(request, response, context); } catch (ProtocolException e) { // TODO Auto-generated catch block e.printStackTrace(); } if (!isRedirect) { int responseCode = response.getStatusLine().getStatusCode(); if (responseCode == 301 || responseCode == 302) { return true; } } return isRedirect; } }); } else { // Yes, allow for 302 REDIRECT // Nothing needed here. } // Prevent CACHE, 304 not modified // httpclient.addRequestInterceptor(new HttpRequestInterceptor() { // public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { // // request.setHeader("If-modified-Since", "Fri, 13 May 2006 23:54:18 GMT"); // // } // }); CookieStore cookieStore = httpclient.getCookieStore(); for (Cookie httpClientCookie : request.getHttpClientCookies()) { // HACK: // httpClientCookie.getValue(); cookieStore.addCookie(httpClientCookie); } // httpclient.setCookieStore(cookieStore); if (ClientExecuteProxy.cookieStore == null) { ClientExecuteProxy.cookieStore = httpclient.getCookieStore(); } else { httpclient.setCookieStore(ClientExecuteProxy.cookieStore); } StringBuffer requestCookieInfo = new StringBuffer(); // Show what cookies are in the store . for (Cookie cookie : ClientExecuteProxy.cookieStore.getCookies()) { log.debug("Cookie in the cookie STORE: " + cookie.toString()); requestCookieInfo.append(cookie.toString() + "\n\n\n"); } if (proxyServer.isProxyEnabled()) { // make sure to use a proxy that supports CONNECT httpclient.getCredentialsProvider().setCredentials(proxyServer.getAuthScope(), proxyServer.getCredentials()); httpclient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyServer.getHttpHost()); } // TWISTING Url originalRequestUrlBeforeTwisting = null; if (twistInfo != null) { String fullurl = realServiceUrl.getFullUrl(); String twistedUrl = twistInfo.getTwistedValue(fullurl); if (twistedUrl != null) { originalRequestUrlBeforeTwisting = realServiceUrl; realServiceUrl = new Url(twistedUrl); } } ResponseFromService responseMessage = null; try { HttpHost htttphost = new HttpHost(realServiceUrl.getHost(), realServiceUrl.getPort(), realServiceUrl.getScheme()); HttpResponse response = httpclient.execute(htttphost, request.postToRealServer(realServiceUrl)); if (response.getStatusLine().getStatusCode() == 302) { log.debug("FYI: 302 redirect occuring from " + realServiceUrl.getFullUrl()); } responseMessage = new ResponseFromService(response); responseMessage.setOriginalRequestUrlBeforeTwisting(originalRequestUrlBeforeTwisting); responseMessage.setRequestUrl(realServiceUrl); } catch (Exception e) { log.error(e); throw new ClientExecuteProxyException("Unable to retrieve a response. ", realServiceUrl, e); } finally { // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } // Parse out the response information we're looking for // StringBuffer responseCookieInfo = new StringBuffer(); // // Show what cookies are in the store . // for (Cookie cookie : ClientExecuteProxy.cookieStore.getCookies()) { // log.info("Cookie in the cookie STORE: " + cookie.toString()); // responseCookieInfo.append(cookie.toString() + "\n\n\n"); // // } // responseMessage.setRequestCookies(requestCookieInfo.toString()); // responseMessage.setResponseCookies(responseCookieInfo.toString()); return responseMessage; }
From source file:org.mobicents.xcap.client.impl.XcapClientImpl.java
/** * /*from w w w . j ava2s . c o m*/ */ public XcapClientImpl() { HttpParams params = new SyncBasicHttpParams(); params.setParameter(HttpProtocolParams.PROTOCOL_VERSION, HttpVersion.HTTP_1_1); //params.setBooleanParameter(HttpProtocolParams.USE_EXPECT_CONTINUE, // false); //params.setBooleanParameter(HttpConnectionParams.STALE_CONNECTION_CHECK, // false); //params.setIntParameter(HttpConnectionParams.SOCKET_BUFFER_SIZE, // 8 * 1024); //params.setIntParameter(HttpConnectionParams.SO_TIMEOUT, // 15000); params.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, "UTF-8"); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); this.client = new DefaultHttpClient(new ThreadSafeClientConnManager(schemeRegistry), params); this.client.setHttpRequestRetryHandler(new HttpRequestRetryHandler() { public boolean retryRequest(final IOException exception, int executionCount, final HttpContext context) { return false; } }); client.setCredentialsProvider(credentialsProvider); }
From source file:edu.uci.ics.crawler4j.crawler.fetcher.PageFetcher.java
public PageFetcher(ICrawlerSettings config) { politenessDelay = config.getPolitenessDelay(); maxDownloadSize = config.getMaxDownloadSize(); show404Pages = config.getShow404Pages(); ignoreBinary = !config.getIncludeBinaryContent(); cache = config.getCacheProvider();/*w ww . jav a 2 s .com*/ HttpParams params = new BasicHttpParams(); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); paramsBean.setContentCharset("UTF-8"); paramsBean.setUseExpectContinue(false); params.setParameter("http.useragent", config.getUserAgent()); params.setIntParameter("http.socket.timeout", config.getSocketTimeout()); params.setIntParameter("http.connection.timeout", config.getConnectionTimeout()); params.setBooleanParameter("http.protocol.handle-redirects", false); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); if (config.getAllowHttps()) { schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); } connectionManager = new ThreadSafeClientConnManager(schemeRegistry); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); connectionManager.setMaxTotal(config.getMaxTotalConnections()); logger.setLevel(Level.INFO); httpclient = new DefaultHttpClient(connectionManager, params); }
From source file:jetbrains.buildServer.commitPublisher.github.api.impl.HttpClientWrapperImpl.java
public HttpClientWrapperImpl() throws UnrecoverableKeyException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { final String serverVersion = ServerVersionHolder.getVersion().getDisplayVersion(); final HttpParams ps = new BasicHttpParams(); DefaultHttpClient.setDefaultHttpParams(ps); final int timeout = TeamCityProperties.getInteger("teamcity.github.http.timeout", 300 * 1000); HttpConnectionParams.setConnectionTimeout(ps, timeout); HttpConnectionParams.setSoTimeout(ps, timeout); HttpProtocolParams.setUserAgent(ps, "JetBrains TeamCity " + serverVersion); final SchemeRegistry schemaRegistry = SchemeRegistryFactory.createDefault(); final SSLSocketFactory sslSocketFactory = new SSLSocketFactory(new TrustStrategy() { public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException { return !TeamCityProperties.getBoolean("teamcity.github.verify.ssl.certificate"); }//from www .ja v a 2 s .c om }) { @Override public Socket connectSocket(int connectTimeout, Socket socket, HttpHost host, InetSocketAddress remoteAddress, InetSocketAddress localAddress, HttpContext context) throws IOException { if (socket instanceof SSLSocket) { try { PropertyUtils.setProperty(socket, "host", host.getHostName()); } catch (Exception ex) { LOG.warn(String.format( "A host name is not passed to SSL connection for the purpose of supporting SNI due to the following exception: %s", ex.toString())); } } return super.connectSocket(connectTimeout, socket, host, remoteAddress, localAddress, context); } }; schemaRegistry.register(new Scheme("https", 443, sslSocketFactory)); final DefaultHttpClient httpclient = new DefaultHttpClient(new ThreadSafeClientConnManager(schemaRegistry), ps); setupProxy(httpclient); httpclient.setRoutePlanner(new ProxySelectorRoutePlanner( httpclient.getConnectionManager().getSchemeRegistry(), ProxySelector.getDefault())); httpclient.addRequestInterceptor(new RequestAcceptEncoding()); httpclient.addResponseInterceptor(new ResponseContentEncoding()); httpclient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(3, true)); myClient = httpclient; }