List of usage examples for org.apache.http.conn.socket PlainConnectionSocketFactory INSTANCE
PlainConnectionSocketFactory INSTANCE
To view the source code for org.apache.http.conn.socket PlainConnectionSocketFactory INSTANCE.
Click Source Link
From source file:ee.ria.xroad.proxy.clientproxy.FastestConnectionSelectingSSLSocketFactoryIntegrationTest.java
private static void createClient() throws Exception { RegistryBuilder<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder .<ConnectionSocketFactory>create(); socketFactoryRegistry.register("http", PlainConnectionSocketFactory.INSTANCE); socketFactoryRegistry.register("https", createSSLSocketFactory()); PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager( socketFactoryRegistry.build()); connMgr.setMaxTotal(CLIENT_MAX_TOTAL_CONNECTIONS); connMgr.setDefaultMaxPerRoute(CLIENT_MAX_CONNECTIONS_PER_ROUTE); int timeout = SystemProperties.getClientProxyTimeout(); RequestConfig.Builder rb = RequestConfig.custom(); rb.setConnectTimeout(timeout);//w w w .jav a 2s. co m rb.setConnectionRequestTimeout(timeout); rb.setSocketTimeout(timeout); HttpClientBuilder cb = HttpClients.custom(); cb.setConnectionManager(connMgr); cb.setDefaultRequestConfig(rb.build()); // Disable request retry cb.setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); client = cb.build(); }
From source file:io.github.cidisk.indexcrawler.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false) .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).setRedirectsEnabled(false) .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout()) .build();//from www .ja va 2 s.c o m RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create(); connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE); if (config.isIncludeHttpsPages()) { try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174 // By always trusting the ssl certificate SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { @Override public boolean isTrusted(final X509Certificate[] chain, String authType) { return true; } }).build(); SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); connRegistryBuilder.register("https", sslsf); } catch (Exception e) { logger.warn("Exception thrown while trying to register https"); logger.debug("Stacktrace", e); } } Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build(); connectionManager = new PoolingHttpClientConnectionManager(connRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setDefaultRequestConfig(requestConfig); clientBuilder.setConnectionManager(connectionManager); clientBuilder.setUserAgent(config.getUserAgentString()); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); clientBuilder.setProxy(proxy); logger.debug("Working through Proxy: {}", proxy.getHostName()); } httpClient = clientBuilder.build(); if (config.getAuthInfos() != null && !config.getAuthInfos().isEmpty()) { doAuthetication(config.getAuthInfos()); } if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:com.crawler.app.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false) .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).setRedirectsEnabled(false) //.setRelativeRedirectsAllowed(true) .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout()) .build();/*from ww w .j a v a 2s . c om*/ RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create(); connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE); if (config.isIncludeHttpsPages()) { try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174 // By always trusting the ssl certificate SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { //@Override public boolean isTrusted(final X509Certificate[] chain, String authType) { return true; } }).build(); SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); connRegistryBuilder.register("https", sslsf); } catch (Exception e) { logger.warn("Exception thrown while trying to register https"); logger.debug("Stacktrace", e); } } Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build(); connectionManager = new PoolingHttpClientConnectionManager(connRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setDefaultRequestConfig(requestConfig); clientBuilder.setConnectionManager(connectionManager); clientBuilder.setUserAgent(config.getUserAgentString()); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); clientBuilder.setProxy(proxy); logger.debug("Working through Proxy: {}", proxy.getHostName()); } httpClient = clientBuilder.build(); if (config.getAuthInfos() != null && !config.getAuthInfos().isEmpty()) { doAuthetication(config.getAuthInfos()); } if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:org.createnet.raptor.auth.AuthHttpClient.java
private CloseableHttpClient getHttpClient() throws KeyStoreException, NoSuchAlgorithmException, KeyManagementException, UnrecoverableKeyException, CertificateException, IOException { if (httpclient == null) { logger.debug("Created http client instance"); // Trust own CA and all self-signed certs SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(new File(config.token.truststore.path), config.token.truststore.password.toCharArray(), new TrustSelfSignedStrategy()).build(); // Allow TLSv1 protocol only SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext, new String[] { "TLSv1.2", "TLSv1.1", "TLSv1" }, null, SSLConnectionSocketFactory.getDefaultHostnameVerifier()); Registry socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create() .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", sslsf).build(); HttpClientConnectionManager poolingConnManager = new PoolingHttpClientConnectionManager( socketFactoryRegistry);//from w ww . jav a 2 s .c o m httpclient = HttpClients.custom() // .setSSLSocketFactory(sslsf) .setConnectionManager(poolingConnManager) // .setConnectionManagerShared(true) .build(); } return httpclient; }
From source file:crawler.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false) .setCookieSpec(config.getCookiePolicy()).setRedirectsEnabled(false) .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout()) .build();//from w w w . j av a2 s. c o m RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create(); connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE); if (config.isIncludeHttpsPages()) { try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174 // By always trusting the ssl certificate SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { @Override public boolean isTrusted(final X509Certificate[] chain, String authType) { return true; } }).build(); SSLConnectionSocketFactory sslsf = new SniSSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); connRegistryBuilder.register("https", sslsf); } catch (Exception e) { logger.warn("Exception thrown while trying to register https"); logger.debug("Stacktrace", e); } } Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build(); connectionManager = new SniPoolingHttpClientConnectionManager(connRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setDefaultRequestConfig(requestConfig); clientBuilder.setConnectionManager(connectionManager); clientBuilder.setUserAgent(config.getUserAgentString()); clientBuilder.setDefaultHeaders(config.getDefaultHeaders()); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); clientBuilder.setProxy(proxy); logger.debug("Working through Proxy: {}", proxy.getHostName()); } httpClient = clientBuilder.build(); if ((config.getAuthInfos() != null) && !config.getAuthInfos().isEmpty()) { doAuthetication(config.getAuthInfos()); } if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:de.comlineag.snc.webcrawler.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false) .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).setRedirectsEnabled(false) .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout()) .build();//w w w. j a v a 2s .c om RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create(); connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE); if (config.isIncludeHttpsPages()) { try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174 // By always trusting the ssl certificate SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { @Override public boolean isTrusted(final X509Certificate[] chain, String authType) { return true; } }).build(); SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); connRegistryBuilder.register("https", sslsf); } catch (Exception e) { logger.debug("Exception thrown while trying to register https:", e); } } Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build(); connectionManager = new PoolingHttpClientConnectionManager(connRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setDefaultRequestConfig(requestConfig); clientBuilder.setConnectionManager(connectionManager); clientBuilder.setUserAgent(config.getUserAgentString()); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); clientBuilder.setProxy(proxy); } clientBuilder.addInterceptorLast(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) { HeaderElement[] codecs = contentEncoding.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } }); httpClient = clientBuilder.build(); if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:ee.ria.xroad.common.opmonitoring.OpMonitoringDaemonHttpClient.java
/** * Creates HTTP client./*from www . j a v a 2s . c o m*/ * @param authKey the client's authentication key * @param clientMaxTotalConnections client max total connections * @param clientMaxConnectionsPerRoute client max connections per route * @param connectionTimeoutMilliseconds connection timeout in milliseconds * @param socketTimeoutMilliseconds socket timeout in milliseconds * @return HTTP client * @throws Exception if creating a HTTPS client and SSLContext * initialization fails */ public static CloseableHttpClient createHttpClient(InternalSSLKey authKey, int clientMaxTotalConnections, int clientMaxConnectionsPerRoute, int connectionTimeoutMilliseconds, int socketTimeoutMilliseconds) throws Exception { log.trace("createHttpClient()"); RegistryBuilder<ConnectionSocketFactory> sfr = RegistryBuilder.<ConnectionSocketFactory>create(); if ("https".equalsIgnoreCase(OpMonitoringSystemProperties.getOpMonitorDaemonScheme())) { sfr.register("https", createSSLSocketFactory(authKey)); } else { sfr.register("http", PlainConnectionSocketFactory.INSTANCE); } PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(sfr.build()); cm.setMaxTotal(clientMaxTotalConnections); cm.setDefaultMaxPerRoute(clientMaxConnectionsPerRoute); cm.setDefaultSocketConfig(SocketConfig.custom().setTcpNoDelay(true).build()); RequestConfig.Builder rb = RequestConfig.custom().setConnectTimeout(connectionTimeoutMilliseconds) .setConnectionRequestTimeout(connectionTimeoutMilliseconds) .setSocketTimeout(socketTimeoutMilliseconds); HttpClientBuilder cb = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(rb.build()); // Disable request retry cb.setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); return cb.build(); }
From source file:crawler.java.edu.uci.ics.crawler4j.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false) .setCookieSpec(CookieSpecs.DEFAULT).setRedirectsEnabled(false) .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout()) .build();/* w w w . j a v a 2 s . c o m*/ RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create(); connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE); if (config.isIncludeHttpsPages()) { try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174 // By always trusting the ssl certificate SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { @Override public boolean isTrusted(final X509Certificate[] chain, String authType) { return true; } }).build(); SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); connRegistryBuilder.register("https", sslsf); } catch (Exception e) { logger.warn("Exception thrown while trying to register https"); logger.debug("Stacktrace", e); } } Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build(); connectionManager = new PoolingHttpClientConnectionManager(connRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setDefaultRequestConfig(requestConfig); clientBuilder.setConnectionManager(connectionManager); clientBuilder.setUserAgent(config.getUserAgentString()); clientBuilder.setDefaultHeaders(config.getDefaultHeaders()); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); clientBuilder.setProxy(proxy); logger.debug("Working through Proxy: {}", proxy.getHostName()); } httpClient = clientBuilder.build(); if ((config.getAuthInfos() != null) && !config.getAuthInfos().isEmpty()) { doAuthetication(config.getAuthInfos()); } if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:sk.datalan.solr.impl.HttpClientUtil.java
public static HttpClientBuilder configureClient(final HttpClientConfiguration config) { HttpClientBuilder clientBuilder = HttpClientBuilder.create(); // max total connections if (config.isSetMaxConnections()) { clientBuilder.setMaxConnTotal(config.getMaxConnections()); }/*from w w w . jav a2 s . c o m*/ // max connections per route if (config.isSetMaxConnectionsPerRoute()) { clientBuilder.setMaxConnPerRoute(config.getMaxConnectionsPerRoute()); } RequestConfig.Builder requestConfigBuilder = RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH) .setExpectContinueEnabled(true).setStaleConnectionCheckEnabled(true); // connection timeout if (config.isSetConnectionTimeout()) { requestConfigBuilder.setConnectTimeout(config.getConnectionTimeout()); } // soucket timeout if (config.isSetSocketTimeout()) { requestConfigBuilder.setSocketTimeout(config.getSocketTimeout()); } // soucket timeout if (config.isSetFollowRedirects()) { requestConfigBuilder.setRedirectsEnabled(config.getFollowRedirects()); } clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); if (config.isSetUseRetry()) { if (config.getUseRetry()) { clientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler()); } else { clientBuilder.setRetryHandler(NO_RETRY); } } // basic authentication if (config.isSetBasicAuthUsername() && config.isSetBasicAuthPassword()) { CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(config.getBasicAuthUsername(), config.getBasicAuthPassword())); } if (config.isSetAllowCompression()) { clientBuilder.addInterceptorFirst(new UseCompressionRequestInterceptor()); clientBuilder.addInterceptorFirst(new UseCompressionResponseInterceptor()); } // SSL context for secure connections can be created either based on // system or application specific properties. SSLContext sslcontext = SSLContexts.createSystemDefault(); // Use custom hostname verifier to customize SSL hostname verification. X509HostnameVerifier hostnameVerifier = new BrowserCompatHostnameVerifier(); // Create a registry of custom connection socket factories for supported // protocol schemes. Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext, hostnameVerifier)).build(); clientBuilder.setConnectionManager(new PoolingHttpClientConnectionManager(socketFactoryRegistry)); return clientBuilder; }
From source file:ee.ria.xroad.proxy.serverproxy.HttpClientCreator.java
private void build() throws HttpClientCreatorException { RegistryBuilder<ConnectionSocketFactory> sfr = RegistryBuilder.create(); sfr.register("http", PlainConnectionSocketFactory.INSTANCE); try {/*from w w w .j a v a 2 s . co m*/ sfr.register("https", createSSLSocketFactory()); } catch (Exception e) { throw new HttpClientCreatorException("Creating SSL Socket Factory failed", e); } connectionManager = new PoolingHttpClientConnectionManager(sfr.build()); connectionManager.setMaxTotal(CLIENT_MAX_TOTAL_CONNECTIONS); connectionManager.setDefaultMaxPerRoute(CLIENT_MAX_CONNECTIONS_PER_ROUTE); connectionManager.setDefaultSocketConfig(SocketConfig.custom().setTcpNoDelay(true).build()); RequestConfig.Builder rb = RequestConfig.custom(); rb.setConnectTimeout(CLIENT_TIMEOUT); rb.setConnectionRequestTimeout(CLIENT_TIMEOUT); rb.setSocketTimeout(CLIENT_TIMEOUT); HttpClientBuilder cb = HttpClients.custom(); cb.setDefaultRequestConfig(rb.build()); cb.setConnectionManager(connectionManager); // Disable request retry cb.setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); httpClient = cb.build(); }