List of usage examples for org.apache.http.client.config CookieSpecs BROWSER_COMPATIBILITY
String BROWSER_COMPATIBILITY
To view the source code for org.apache.http.client.config CookieSpecs BROWSER_COMPATIBILITY.
Click Source Link
From source file:org.sonar.runner.Main.java
Main(Exit exit, Cli cli, Conf conf, RunnerFactory runnerFactory) {
this.exit = exit;
this.cli = cli;
this.conf = conf;
this.runnerFactory = runnerFactory;
// apache http client setup
this.httpCookieStore = new BasicCookieStore();
this.globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).build();
this.httpClientContext = HttpClientContext.create();
this.httpClientContext.setCookieStore(httpCookieStore);
this.httpClientBuilder = HttpClientBuilder.create().setDefaultRequestConfig(globalConfig)
.setDefaultCookieStore(httpCookieStore);
}
From source file:org.aliuge.crawler.fetcher.DefaultFetcher.java
public DefaultFetcher createFetcher(FetchConfig config) { // /*w w w. j ava 2s. c o m*/ connectionManager = new PoolingHttpClientConnectionManager(); BasicCookieStore cookieStore = new BasicCookieStore(); CookieSpecProvider easySpecProvider = new CookieSpecProvider() { public CookieSpec create(HttpContext context) { return new BrowserCompatSpec() { @Override public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException { // Oh, I am easy } }; } }; Registry<CookieSpecProvider> r = RegistryBuilder.<CookieSpecProvider>create() .register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory()) .register(CookieSpecs.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory()) .register("easy", easySpecProvider).build(); // Create global request configuration defaultRequestConfig = RequestConfig.custom().setCookieSpec("easy").setSocketTimeout(10000) .setConnectTimeout(10000).build(); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); // Create an HttpClient with the given custom dependencies and // configuration. httpClient = HttpClients.custom().setConnectionManager(connectionManager).setDefaultCookieStore(cookieStore) .setDefaultCookieSpecRegistry(r) /* .setProxy(new HttpHost("myproxy", 8080)) */ .setDefaultRequestConfig(defaultRequestConfig).build(); if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } /* * connectionMonitorThread.start(); try { * connectionMonitorThread.join(); } catch (InterruptedException e) { // * TODO Auto-generated catch block e.printStackTrace(); } */ return this; }
From source file:tradeok.HttpTool.java
public static String postJsonBody(String url, int timeout, Map<String, Object> map, String encoding) throws Exception { HttpPost post = new HttpPost(url); try {/*from w w w . jav a 2s . co m*/ RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout) .setConnectTimeout(timeout).setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false) .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).build(); post.setConfig(requestConfig); post.setHeader("User-Agent", USER_AGENT); post.setHeader("Accept", "text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8"); post.setHeader("Connection", "keep-alive"); post.setHeader("Content-Type", "application/json; charset=UTF-8"); String str1 = object2json(map).replace("\\", ""); post.setEntity(new StringEntity(str1, encoding)); CloseableHttpResponse response = httpclient.execute(post); try { HttpEntity entity = response.getEntity(); try { if (entity != null) { String str = EntityUtils.toString(entity, encoding); return str; } } finally { if (entity != null) { entity.getContent().close(); } } } finally { if (response != null) { response.close(); } } } finally { post.releaseConnection(); } return ""; }
From source file:org.sonatype.nexus.proxy.storage.remote.httpclient.HttpClientManagerImpl.java
/** * Configures the fresh instance of HttpClient for given proxy repository specific needs. Right now it sets * appropriate redirect strategy only.//from ww w .jav a 2 s . c o m */ private void configure(final ProxyRepository proxyRepository, final Builder builder) { // set proxy redirect strategy builder.getHttpClientBuilder().setRedirectStrategy(new NexusRedirectStrategy()); // MEXUS-7915: Allow use of circular redirects, if set final String proxyHostName = normalizeHostname(proxyRepository); if (enableCircularRedirectsForHosts.contains(proxyHostName)) { log.info("Allowing circular redirects in proxy {}", proxyRepository); builder.getRequestConfigBuilder().setCircularRedirectsAllowed(true); // allow circular redirects builder.getRequestConfigBuilder().setMaxRedirects(10); // lessen max redirects from default 50 } // MEXUS-7915: Allow use of cookies, if set if (useCookiesForHosts.contains(proxyHostName)) { log.info("Allowing cookie use in proxy {}", proxyRepository); builder.getHttpClientBuilder().setDefaultCookieStore(new BasicCookieStore()); // in memory only builder.getRequestConfigBuilder().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY); // emulate browsers } }
From source file:com.jaeksoft.searchlib.crawler.web.spider.HttpAbstract.java
protected void execute(HttpRequestBase httpBaseRequest, CredentialItem credentialItem, List<CookieItem> cookies) throws ClientProtocolException, IOException, URISyntaxException { if (!CollectionUtils.isEmpty(cookies)) { List<Cookie> cookieList = cookieStore.getCookies(); for (CookieItem cookie : cookies) { Cookie newCookie = cookie.getCookie(); if (!cookieList.contains(newCookie)) cookieStore.addCookie(newCookie); }/*w ww . j av a 2 s.co m*/ } this.httpBaseRequest = httpBaseRequest; // No more than one 1 minute to establish the connection // No more than 10 minutes to establish the socket // Enable stales connection checking // Cookies uses browser compatibility RequestConfig.Builder configBuilber = RequestConfig.custom().setSocketTimeout(1000 * 60 * 10) .setConnectTimeout(1000 * 60).setStaleConnectionCheckEnabled(true) .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY); if (credentialItem == null) credentialsProvider.clear(); else credentialItem.setUpCredentials(credentialsProvider); URI uri = httpBaseRequest.getURI(); if (proxyHandler != null) proxyHandler.check(configBuilber, uri, credentialsProvider); httpBaseRequest.setConfig(configBuilber.build()); httpClientContext = new HttpClientContext(); httpResponse = httpClient.execute(httpBaseRequest, httpClientContext); if (httpResponse == null) return; statusLine = httpResponse.getStatusLine(); httpEntity = httpResponse.getEntity(); }
From source file:tradeok.HttpTool.java
@SuppressWarnings("deprecation") public static String invokeGet(String url, Map<String, String> params, String encode, int connectTimeout) throws Exception { String responseString = null; RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout) .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).build(); StringBuilder sb = new StringBuilder(); sb.append(url);//from w ww. j av a 2 s .c om int i = 0; if (params != null) { for (Entry<String, String> entry : params.entrySet()) { if (i == 0 && !url.contains("?")) { sb.append("?"); } else { sb.append("&"); } sb.append(entry.getKey()); sb.append("="); String value = entry.getValue(); try { sb.append(URLEncoder.encode(value, "UTF-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); System.out.printf("\nwarn:encode http get params error, value is " + value, e); sb.append(URLEncoder.encode(value)); } i++; } } // System.out.printf("\ninfo:[HttpUtils Get] begin invoke:" // + sb.toString()); HttpGet get = new HttpGet(sb.toString()); get.setConfig(requestConfig); get.setHeader("Connection", "keep-alive"); try { CloseableHttpResponse response = httpclient.execute(get); try { HttpEntity entity = response.getEntity(); try { if (entity != null) { responseString = EntityUtils.toString(entity, encode); } } finally { if (entity != null) { entity.getContent().close(); } } } finally { if (response != null) { response.close(); } } } finally { get.releaseConnection(); } return responseString; }
From source file:com.tremolosecurity.scale.config.ScaleCommonConfig.java
public HttpClientInfo createHttpClientInfo() { SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslctx, new AllowAllHostnameVerifier()); PlainConnectionSocketFactory sf = PlainConnectionSocketFactory.getSocketFactory(); Registry<ConnectionSocketFactory> r = RegistryBuilder.<ConnectionSocketFactory>create().register("http", sf) .register("https", sslsf).build(); RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY) .setRedirectsEnabled(true).build(); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(r); return new HttpClientInfo(cm, globalConfig); }
From source file:com.twinsoft.convertigo.engine.util.HttpUtils.java
@SuppressWarnings("deprecation") public static CloseableHttpClient makeHttpClient4(boolean usePool) { HttpClientBuilder httpClientBuilder = HttpClients.custom(); httpClientBuilder.setDefaultRequestConfig( RequestConfig.custom().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).build()); if (usePool) { PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); int maxTotalConnections = 100; try {//from w ww . ja v a2s.c o m maxTotalConnections = new Integer( EnginePropertiesManager.getProperty(PropertyName.HTTP_CLIENT_MAX_TOTAL_CONNECTIONS)) .intValue(); } catch (NumberFormatException e) { Engine.logEngine.warn("Unable to retrieve the max number of connections; defaults to 100."); } int maxConnectionsPerHost = 50; try { maxConnectionsPerHost = new Integer( EnginePropertiesManager.getProperty(PropertyName.HTTP_CLIENT_MAX_CONNECTIONS_PER_HOST)) .intValue(); } catch (NumberFormatException e) { Engine.logEngine .warn("Unable to retrieve the max number of connections per host; defaults to 100."); } connManager.setDefaultMaxPerRoute(maxConnectionsPerHost); connManager.setMaxTotal(maxTotalConnections); httpClientBuilder.setConnectionManager(connManager); } return httpClientBuilder.build(); }
From source file:de.tudarmstadt.ukp.shibhttpclient.ShibHttpClient.java
/** * Create a new client (with an explicit proxy and possibly transparent authentication) * // w ww .j a v a2 s .co m * @param aIdpUrl * the URL of the IdP. Should probably be something ending in "/SAML2/SOAP/ECP" * @param aUsername * the user name to log into the IdP. * @param aPassword * the password to log in to the IdP. * @param aProxy * if not {@code null}, use this proxy instead of the default system proxy (if any) * @param anyCert * if {@code true}, accept any certificate from any remote host. Otherwise, * certificates need to be installed in the JRE. * @param transparentAuth * if {@code true} (default), add a HttpRequestPostProcessor to transparently * authenticate. Otherwise, you must handle the authentication process yourself. */ public ShibHttpClient(String aIdpUrl, String aUsername, String aPassword, HttpHost aProxy, boolean anyCert, boolean transparentAuth) { setIdpUrl(aIdpUrl); setUsername(aUsername); setPassword(aPassword); // Use a pooling connection manager, because we'll have to do a call out to the IdP // while still being in a connection with the SP PoolingHttpClientConnectionManager connMgr; if (anyCert) { try { SSLContextBuilder builder = new SSLContextBuilder(); builder.loadTrustMaterial(null, new TrustSelfSignedStrategy()); Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder .<ConnectionSocketFactory>create().register("http", new PlainConnectionSocketFactory()) .register("https", new SSLConnectionSocketFactory(builder.build(), SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER)) .build(); connMgr = new PoolingHttpClientConnectionManager(socketFactoryRegistry); } catch (GeneralSecurityException e) { // There shouldn't be any of these exceptions, because we do not use an actual // keystore throw new IllegalStateException(e); } } else { connMgr = new PoolingHttpClientConnectionManager(); } connMgr.setMaxTotal(10); connMgr.setDefaultMaxPerRoute(5); // The client needs to remember the auth cookie cookieStore = new BasicCookieStore(); RequestConfig globalRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY) .build(); // Let's throw all common client elements into one builder object HttpClientBuilder customClient = HttpClients.custom().setConnectionManager(connMgr) // The client needs to remember the auth cookie .setDefaultRequestConfig(globalRequestConfig).setDefaultCookieStore(cookieStore) // Add the ECP/PAOS headers - needs to be added first so the cookie we get from // the authentication can be handled by the RequestAddCookies interceptor later .addInterceptorFirst(new HttpRequestPreprocessor()); // Automatically log into IdP if transparent Shibboleth authentication handling is requested (default) if (transparentAuth) { customClient = customClient.addInterceptorFirst(new HttpRequestPostprocessor()); } // Build the client with/without proxy settings if (aProxy == null) { // use the proxy settings of the JVM, if specified client = customClient.setRoutePlanner(new SystemDefaultRoutePlanner(ProxySelector.getDefault())) .build(); } else { // use the explicit proxy client = customClient.setProxy(aProxy).build(); } parserPool = new BasicParserPool(); parserPool.setNamespaceAware(true); }