Example usage for org.apache.http.client.config CookieSpecs BROWSER_COMPATIBILITY

Introduction

In this page you can find the example usage for org.apache.http.client.config CookieSpecs BROWSER_COMPATIBILITY.

Prototype

String BROWSER_COMPATIBILITY

To view the source code for org.apache.http.client.config CookieSpecs BROWSER_COMPATIBILITY.

Click Source Link

Document

The policy that provides high degree of compatibility with common cookie management of popular HTTP agents.

Usage

From source file:network.HttpClientAdaptor.java

public String doGet(String url) {

    try {/*from w  w w . j  av a 2 s. c  o  m*/

        HttpGet httpGet = new HttpGet(url);

        RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout)
                .setConnectTimeout(timeout).setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).build();//?????????? 
        httpGet.setConfig(requestConfig);

        CloseableHttpResponse response = this.httpclient.execute(httpGet, localContext);
        BufferedReader br = new BufferedReader(
                new InputStreamReader(response.getEntity().getContent(), this.encode));
        String htmlStr = null;
        int c = 0;
        StringBuilder temp = new StringBuilder();
        while ((c = br.read()) != -1) {
            temp.append((char) c);
        }
        htmlStr = temp.toString();

        httpGet.releaseConnection();
        httpGet.completed();

        return htmlStr;
    } catch (IOException ex) {
        Logger.getLogger(HttpClientAdaptor.class.getName()).log(Level.SEVERE, null, ex);
    }

    return null;
}

---

From source file:org.esupportail.filex.web.WebController.java

@Autowired
public void setRestTemplate(RestTemplate restTemplate) {
    this.restTemplate = restTemplate;
    RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(50 * 1000)
            .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).build();
    HttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build();
    HttpComponentsClientHttpRequestFactory factory = (HttpComponentsClientHttpRequestFactory) restTemplate
            .getRequestFactory();// www.  j a  v a 2  s  . c om
    factory.setHttpClient(httpClient);
}

---

From source file:edu.mit.scratch.ScratchSession.java

public void logout() throws ScratchUserException {
    final RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY)
            .build();/*w  w w.  j  av a 2s  .c  o m*/

    final CookieStore cookieStore = new BasicCookieStore();
    final BasicClientCookie lang = new BasicClientCookie("scratchlanguage", "en");
    final BasicClientCookie sessid = new BasicClientCookie("scratchsessionsid", this.getSessionID());
    final BasicClientCookie token = new BasicClientCookie("scratchcsrftoken", this.getCSRFToken());
    final BasicClientCookie debug = new BasicClientCookie("DEBUG", "true");
    lang.setDomain(".scratch.mit.edu");
    lang.setPath("/");
    sessid.setDomain(".scratch.mit.edu");
    sessid.setPath("/");
    token.setDomain(".scratch.mit.edu");
    token.setPath("/");
    debug.setDomain(".scratch.mit.edu");
    debug.setPath("/");
    cookieStore.addCookie(lang);
    cookieStore.addCookie(sessid);
    cookieStore.addCookie(token);
    cookieStore.addCookie(debug);

    final CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(globalConfig)
            .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64)"
                    + " AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/" + "537.36")
            .setDefaultCookieStore(cookieStore).build();
    CloseableHttpResponse resp;

    final JSONObject loginObj = new JSONObject();
    loginObj.put("csrftoken", this.getCSRFToken());
    try {
        final HttpUriRequest logout = RequestBuilder.post().setUri("https://scratch.mit.edu/accounts/logout/")
                .addHeader("Accept", "application/json, text/javascript, */*; q=0.01")
                .addHeader("Referer", "https://scratch.mit.edu").addHeader("Origin", "https://scratch.mit.edu")
                .addHeader("Accept-Encoding", "gzip, deflate").addHeader("Accept-Language", "en-US,en;q=0.8")
                .addHeader("Content-Type", "application/json").addHeader("X-Requested-With", "XMLHttpRequest")
                .addHeader("X-CSRFToken", this.getCSRFToken()).setEntity(new StringEntity(loginObj.toString()))
                .build();
        resp = httpClient.execute(logout);
    } catch (final Exception e) {
        throw new ScratchUserException();
    }

    this.session_id = null;
    this.token = null;
    this.expires = null;
    this.username = null;
}

---

From source file:nya.miku.wishmaster.http.client.ExtendedHttpClient.java

/**
 *    ? ?    ?  ??/*from w ww  .  j a v a 2 s  . c om*/
 * @param timeout  
 */
public static RequestConfig.Builder getDefaultRequestConfigBuilder(int timeout) {
    return RequestConfig.custom().setConnectTimeout(timeout).setConnectionRequestTimeout(timeout)
            .setSocketTimeout(timeout).setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY)
            .setStaleConnectionCheckEnabled(false);
}

---

From source file:network.HttpClientAdaptor.java

public String doPost(String url, List<NameValuePair> parameters) {

    try {//ww w . ja  v  a2s.c  o m

        HttpPost httpPost = new HttpPost(url);

        RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout)
                .setConnectTimeout(timeout).setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).build();//?????????? 

        httpPost.setConfig(requestConfig);
        httpPost.setEntity(new UrlEncodedFormEntity(parameters));

        CloseableHttpResponse response = this.httpclient.execute(httpPost, localContext);
        BufferedReader br = new BufferedReader(
                new InputStreamReader(response.getEntity().getContent(), this.encode));
        String htmlStr = null;
        int c = 0;
        StringBuilder temp = new StringBuilder();
        while ((c = br.read()) != -1) {
            temp.append((char) c);
        }
        htmlStr = temp.toString();

        httpPost.completed();
        httpPost.releaseConnection();

        return htmlStr;
    } catch (UnsupportedEncodingException ex) {
        Logger.getLogger(HttpClientAdaptor.class.getName()).log(Level.SEVERE, null, ex);
    } catch (IOException ex) {
        Logger.getLogger(HttpClientAdaptor.class.getName()).log(Level.SEVERE, null, ex);
    }

    return null;
}

---

From source file:org.opensextant.xtext.collectors.sharepoint.SharepointClient.java

/**
 * Sharepoint requires NTLM. This client requires a non-null user/passwd/domain.
 * //  www.  jav  a  2  s  .c  o m
 */
@Override
public HttpClient getClient() {

    if (currentConn != null) {
        return currentConn;
    }

    HttpClientBuilder clientHelper = HttpClientBuilder.create();

    if (proxyHost != null) {
        clientHelper.setProxy(proxyHost);
    }

    RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY)
            .build();

    CredentialsProvider creds = new BasicCredentialsProvider();
    creds.setCredentials(AuthScope.ANY, new NTCredentials(user, passwd, server, domain));
    clientHelper.setDefaultCredentialsProvider(creds);
    HttpClient httpClient = clientHelper.setDefaultRequestConfig(globalConfig).build();

    return httpClient;

}

---

From source file:org.ops4j.pax.url.mvn.internal.wagon.ConfigurableHttpWagon.java

@Override
protected CloseableHttpResponse execute(HttpUriRequest httpMethod) throws HttpException, IOException {
    setHeaders(httpMethod);// w  w w .  ja  v a  2 s .  c  o m
    String userAgent = getUserAgent(httpMethod);
    if (userAgent != null) {
        httpMethod.setHeader(HTTP.USER_AGENT, userAgent);
    }

    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    // WAGON-273: default the cookie-policy to browser compatible
    requestConfigBuilder.setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY);

    Repository repo = getRepository();
    ProxyInfo proxyInfo = getProxyInfo(repo.getProtocol(), repo.getHost());
    if (proxyInfo != null) {
        HttpHost proxy = new HttpHost(proxyInfo.getHost(), proxyInfo.getPort());
        requestConfigBuilder.setProxy(proxy);
    }

    HttpMethodConfiguration config = getHttpConfiguration() == null ? null
            : getHttpConfiguration().getMethodConfiguration(httpMethod);

    if (config != null) {
        copyConfig(config, requestConfigBuilder);
    } else {
        requestConfigBuilder.setSocketTimeout(getReadTimeout());
        requestConfigBuilder.setConnectTimeout(getTimeout());
    }

    getLocalContext().setRequestConfig(requestConfigBuilder.build());

    if (config != null && config.isUsePreemptive()) {
        HttpHost targetHost = new HttpHost(repo.getHost(), repo.getPort(), repo.getProtocol());
        AuthScope targetScope = getBasicAuthScope().getScope(targetHost);

        if (getCredentialsProvider().getCredentials(targetScope) != null) {
            BasicScheme targetAuth = new BasicScheme();
            targetAuth.processChallenge(new BasicHeader(AUTH.WWW_AUTH, "BASIC preemptive"));
            getAuthCache().put(targetHost, targetAuth);
        }
    }

    if (proxyInfo != null) {
        if (proxyInfo.getHost() != null) {
            HttpHost proxyHost = new HttpHost(proxyInfo.getHost(), proxyInfo.getPort());
            AuthScope proxyScope = getProxyBasicAuthScope().getScope(proxyHost);

            String proxyUsername = proxyInfo.getUserName();
            String proxyPassword = proxyInfo.getPassword();
            String proxyNtlmHost = proxyInfo.getNtlmHost();
            String proxyNtlmDomain = proxyInfo.getNtlmDomain();

            if (proxyUsername != null && proxyPassword != null) {
                Credentials creds;
                if (proxyNtlmHost != null || proxyNtlmDomain != null) {
                    creds = new NTCredentials(proxyUsername, proxyPassword, proxyNtlmHost, proxyNtlmDomain);
                } else {
                    creds = new UsernamePasswordCredentials(proxyUsername, proxyPassword);
                }

                getCredentialsProvider().setCredentials(proxyScope, creds);
                BasicScheme proxyAuth = new BasicScheme();
                proxyAuth.processChallenge(new BasicHeader(AUTH.PROXY_AUTH, "BASIC preemptive"));
                getAuthCache().put(proxyHost, proxyAuth);
            }
        }
    }

    return client.execute(httpMethod, getLocalContext());
}

---

From source file:com.crawler.app.fetcher.PageFetcher.java

public PageFetcher(CrawlConfig config) {
    super(config);

    RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false)
            .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).setRedirectsEnabled(false)
            //.setRelativeRedirectsAllowed(true)
            .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout())
            .build();// w  ww  . j  a v a2  s .  com

    RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create();
    connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE);
    if (config.isIncludeHttpsPages()) {
        try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174
            // By always trusting the ssl certificate
            SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
                //@Override
                public boolean isTrusted(final X509Certificate[] chain, String authType) {
                    return true;
                }
            }).build();
            SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
                    SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
            connRegistryBuilder.register("https", sslsf);
        } catch (Exception e) {
            logger.warn("Exception thrown while trying to register https");
            logger.debug("Stacktrace", e);
        }
    }

    Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build();
    connectionManager = new PoolingHttpClientConnectionManager(connRegistry);
    connectionManager.setMaxTotal(config.getMaxTotalConnections());
    connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost());

    HttpClientBuilder clientBuilder = HttpClientBuilder.create();
    clientBuilder.setDefaultRequestConfig(requestConfig);
    clientBuilder.setConnectionManager(connectionManager);
    clientBuilder.setUserAgent(config.getUserAgentString());

    if (config.getProxyHost() != null) {
        if (config.getProxyUsername() != null) {
            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()),
                    new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword()));
            clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }

        HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort());
        clientBuilder.setProxy(proxy);
        logger.debug("Working through Proxy: {}", proxy.getHostName());
    }

    httpClient = clientBuilder.build();
    if (config.getAuthInfos() != null && !config.getAuthInfos().isEmpty()) {
        doAuthetication(config.getAuthInfos());
    }

    if (connectionMonitorThread == null) {
        connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager);
    }
    connectionMonitorThread.start();
}

---

From source file:io.github.cidisk.indexcrawler.fetcher.PageFetcher.java

public PageFetcher(CrawlConfig config) {
    super(config);

    RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false)
            .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).setRedirectsEnabled(false)
            .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout())
            .build();/*w w w  . j  a  v a  2s.  c o  m*/

    RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create();
    connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE);
    if (config.isIncludeHttpsPages()) {
        try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174
            // By always trusting the ssl certificate
            SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
                @Override
                public boolean isTrusted(final X509Certificate[] chain, String authType) {
                    return true;
                }
            }).build();
            SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
                    SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
            connRegistryBuilder.register("https", sslsf);
        } catch (Exception e) {
            logger.warn("Exception thrown while trying to register https");
            logger.debug("Stacktrace", e);
        }
    }

    Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build();
    connectionManager = new PoolingHttpClientConnectionManager(connRegistry);
    connectionManager.setMaxTotal(config.getMaxTotalConnections());
    connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost());

    HttpClientBuilder clientBuilder = HttpClientBuilder.create();
    clientBuilder.setDefaultRequestConfig(requestConfig);
    clientBuilder.setConnectionManager(connectionManager);
    clientBuilder.setUserAgent(config.getUserAgentString());

    if (config.getProxyHost() != null) {
        if (config.getProxyUsername() != null) {
            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()),
                    new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword()));
            clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }

        HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort());
        clientBuilder.setProxy(proxy);
        logger.debug("Working through Proxy: {}", proxy.getHostName());
    }

    httpClient = clientBuilder.build();
    if (config.getAuthInfos() != null && !config.getAuthInfos().isEmpty()) {
        doAuthetication(config.getAuthInfos());
    }

    if (connectionMonitorThread == null) {
        connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager);
    }
    connectionMonitorThread.start();
}

---

From source file:de.comlineag.snc.webcrawler.fetcher.PageFetcher.java

public PageFetcher(CrawlConfig config) {
    super(config);

    RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false)
            .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).setRedirectsEnabled(false)
            .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout())
            .build();/*from w ww.j ava  2 s.  c  o m*/

    RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create();
    connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE);
    if (config.isIncludeHttpsPages()) {
        try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174
            // By always trusting the ssl certificate
            SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
                @Override
                public boolean isTrusted(final X509Certificate[] chain, String authType) {
                    return true;
                }
            }).build();
            SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
                    SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
            connRegistryBuilder.register("https", sslsf);
        } catch (Exception e) {
            logger.debug("Exception thrown while trying to register https:", e);
        }
    }

    Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build();
    connectionManager = new PoolingHttpClientConnectionManager(connRegistry);
    connectionManager.setMaxTotal(config.getMaxTotalConnections());
    connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost());

    HttpClientBuilder clientBuilder = HttpClientBuilder.create();
    clientBuilder.setDefaultRequestConfig(requestConfig);
    clientBuilder.setConnectionManager(connectionManager);
    clientBuilder.setUserAgent(config.getUserAgentString());
    if (config.getProxyHost() != null) {

        if (config.getProxyUsername() != null) {
            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()),
                    new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword()));
            clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }

        HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort());
        clientBuilder.setProxy(proxy);
    }
    clientBuilder.addInterceptorLast(new HttpResponseInterceptor() {
        @Override
        public void process(final HttpResponse response, final HttpContext context)
                throws HttpException, IOException {
            HttpEntity entity = response.getEntity();
            Header contentEncoding = entity.getContentEncoding();
            if (contentEncoding != null) {
                HeaderElement[] codecs = contentEncoding.getElements();
                for (HeaderElement codec : codecs) {
                    if (codec.getName().equalsIgnoreCase("gzip")) {
                        response.setEntity(new GzipDecompressingEntity(response.getEntity()));
                        return;
                    }
                }
            }
        }
    });

    httpClient = clientBuilder.build();

    if (connectionMonitorThread == null) {
        connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager);
    }
    connectionMonitorThread.start();
}

---

• «
• 1
• 2
• 3
• »