Example usage for org.apache.commons.httpclient.params HttpMethodParams SINGLE_COOKIE_HEADER

List of usage examples for org.apache.commons.httpclient.params HttpMethodParams SINGLE_COOKIE_HEADER

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.httpclient.params.*
  5. HttpMethodParams
  6. SINGLE_COOKIE_HEADER

Introduction

In this page you can find the example usage for org.apache.commons.httpclient.params HttpMethodParams SINGLE_COOKIE_HEADER.

Prototype

String SINGLE_COOKIE_HEADER

To view the source code for org.apache.commons.httpclient.params HttpMethodParams SINGLE_COOKIE_HEADER.

Click Source Link

Usage

From source file:com.honnix.cheater.network.CheaterHttpClient.java

private void initHttpClient() {
    httpClient = new HttpClient();

    httpClient.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    httpClient.getParams().setParameter("http.useragent", CheaterConstant.USER_AGENT);
    httpClient.getParams().setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, Boolean.TRUE);
}

From source file:com.iflytek.spider.protocol.httpclient.HttpResponse.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 * /*from   w  w  w .  ja  v a 2s  .c om*/
 * @param http
 *            An instance of the implementation class of this plugin
 * @param url
 *            URL to be fetched
 * @param datum
 *            Crawl data
 * @param followRedirects
 *            Whether to follow redirects; follows redirect if and only if
 *            this is true
 * @return HTTP response
 * @throws IOException
 *             When an error occurs
 */
HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
    try {
        code = Http.getClient().executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBase.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead < contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            in.close();
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Log trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace);
        }
    } finally {
        get.releaseConnection();
    }
}

From source file:com.iflytek.spider.protocol.httpclient.HttpResponseSmiply.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 * //from w  w  w  .ja  va2s . c  om
 * @param http
 *            An instance of the implementation class of this plugin
 * @param url
 *            URL to be fetched
 * @param datum
 *            Crawl data
 * @param followRedirects
 *            Whether to follow redirects; follows redirect if and only if
 *            this is true
 * @return HTTP response
 * @throws IOException
 *             When an error occurs
 */
HttpResponseSmiply(HttpSimply http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
    try {
        code = HttpSimply.getClient().executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBaseSimply.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead < contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            in.close();
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Log trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace);
        }
    } finally {
        get.releaseConnection();
    }
}

From source file:com.fatwire.dta.sscrawler.URLReaderService.java

protected HttpClient initClient() {
    final HttpClient client = new HttpClient(connectionManager);
    client.getHostConfiguration().setHost(hostConfig.getHostname(), hostConfig.getPort(),
            hostConfig.getProtocol());//from  w ww .  j  ava  2 s  .c  o  m

    if (hostConfig.getProxyHost() != null) {
        client.getHostConfiguration().setProxyHost(hostConfig.getProxyHost());
        if (hostConfig.getProxyCredentials() != null) {
            client.getState().setProxyCredentials(AuthScope.ANY, hostConfig.getProxyCredentials());
        }
    }

    client.getParams().setParameter(HttpMethodParams.USER_AGENT, "ss-crawler-0.9");

    // RFC 2101 cookie management spec is used per default
    // to parse, validate, format & match cookies
    // client.getParams().setCookiePolicy(CookiePolicy.RFC_2109);
    // client.getParams().setCookiePolicy(CookiePolicy.DEFAULT);
    client.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);

    // client.getParams().makeStrict();
    client.getParams().getDefaults().setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);

    client.getState().addCookie(new Cookie(hostConfig.getHostname(), HelperStrings.SS_CLIENT_INDICATOR,
            Boolean.TRUE.toString(), hostConfig.getDomain(), -1, false));
    return client;
}

From source file:JiraWebSession.java

public void doInSession(JiraWebSessionCallback callback, IProgressMonitor monitor) throws JiraException {
    monitor = Policy.monitorFor(monitor);
    boolean doLogin = hostConfiguration == null || reauthenticate;
    int MAX_RETRIES = doLogin ? 1 : 2; //if login gets skipped, a second try might be needed
    for (int i = 1; i <= MAX_RETRIES; i++) {
        try {/*from  w  ww.j a va  2s  .c  om*/
            lock(monitor);
            if (httpClient == null) {
                HttpConnectionManager connectionManager = WebUtil.getConnectionManager();
                httpClient = new HttpClient(connectionManager);
                WebUtil.configureHttpClient(httpClient, "JiraConnector"); //$NON-NLS-1$
                httpClient.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
                httpClient.getParams().setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, Boolean.TRUE);
            }
            if (doLogin) {
                reauthenticate = false;
                hostConfiguration = login(httpClient, monitor);
            }
        } finally {
            unlock(monitor);
        }
        try {
            // check if session is expired
            if (doLogin || isAuthenticated(httpClient, hostConfiguration, monitor)) {
                callback.configure(httpClient, hostConfiguration, baseUrl,
                        client.getLocalConfiguration().getFollowRedirects());
                callback.run(client, baseUrl, monitor);
                return;
            } else {
                doLogin = true;
            }
        } catch (IOException e) {
            throw new JiraException(e);
        } catch (JiraException e) {
            //if an exception occurs because of a missing login
            if (isAuthenticationFailure(e)) {
                //rinse and repeat with login
                doLogin = true;
            } else {
                throw e;
            }
        }
    }
}

From source file:com.ning.http.client.providers.apache.ApacheAsyncHttpProvider.java

public ApacheAsyncHttpProvider(AsyncHttpClientConfig config) {
    this.config = config;
    connectionManager = new MultiThreadedHttpConnectionManager();

    params = new HttpClientParams();
    params.setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, Boolean.TRUE);
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());

    AsyncHttpProviderConfig<?, ?> providerConfig = config.getAsyncHttpProviderConfig();
    if (providerConfig != null && ApacheAsyncHttpProvider.class.isAssignableFrom(providerConfig.getClass())) {
        configure(ApacheAsyncHttpProviderConfig.class.cast(providerConfig));
    }//from   w ww  .  ja  v a 2s. c om
}

From source file:com.ning.http.client.providers.apache.TestableApacheAsyncHttpProvider.java

public TestableApacheAsyncHttpProvider(AsyncHttpClientConfig config) {
    super(config);
    this.config = config;
    connectionManager = new MultiThreadedHttpConnectionManager();

    params = new HttpClientParams();
    params.setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, Boolean.TRUE);
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());

}

From source file:com.celamanzi.liferay.portlets.rails286.OnlineClient.java

/** 
 * Prepares client.//from w w  w .j  a  va  2 s  .  co m
 */
protected HttpClient preparedClient() {
    // Create an instance of HttpClient and prepare it
    HttpClient client = new HttpClient();

    client.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    client.getParams().setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // magic cookie line (all cookies on one header line)
    client.getParams().setParameter("http.protocol.single-cookie-header", true);

    // Set state (session cookies)
    client.setState(preparedHttpState());
    // Set timeout
    client.getHttpConnectionManager().getParams().setConnectionTimeout(timeout);

    return client;
}

From source file:com.cyberway.issue.crawler.fetcher.FetchHTTP.java

protected void configureHttp() throws RuntimeException {
    // Get timeout.  Use it for socket and for connection timeout.
    int timeout = (getSoTimeout(null) > 0) ? getSoTimeout(null) : 0;

    // HttpConnectionManager cm = new ThreadLocalHttpConnectionManager();
    HttpConnectionManager cm = new SingleHttpConnectionManager();

    // TODO: The following settings should be made in the corresponding
    // HttpConnectionManager, not here.
    HttpConnectionManagerParams hcmp = cm.getParams();
    hcmp.setConnectionTimeout(timeout);//from  ww w .  ja  v  a2s. co  m
    hcmp.setStaleCheckingEnabled(true);
    // Minimizes bandwidth usage.  Setting to true disables Nagle's
    // algorithm.  IBM JVMs < 142 give an NPE setting this boolean
    // on ssl sockets.
    hcmp.setTcpNoDelay(false);

    this.http = new HttpClient(cm);
    HttpClientParams hcp = this.http.getParams();
    // Set default socket timeout.
    hcp.setSoTimeout(timeout);
    // Set client to be version 1.0.
    hcp.setVersion(HttpVersion.HTTP_1_0);

    configureHttpCookies();

    // Configure how we want the method to act.
    this.http.getParams().setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, new Boolean(true));
    this.http.getParams().setParameter(HttpMethodParams.UNAMBIGUOUS_STATUS_LINE, new Boolean(false));
    this.http.getParams().setParameter(HttpMethodParams.STRICT_TRANSFER_ENCODING, new Boolean(false));
    this.http.getParams().setIntParameter(HttpMethodParams.STATUS_LINE_GARBAGE_LIMIT, 10);

    // modify the default config with any global settings
    HostConfiguration config = this.http.getHostConfiguration();
    configureProxy(null, config);
    configureBindAddress(null, config);

    // Use our own protocol factory, one that gets IP to use from
    // heritrix cache (They're cached in CrawlHost instances).
    final ServerCache cache = getController().getServerCache();
    hcmp.setParameter(SERVER_CACHE_KEY, cache);
    hcmp.setParameter(SSL_FACTORY_KEY, this.sslfactory);
}

From source file:org.apache.nutch.protocol.httpclient.HttpResponse.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 * /*from   w  w w. ja  va2s  .co  m*/
 * @param http
 *          An instance of the implementation class of this plugin
 * @param url
 *          URL to be fetched
 * @param datum
 *          Crawl data
 * @param followRedirects
 *          Whether to follow redirects; follows redirect if and only if this
 *          is true
 * @return HTTP response
 * @throws IOException
 *           When an error occurs
 */
HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");

    if (http.isCookieEnabled()) {
        params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
        params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    } else {
        params.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
    }
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);

    if (http.isCookieEnabled() && datum.getMetaData().containsKey(http.COOKIE)) {
        String cookie = ((Text) datum.getMetaData().get(http.COOKIE)).toString();
        get.addRequestHeader("Cookie", cookie);
    }

    try {
        HttpClient client = Http.getClient();
        client.getParams().setParameter("http.useragent", http.getUserAgent()); // NUTCH-1941
        code = client.executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBase.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1
                    && totalRead + bufferFilled <= contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            if (in != null) {
                in.close();
            }
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Logger trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace.toString());
        }
    } finally {
        get.releaseConnection();
    }
}