Example usage for org.apache.commons.httpclient.params HttpMethodParams SINGLE_COOKIE

Introduction

In this page you can find the example usage for org.apache.commons.httpclient.params HttpMethodParams SINGLE_COOKIE_HEADER.

Prototype

String SINGLE_COOKIE_HEADER

To view the source code for org.apache.commons.httpclient.params HttpMethodParams SINGLE_COOKIE_HEADER.

Click Source Link

Usage

From source file:org.apache.nutch.protocol.httpclient.HttpResponseBak.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 *
 * @param http                An instance of the implementation class
 *                            of this plugin
 * @param url                 URL to be fetched
 * @param datum               Crawl data
 * @param followRedirects     Whether to follow redirects; follows
 *                            redirect if and only if this is true
 * @return                    HTTP response
 * @throws IOException        When an error occurs
 *//*from  w w w .j  ava 2s  .c  om*/
HttpResponseBak(HttpBak http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
    try {
        code = Http.getClient().executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBase.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1
                    && totalRead + bufferFilled <= contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            if (in != null) {
                in.close();
            }
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Logger trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace.toString());
        }
    } finally {
        get.releaseConnection();
    }
}

From source file:org.apache.nutch.protocol.httpclient.proxy.HttpResponse.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 * /* ww w.  j a  v  a 2  s . c o m*/
 * @param http
 *          An instance of the implementation class of this plugin
 * @param url
 *          URL to be fetched
 * @param datum
 *          Crawl data
 * @param followRedirects
 *          Whether to follow redirects; follows redirect if and only if this
 *          is true
 * @return HTTP response
 * @throws IOException
 *           When an error occurs
 */
HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
    try {
        HttpClient client = Http.getClient();
        client.getParams().setParameter("http.useragent", http.getUserAgent()); // NUTCH-1941
        code = client.executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBase.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1
                    && totalRead + bufferFilled <= contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            if (in != null) {
                in.close();
            }
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Logger trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace.toString());
        }
    } finally {
        get.releaseConnection();
    }
}

From source file:org.archive.crawler.fetcher.OptimizeFetchHTTP.java

protected HttpClient configureHttp() throws RuntimeException {
    // Get timeout.  Use it for socket and for connection timeout.
    int timeout = (getSoTimeout(null) > 0) ? getSoTimeout(null) : 0;

    // HttpConnectionManager cm = new ThreadLocalHttpConnectionManager();
    HttpConnectionManager cm = new SingleHttpConnectionManager();

    // TODO: The following settings should be made in the corresponding
    // HttpConnectionManager, not here.
    HttpConnectionManagerParams hcmp = cm.getParams();
    hcmp.setConnectionTimeout(timeout);/*from   ww  w .  j  a  va  2  s.c  o m*/
    hcmp.setStaleCheckingEnabled(true);
    // Minimizes bandwidth usage.  Setting to true disables Nagle's
    // algorithm.  IBM JVMs < 142 give an NPE setting this boolean
    // on ssl sockets.
    hcmp.setTcpNoDelay(false);

    HttpClient http = new HttpClient(cm);
    HttpClientParams hcp = http.getParams();
    // Set default socket timeout.
    hcp.setSoTimeout(timeout);
    // Set client to be version 1.0.
    hcp.setVersion(HttpVersion.HTTP_1_0);

    configureHttpCookies(http);

    // Configure how we want the method to act.
    http.getParams().setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, new Boolean(true));
    http.getParams().setParameter(HttpMethodParams.UNAMBIGUOUS_STATUS_LINE, new Boolean(false));
    http.getParams().setParameter(HttpMethodParams.STRICT_TRANSFER_ENCODING, new Boolean(false));
    http.getParams().setIntParameter(HttpMethodParams.STATUS_LINE_GARBAGE_LIMIT, 10);

    // modify the default config with any global settings
    HostConfiguration config = http.getHostConfiguration();
    configureProxy(null, config);
    configureBindAddress(null, config);

    // Use our own protocol factory, one that gets IP to use from
    // heritrix cache (They're cached in CrawlHost instances).
    final ServerCache cache = getController().getServerCache();
    hcmp.setParameter(SERVER_CACHE_KEY, cache);
    hcmp.setParameter(SSL_FACTORY_KEY, this.sslfactory);

    return http;
}

From source file:org.eclipse.smila.connectivity.framework.crawler.web.http.HttpResponse.java

/**
 * Sets the http parameters./*from w  w w .  ja  v  a  2s . c  om*/
 * 
 * @param http
 *          the http
 * @param httpMethod
 *          the http method
 */
private void setHttpParameters(HttpBase http, HttpMethodBase httpMethod) {
    httpMethod.setFollowRedirects(false);
    httpMethod.setRequestHeader("User-Agent", http.getUserAgent());
    httpMethod.setRequestHeader("Referer", http.getReferer());

    httpMethod.setDoAuthentication(true);

    for (Header header : http.getHeaders()) {
        httpMethod.addRequestHeader(header);
    }

    final HttpMethodParams params = httpMethod.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");

    if (http.isCookiesEnabled()) {
        params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    } else {
        params.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
    }
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // the default is to retry 3 times; if
    // the request body was sent the method is not retried, so there is
    // little danger in retrying
    // retries are handled on the higher level
    params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
}

From source file:org.lockss.plugin.silverchair.PostHttpClientUrlConnection.java

/** Called by org.lockss.config.MiscConfig
 *//*  w  w w  .  ja v  a 2s  . c  om*/
public static void setConfig(Configuration config, Configuration oldConfig, Configuration.Differences diffs) {
    if (diffs.contains(PREFIX)) {
        acceptHeader = config.get(PARAM_ACCEPT_HEADER, DEFAULT_ACCEPT_HEADER);

        Set<String> set = new HashSet();
        for (String s : (List<String>) config.getList(PARAM_SINGLE_VALUED_HEADERS,
                DEFAULT_SINGLE_VALUED_HEADERS)) {
            set.add(s.toLowerCase());
        }
        singleValuedHdrs = set;

        HttpParams params = DefaultHttpParams.getDefaultParams();

        if (diffs.contains(PARAM_COOKIE_POLICY)) {
            String policy = config.get(PARAM_COOKIE_POLICY, DEFAULT_COOKIE_POLICY);
            params.setParameter(HttpMethodParams.COOKIE_POLICY, getCookiePolicy(policy));
        }
        if (diffs.contains(PARAM_SINGLE_COOKIE_HEADER)) {
            boolean val = config.getBoolean(PARAM_SINGLE_COOKIE_HEADER, DEFAULT_SINGLE_COOKIE_HEADER);
            params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, val);
        }
        ServerTrustLevel stl = (ServerTrustLevel) config.getEnum(ServerTrustLevel.class,
                PARAM_SERVER_TRUST_LEVEL, DEFAULT_SERVER_TRUST_LEVEL);
        DISP_FACT.setDefaultFactory(getDefaultSocketFactory(stl));
    }
}

From source file:org.lockss.util.urlconn.HttpClientUrlConnection.java

/** Called by org.lockss.config.MiscConfig
 *///ww w .ja  v  a 2s .  com
public static void setConfig(Configuration config, Configuration oldConfig, Configuration.Differences diffs) {
    if (diffs.contains(PREFIX)) {
        acceptHeader = config.get(PARAM_ACCEPT_HEADER, DEFAULT_ACCEPT_HEADER);

        HttpParams params = DefaultHttpParams.getDefaultParams();
        if (diffs.contains(PARAM_COOKIE_POLICY)) {
            String policy = config.get(PARAM_COOKIE_POLICY, DEFAULT_COOKIE_POLICY);
            params.setParameter(HttpMethodParams.COOKIE_POLICY, getCookiePolicy(policy));
        }
        if (diffs.contains(PARAM_SINGLE_COOKIE_HEADER)) {
            boolean val = config.getBoolean(PARAM_SINGLE_COOKIE_HEADER, DEFAULT_SINGLE_COOKIE_HEADER);
            params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, val);
        }
        ServerTrustLevel stl = (ServerTrustLevel) config.getEnum(ServerTrustLevel.class,
                PARAM_SERVER_TRUST_LEVEL, DEFAULT_SERVER_TRUST_LEVEL);
        DISP_FACT.setDefaultFactory(getDefaultSocketFactory(stl));
    }
}

From source file:org.parosproxy.paros.network.HttpSender.java

/**
 * Constructs an {@code HttpSender}./*from ww  w .  ja va 2s . co  m*/
 * <p>
 * If {@code useGlobalState} is {@code true} the HttpSender will use the HTTP state given by
 * {@code ConnectionParam#getHttpState()} iff {@code ConnectionParam#isHttpStateEnabled()} returns {@code true} otherwise it
 * doesn't have any state (i.e. cookies are disabled). If {@code useGlobalState} is {@code false} it uses a non shared HTTP
 * state. The actual state used is overridden, per message, when {@code HttpMessage#getRequestingUser()} returns non
 * {@code null}.
 * <p>
 * The {@code initiator} is used to indicate the component that is sending the messages when the {@code HttpSenderListener}s
 * are notified of messages sent and received.
 *
 * @param connectionParam the parameters used to setup the connections to target hosts
 * @param useGlobalState {@code true} if the messages sent/received should use the global HTTP state, {@code false} if
 *         should use a non shared HTTP state
 * @param initiator the ID of the initiator of the HTTP messages sent
 * @see ConnectionParam#getHttpState()
 * @see HttpSenderListener
 * @see HttpMessage#getRequestingUser()
 */
public HttpSender(ConnectionParam connectionParam, boolean useGlobalState, int initiator) {
    this.param = connectionParam;
    this.initiator = initiator;

    client = createHttpClient();
    clientViaProxy = createHttpClientViaProxy();

    // Set how cookie headers are sent no matter of the "allowState", in case a state is forced by
    // other extensions (e.g. Authentication)
    final boolean singleCookieRequestHeader = param.isSingleCookieRequestHeader();
    client.getParams().setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, singleCookieRequestHeader);
    clientViaProxy.getParams().setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER,
            singleCookieRequestHeader);

    if (useGlobalState) {
        checkState();
    }
}

From source file:org.sakaiproject.entitybroker.util.http.HttpRESTUtils.java

/**
 * Generates a reusable http client wrapper which can be given to {@link #fireRequest(HttpClientWrapper, String, Method, Map, Object, boolean)}
 * as an efficiency mechanism/*from  w  ww  .ja v a 2  s .  c om*/
 * 
 * @param multiThreaded true if you want to allow the client to run in multiple threads
 * @param idleConnectionTimeout if this is 0 then it will use the defaults, otherwise connections will be timed out after this long (ms)
 * @param cookies to send along with every request from this client
 * @return the reusable http client wrapper
 */
public static HttpClientWrapper makeReusableHttpClient(boolean multiThreaded, int idleConnectionTimeout,
        Cookie[] cookies) {
    HttpClientWrapper wrapper;
    HttpClient client;
    MultiThreadedHttpConnectionManager connectionManager = null;
    if (multiThreaded) {
        connectionManager = new MultiThreadedHttpConnectionManager();
        client = new HttpClient(connectionManager);
    } else {
        client = new HttpClient();
    }
    if (idleConnectionTimeout <= 0) {
        idleConnectionTimeout = 5000;
    }
    client.getHttpConnectionManager().closeIdleConnections(idleConnectionTimeout);
    client.getHttpConnectionManager().getParams().setConnectionTimeout(idleConnectionTimeout);
    // create the initial state
    HttpState initialState = new HttpState();
    if (cookies != null && cookies.length > 0) {
        for (int i = 0; i < cookies.length; i++) {
            Cookie c = cookies[i];
            org.apache.commons.httpclient.Cookie mycookie = new org.apache.commons.httpclient.Cookie(
                    c.getDomain(), c.getName(), c.getValue(), c.getPath(), c.getMaxAge(), c.getSecure());
            initialState.addCookie(mycookie);
        }
        client.setState(initialState);
    }
    // set some defaults
    client.getParams().setParameter(HttpMethodParams.USER_AGENT,
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.1) Gecko/2008070208 Firefox/3.0.1");
    client.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    client.getParams().setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    wrapper = new HttpClientWrapper(client, connectionManager, initialState);
    return wrapper;
}

Example usage for org.apache.commons.httpclient.params HttpMethodParams SINGLE_COOKIE_HEADER

Introduction

Prototype

Usage