Example usage for org.apache.commons.httpclient.params HttpMethodParams setBooleanParameter

Introduction

In this page you can find the example usage for org.apache.commons.httpclient.params HttpMethodParams setBooleanParameter.

Prototype

public void setBooleanParameter(String paramString, boolean paramBoolean) 

Source Link

Usage

From source file:com.vmware.aurora.vc.VcFileManager.java

static private void uploadFileWork(String url, boolean isPost, File file, String contentType, String cookie,
        ProgressListener listener) throws Exception {
    EntityEnclosingMethod method;/*  w w w .j  a va2  s  .  c o  m*/
    final RequestEntity entity = new ProgressListenerRequestEntity(file, contentType, listener);
    if (isPost) {
        method = new PostMethod(url);
        method.setContentChunked(true);
    } else {
        method = new PutMethod(url);
        method.addRequestHeader("Cookie", cookie);
        method.setContentChunked(false);
        HttpMethodParams params = new HttpMethodParams();
        params.setBooleanParameter(HttpMethodParams.USE_EXPECT_CONTINUE, true);
        method.setParams(params);
    }
    method.setRequestEntity(entity);

    logger.info("upload " + file + " to " + url);
    long t1 = System.currentTimeMillis();
    boolean ok = false;
    try {
        HttpClient httpClient = new HttpClient();
        int statusCode = httpClient.executeMethod(method);
        String response = method.getResponseBodyAsString(100);
        logger.debug("status: " + statusCode + " response: " + response);
        if (statusCode != HttpStatus.SC_CREATED && statusCode != HttpStatus.SC_OK) {
            throw new Exception("Http post failed");
        }
        method.releaseConnection();
        ok = true;
    } finally {
        if (!ok) {
            method.abort();
        }
    }
    long t2 = System.currentTimeMillis();
    logger.info("upload " + file + " done in " + (t2 - t1) + " ms");
}

---

From source file:com.iflytek.spider.protocol.httpclient.HttpResponseSmiply.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 * /*  ww  w  . j  a v a2 s.  com*/
 * @param http
 *            An instance of the implementation class of this plugin
 * @param url
 *            URL to be fetched
 * @param datum
 *            Crawl data
 * @param followRedirects
 *            Whether to follow redirects; follows redirect if and only if
 *            this is true
 * @return HTTP response
 * @throws IOException
 *             When an error occurs
 */
HttpResponseSmiply(HttpSimply http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
    try {
        code = HttpSimply.getClient().executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBaseSimply.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead < contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            in.close();
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Log trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace);
        }
    } finally {
        get.releaseConnection();
    }
}

---

From source file:com.iflytek.spider.protocol.httpclient.HttpResponse.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 * /* ww  w.  j  av a  2 s . c o m*/
 * @param http
 *            An instance of the implementation class of this plugin
 * @param url
 *            URL to be fetched
 * @param datum
 *            Crawl data
 * @param followRedirects
 *            Whether to follow redirects; follows redirect if and only if
 *            this is true
 * @return HTTP response
 * @throws IOException
 *             When an error occurs
 */
HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
    try {
        code = Http.getClient().executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBase.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead < contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            in.close();
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Log trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace);
        }
    } finally {
        get.releaseConnection();
    }
}

---

From source file:com.twinsoft.convertigo.beans.connectors.SiteClipperConnector.java

private void doProcessRequest(Shuttle shuttle) throws IOException, ServletException, EngineException {
    shuttle.statisticsTaskID = context.statistics.start(EngineStatistics.GET_DOCUMENT);
    try {//w w  w  .  j av a 2s  . co  m
        shuttle.sharedScope = context.getSharedScope();

        String domain = shuttle.getRequest(QueryPart.host) + shuttle.getRequest(QueryPart.port);
        Engine.logSiteClipper.trace("(SiteClipperConnector) Prepare the request for the domain " + domain);
        if (!shouldRewrite(domain)) {
            Engine.logSiteClipper.info(
                    "(SiteClipperConnector) The domain " + domain + " is not allowed with this connector");
            shuttle.response.sendError(HttpServletResponse.SC_FORBIDDEN,
                    "The domain " + domain + " is not allowed with this connector");
            return;
        }

        String uri = shuttle.getRequest(QueryPart.uri);

        Engine.logSiteClipper.info("Preparing " + shuttle.request.getMethod() + " " + shuttle.getRequestUrl());

        HttpMethod httpMethod = null;
        XulRecorder xulRecorder = context.getXulRecorder();
        if (xulRecorder != null) {
            httpMethod = shuttle.httpMethod = xulRecorder.getRecord(shuttle.getRequestUrlAndQuery());
        }
        if (httpMethod == null) {
            try {
                switch (shuttle.getRequestHttpMethodType()) {
                case GET:
                    httpMethod = new GetMethod(uri);
                    break;
                case POST:
                    httpMethod = new PostMethod(uri);
                    ((PostMethod) httpMethod)
                            .setRequestEntity(new InputStreamRequestEntity(shuttle.request.getInputStream()));
                    break;
                case PUT:
                    httpMethod = new PutMethod(uri);
                    ((PutMethod) httpMethod)
                            .setRequestEntity(new InputStreamRequestEntity(shuttle.request.getInputStream()));
                    break;
                case DELETE:
                    httpMethod = new DeleteMethod(uri);
                    break;
                case HEAD:
                    httpMethod = new HeadMethod(uri);
                    break;
                case OPTIONS:
                    httpMethod = new OptionsMethod(uri);
                    break;
                case TRACE:
                    httpMethod = new TraceMethod(uri);
                    break;
                default:
                    throw new ServletException(
                            "(SiteClipperConnector) unknown http method " + shuttle.request.getMethod());
                }
                httpMethod.setFollowRedirects(false);
            } catch (Exception e) {
                throw new ServletException(
                        "(SiteClipperConnector) unexpected exception will building the http method : "
                                + e.getMessage());
            }
            shuttle.httpMethod = httpMethod;

            SiteClipperScreenClass screenClass = getCurrentScreenClass();
            Engine.logSiteClipper.info("Request screen class: " + screenClass.getName());

            for (String name : Collections
                    .list(GenericUtils.<Enumeration<String>>cast(shuttle.request.getHeaderNames()))) {
                if (requestHeadersToIgnore.contains(HeaderName.parse(name))) {
                    Engine.logSiteClipper.trace("(SiteClipperConnector) Ignoring request header " + name);
                } else {
                    String value = shuttle.request.getHeader(name);
                    Engine.logSiteClipper
                            .trace("(SiteClipperConnector) Copying request header " + name + "=" + value);
                    shuttle.setRequestCustomHeader(name, value);
                }
            }

            Engine.logSiteClipper.debug("(SiteClipperConnector) applying request rules for the screenclass "
                    + screenClass.getName());
            for (IRequestRule rule : screenClass.getRequestRules()) {
                if (rule.isEnabled()) {
                    Engine.logSiteClipper
                            .trace("(SiteClipperConnector) applying request rule " + rule.getName());
                    rule.fireEvents();
                    boolean done = rule.applyOnRequest(shuttle);
                    Engine.logSiteClipper.debug("(SiteClipperConnector) the request rule " + rule.getName()
                            + " is " + (done ? "well" : "not") + " applied");
                } else {
                    Engine.logSiteClipper
                            .trace("(SiteClipperConnector) skip the disabled request rule " + rule.getName());
                }
            }

            for (Entry<String, String> header : shuttle.requestCustomHeaders.entrySet()) {
                Engine.logSiteClipper.trace("(SiteClipperConnector) Push request header " + header.getKey()
                        + "=" + header.getValue());
                httpMethod.addRequestHeader(header.getKey(), header.getValue());
            }

            String queryString = shuttle.request.getQueryString();

            if (queryString != null) {
                try {
                    // Fake test in order to check query string validity
                    new URI("http://localhost/index?" + queryString, true,
                            httpMethod.getParams().getUriCharset());
                } catch (URIException e) {
                    // Bugfix #2103
                    StringBuffer newQuery = new StringBuffer();
                    for (String part : RegexpUtils.pattern_and.split(queryString)) {
                        String[] pair = RegexpUtils.pattern_equals.split(part, 2);
                        try {
                            newQuery.append('&')
                                    .append(URLEncoder.encode(URLDecoder.decode(pair[0], "UTF-8"), "UTF-8"));
                            if (pair.length > 1) {
                                newQuery.append('=').append(
                                        URLEncoder.encode(URLDecoder.decode(pair[1], "UTF-8"), "UTF-8"));
                            }
                        } catch (UnsupportedEncodingException ee) {
                            Engine.logSiteClipper
                                    .trace("(SiteClipperConnector) failed to encode query part : " + part);
                        }
                    }

                    queryString = newQuery.length() > 0 ? newQuery.substring(1) : newQuery.toString();
                    Engine.logSiteClipper.trace("(SiteClipperConnector) re-encode query : " + queryString);
                }
            }

            Engine.logSiteClipper.debug("(SiteClipperConnector) Copying the query string : " + queryString);
            httpMethod.setQueryString(queryString);

            //            if (context.httpState == null) {
            //               Engine.logSiteClipper.debug("(SiteClipperConnector) Creating new HttpState for context id " + context.contextID);
            //               context.httpState = new HttpState();
            //            } else {
            //               Engine.logSiteClipper.debug("(SiteClipperConnector) Using HttpState of context id " + context.contextID);
            //            }

            getHttpState(shuttle);

            HostConfiguration hostConfiguration = getHostConfiguration(shuttle);

            HttpMethodParams httpMethodParams = httpMethod.getParams();
            httpMethodParams.setBooleanParameter("http.connection.stalecheck", true);
            httpMethodParams.setParameter(HttpMethodParams.RETRY_HANDLER,
                    new DefaultHttpMethodRetryHandler(3, true));

            Engine.logSiteClipper.info("Requesting " + httpMethod.getName() + " "
                    + hostConfiguration.getHostURL() + httpMethod.getURI().toString());

            HttpClient httpClient = context.getHttpClient3(shuttle.getHttpPool());
            HttpUtils.logCurrentHttpConnection(httpClient, hostConfiguration, shuttle.getHttpPool());
            httpClient.executeMethod(hostConfiguration, httpMethod, context.httpState);
        } else {
            Engine.logSiteClipper.info("Retrieve recorded response from Context");
        }

        int status = httpMethod.getStatusCode();

        shuttle.processState = ProcessState.response;

        Engine.logSiteClipper.info("Request terminated with status " + status);
        shuttle.response.setStatus(status);

        if (Engine.isStudioMode() && status == HttpServletResponse.SC_OK
                && shuttle.getResponseMimeType().startsWith("text/")) {
            fireDataChanged(new ConnectorEvent(this, shuttle.getResponseAsString()));
        }

        SiteClipperScreenClass screenClass = getCurrentScreenClass();
        Engine.logSiteClipper.info("Response screen class: " + screenClass.getName());

        if (Engine.isStudioMode()) {
            Engine.theApp.fireObjectDetected(new EngineEvent(screenClass));
        }

        for (Header header : httpMethod.getResponseHeaders()) {
            String name = header.getName();
            if (responseHeadersToIgnore.contains(HeaderName.parse(name))) {
                Engine.logSiteClipper.trace("(SiteClipperConnector) Ignoring response header " + name);
            } else {
                String value = header.getValue();
                Engine.logSiteClipper
                        .trace("(SiteClipperConnector) Copying response header " + name + "=" + value);
                shuttle.responseCustomHeaders.put(name, value);
            }
        }

        Engine.logSiteClipper.debug(
                "(SiteClipperConnector) applying response rules for the screenclass " + screenClass.getName());
        for (IResponseRule rule : screenClass.getResponseRules()) {
            if (rule.isEnabled()) {
                Engine.logSiteClipper.trace("(SiteClipperConnector) applying response rule " + rule.getName());
                rule.fireEvents();
                boolean done = rule.applyOnResponse(shuttle);
                Engine.logSiteClipper.debug("(SiteClipperConnector) the response rule " + rule.getName()
                        + " is " + (done ? "well" : "not") + " applied");
            } else {
                Engine.logSiteClipper
                        .trace("(SiteClipperConnector) skip the disabled response rule " + rule.getName());
            }
        }

        for (Entry<String, String> header : shuttle.responseCustomHeaders.entrySet()) {
            Engine.logSiteClipper.trace(
                    "(SiteClipperConnector) Push request header " + header.getKey() + "=" + header.getValue());
            shuttle.response.addHeader(header.getKey(), header.getValue());
        }

        if (shuttle.postInstructions != null) {
            JSONArray instructions = new JSONArray();
            for (IClientInstruction instruction : shuttle.postInstructions) {
                try {
                    instructions.put(instruction.getInstruction());
                } catch (JSONException e) {
                    Engine.logSiteClipper.error(
                            "(SiteClipperConnector) Failed to add a post instruction due to a JSONException",
                            e);
                }
            }
            String codeToInject = "<script>C8O_postInstructions = " + instructions.toString() + "</script>\n"
                    + "<script src=\"" + shuttle.getRequest(QueryPart.full_convertigo_path)
                    + "/scripts/jquery.min.js\"></script>\n" + "<script src=\""
                    + shuttle.getRequest(QueryPart.full_convertigo_path)
                    + "/scripts/siteclipper.js\"></script>\n";

            String content = shuttle.getResponseAsString();
            Matcher matcher = HtmlLocation.head_top.matcher(content);
            String newContent = RegexpUtils.inject(matcher, codeToInject);
            if (newContent == null) {
                matcher = HtmlLocation.body_top.matcher(content);
                newContent = RegexpUtils.inject(matcher, codeToInject);
            }
            if (newContent != null) {
                shuttle.setResponseAsString(newContent);
            } else {
                Engine.logSiteClipper.info(
                        "(SiteClipperConnector) Failed to find a head or body tag in the response content");
                Engine.logSiteClipper.trace("(SiteClipperConnector) Response content : \"" + content + "\"");
            }
        }

        long nbBytes = 0L;
        if (shuttle.responseAsString != null
                && shuttle.responseAsString.hashCode() != shuttle.responseAsStringOriginal.hashCode()) {
            OutputStream os = shuttle.response.getOutputStream();
            switch (shuttle.getResponseContentEncoding()) {
            case gzip:
                os = new GZIPOutputStream(os);
                break;
            case deflate:
                os = new DeflaterOutputStream(os,
                        new Deflater(Deflater.DEFAULT_COMPRESSION | Deflater.DEFAULT_STRATEGY, true));
                break;
            default:
                break;
            }
            nbBytes = shuttle.responseAsByte.length;
            IOUtils.write(shuttle.responseAsString, os, shuttle.getResponseCharset());
            os.close();
        } else {
            InputStream is = (shuttle.responseAsByte == null) ? httpMethod.getResponseBodyAsStream()
                    : new ByteArrayInputStream(shuttle.responseAsByte);
            if (is != null) {
                nbBytes = 0;
                OutputStream os = shuttle.response.getOutputStream();
                int read = is.read();
                while (read >= 0) {
                    os.write(read);
                    os.flush();
                    read = is.read();
                    nbBytes++;
                }
                is.close();
                //               nbBytes = IOUtils.copyLarge(is, shuttle.response.getOutputStream());
                Engine.logSiteClipper
                        .trace("(SiteClipperConnector) Response body copyied (" + nbBytes + " bytes)");
            }
        }
        shuttle.response.getOutputStream().close();

        shuttle.score = getScore(nbBytes);
        Engine.logSiteClipper
                .debug("(SiteClipperConnector) Request terminated with a score of " + shuttle.score);
    } finally {
        long duration = context.statistics.stop(shuttle.statisticsTaskID);
        if (context.requestedObject != null) {
            try {
                Engine.theApp.billingManager.insertBilling(context, Long.valueOf(duration),
                        Long.valueOf(shuttle.score));
            } catch (Exception e) {
                Engine.logContext.warn("Unable to insert billing ticket (the billing is thus ignored): ["
                        + e.getClass().getName() + "] " + e.getMessage());
            }
        }
    }
}

---

From source file:org.apache.maven.wagon.providers.webdav.HttpMethodConfiguration.java

private void fillParams(HttpMethodParams p) {
    if (!hasParams()) {
        return;/*from  w w  w  .  ja va 2s .  c  o  m*/
    }

    if (connectionTimeout > 0) {
        p.setSoTimeout(connectionTimeout);
    }

    if (params != null) {
        Pattern coercePattern = Pattern.compile(COERCE_PATTERN);

        for (Map.Entry<Object, Object> entry : params.entrySet()) {
            String key = (String) entry.getKey();
            String value = (String) entry.getValue();

            Matcher matcher = coercePattern.matcher(value);
            if (matcher.matches()) {
                char type = matcher.group(1).charAt(0);
                value = matcher.group(2);

                switch (type) {
                case 'i': {
                    p.setIntParameter(key, Integer.parseInt(value));
                    break;
                }
                case 'd': {
                    p.setDoubleParameter(key, Double.parseDouble(value));
                    break;
                }
                case 'l': {
                    p.setLongParameter(key, Long.parseLong(value));
                    break;
                }
                case 'b': {
                    p.setBooleanParameter(key, Boolean.valueOf(value).booleanValue());
                    break;
                }
                case 'c': {
                    String[] entries = value.split(",");
                    List<String> collection = new ArrayList<String>();
                    for (String e : entries) {
                        collection.add(e.trim());
                    }

                    p.setParameter(key, collection);
                    break;
                }
                case 'm': {
                    String[] entries = value.split(",");

                    Map<String, String> map = new LinkedHashMap<String, String>();
                    for (String e : entries) {
                        int idx = e.indexOf("=>");
                        if (idx < 1) {
                            break;
                        }

                        String mapKey = e.substring(0, idx);
                        String mapVal = e.substring(idx + 1, e.length());
                        map.put(mapKey.trim(), mapVal.trim());
                    }

                    p.setParameter(key, map);
                    break;
                }
                }
            } else {
                p.setParameter(key, value);
            }
        }
    }
}

---

From source file:org.apache.nutch.protocol.httpclient.HttpResponse.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 * //from   w w w.  j  a  v  a2s. c o m
 * @param http
 *          An instance of the implementation class of this plugin
 * @param url
 *          URL to be fetched
 * @param datum
 *          Crawl data
 * @param followRedirects
 *          Whether to follow redirects; follows redirect if and only if this
 *          is true
 * @return HTTP response
 * @throws IOException
 *           When an error occurs
 */
HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");

    if (http.isCookieEnabled()) {
        params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
        params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    } else {
        params.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
    }
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);

    if (http.isCookieEnabled() && datum.getMetaData().containsKey(http.COOKIE)) {
        String cookie = ((Text) datum.getMetaData().get(http.COOKIE)).toString();
        get.addRequestHeader("Cookie", cookie);
    }

    try {
        HttpClient client = Http.getClient();
        client.getParams().setParameter("http.useragent", http.getUserAgent()); // NUTCH-1941
        code = client.executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBase.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1
                    && totalRead + bufferFilled <= contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            if (in != null) {
                in.close();
            }
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Logger trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace.toString());
        }
    } finally {
        get.releaseConnection();
    }
}

---

From source file:org.apache.nutch.protocol.httpclient.HttpResponseBak.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 *
 * @param http                An instance of the implementation class
 *                            of this plugin
 * @param url                 URL to be fetched
 * @param datum               Crawl data
 * @param followRedirects     Whether to follow redirects; follows
 *                            redirect if and only if this is true
 * @return                    HTTP response
 * @throws IOException        When an error occurs
 *//*w  w  w .  j a v a  2 s .  c  o m*/
HttpResponseBak(HttpBak http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
    try {
        code = Http.getClient().executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBase.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1
                    && totalRead + bufferFilled <= contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            if (in != null) {
                in.close();
            }
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Logger trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace.toString());
        }
    } finally {
        get.releaseConnection();
    }
}

---

From source file:org.apache.nutch.protocol.httpclient.proxy.HttpResponse.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 * /*from   w  ww .  j  a v  a 2s.  co m*/
 * @param http
 *          An instance of the implementation class of this plugin
 * @param url
 *          URL to be fetched
 * @param datum
 *          Crawl data
 * @param followRedirects
 *          Whether to follow redirects; follows redirect if and only if this
 *          is true
 * @return HTTP response
 * @throws IOException
 *           When an error occurs
 */
HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
    try {
        HttpClient client = Http.getClient();
        client.getParams().setParameter("http.useragent", http.getUserAgent()); // NUTCH-1941
        code = client.executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBase.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1
                    && totalRead + bufferFilled <= contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            if (in != null) {
                in.close();
            }
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Logger trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace.toString());
        }
    } finally {
        get.releaseConnection();
    }
}

---

From source file:org.eclipse.smila.connectivity.framework.crawler.web.http.HttpResponse.java

/**
 * Sets the http parameters.//from ww  w .  j av  a 2 s  .c o m
 * 
 * @param http
 *          the http
 * @param httpMethod
 *          the http method
 */
private void setHttpParameters(HttpBase http, HttpMethodBase httpMethod) {
    httpMethod.setFollowRedirects(false);
    httpMethod.setRequestHeader("User-Agent", http.getUserAgent());
    httpMethod.setRequestHeader("Referer", http.getReferer());

    httpMethod.setDoAuthentication(true);

    for (Header header : http.getHeaders()) {
        httpMethod.addRequestHeader(header);
    }

    final HttpMethodParams params = httpMethod.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");

    if (http.isCookiesEnabled()) {
        params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    } else {
        params.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
    }
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // the default is to retry 3 times; if
    // the request body was sent the method is not retried, so there is
    // little danger in retrying
    // retries are handled on the higher level
    params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
}

---

From source file:org.jboss.test.NamingUtil.java

/**
 * This methods calls servlet which must be deployed at server to create JNDI objects remotely to byepass security.
 * //  w w w  . j  a v a  2 s . c  o m
 * @param jndiName
 * @param dataKey
 * @param data
 * @param useHAJNDI
 * @throws Exception
 */
public static void createRemoteTestJNDIBinding(String jndiName, String dataKey, Object data, String serverHost,
        boolean useHAJNDI) throws Exception {

    HttpClient httpClient = new HttpClient();
    String url = "http://" + serverHost + ":8080/naming-util/naming-util-servlet";

    HttpMethodParams params = new HttpMethodParams();
    params.setParameter("jndiName", jndiName);
    if (data != null) {
        params.setParameter("dataKey", dataKey);
        params.setParameter("data", data);
    }
    params.setBooleanParameter("useHAJndi", useHAJNDI);

    url = url + "?jndiName=" + jndiName;
    url = url + "&dataKey=" + dataKey;
    url = url + "&data=" + data;
    url = url + "&useHAJndi=" + Boolean.toString(useHAJNDI);

    GetMethod jndiGet = new GetMethod(url);
    //jndiGet.setParams(params);
    int responseCode = httpClient.executeMethod(jndiGet);
    String body = jndiGet.getResponseBodyAsString();

    if (responseCode != HttpURLConnection.HTTP_OK || !body.contains("OK")) {
        throw new Exception(body);
    }

}

---

• «
• 1
• 2
• »