Example usage for org.apache.commons.httpclient.methods GetMethod GetMethod

Introduction

In this page you can find the example usage for org.apache.commons.httpclient.methods GetMethod GetMethod.

Prototype

public GetMethod(String uri)

Source Link

Usage

From source file:com.iflytek.spider.protocol.httpclient.HttpResponse.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 * /*w  w w  .  ja v  a  2 s  . com*/
 * @param http
 *            An instance of the implementation class of this plugin
 * @param url
 *            URL to be fetched
 * @param datum
 *            Crawl data
 * @param followRedirects
 *            Whether to follow redirects; follows redirect if and only if
 *            this is true
 * @return HTTP response
 * @throws IOException
 *             When an error occurs
 */
HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
    try {
        code = Http.getClient().executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBase.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead < contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            in.close();
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Log trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace);
        }
    } finally {
        get.releaseConnection();
    }
}

From source file:com.iflytek.spider.protocol.httpclient.HttpResponseSmiply.java

/**
 * Fetches the given <code>url</code> and prepares HTTP response.
 * /*  w  ww  . java2 s  . c  o m*/
 * @param http
 *            An instance of the implementation class of this plugin
 * @param url
 *            URL to be fetched
 * @param datum
 *            Crawl data
 * @param followRedirects
 *            Whether to follow redirects; follows redirect if and only if
 *            this is true
 * @return HTTP response
 * @throws IOException
 *             When an error occurs
 */
HttpResponseSmiply(HttpSimply http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException {

    // Prepare GET method for HTTP request
    this.url = url;
    GetMethod get = new GetMethod(url.toString());
    get.setFollowRedirects(followRedirects);
    get.setDoAuthentication(true);
    if (datum.getModifiedTime() > 0) {
        get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime()));
    }

    // Set HTTP parameters
    HttpMethodParams params = get.getParams();
    if (http.getUseHttp11()) {
        params.setVersion(HttpVersion.HTTP_1_1);
    } else {
        params.setVersion(HttpVersion.HTTP_1_0);
    }
    params.makeLenient();
    params.setContentCharset("UTF-8");
    params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
    // XXX (ab) not sure about this... the default is to retry 3 times; if
    // XXX the request body was sent the method is not retried, so there is
    // XXX little danger in retrying...
    // params.setParameter(HttpMethodParams.RETRY_HANDLER, null);
    try {
        code = HttpSimply.getClient().executeMethod(get);

        Header[] heads = get.getResponseHeaders();

        for (int i = 0; i < heads.length; i++) {
            headers.set(heads[i].getName(), heads[i].getValue());
        }

        // Limit download size
        int contentLength = Integer.MAX_VALUE;
        String contentLengthString = headers.get(Response.CONTENT_LENGTH);
        if (contentLengthString != null) {
            try {
                contentLength = Integer.parseInt(contentLengthString.trim());
            } catch (NumberFormatException ex) {
                throw new HttpException("bad content length: " + contentLengthString);
            }
        }
        if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) {
            contentLength = http.getMaxContent();
        }

        // always read content. Sometimes content is useful to find a cause
        // for error.
        InputStream in = get.getResponseBodyAsStream();
        try {
            byte[] buffer = new byte[HttpBaseSimply.BUFFER_SIZE];
            int bufferFilled = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead < contentLength) {
                totalRead += bufferFilled;
                out.write(buffer, 0, bufferFilled);
            }

            content = out.toByteArray();
        } catch (Exception e) {
            if (code == 200)
                throw new IOException(e.toString());
            // for codes other than 200 OK, we are fine with empty content
        } finally {
            in.close();
            get.abort();
        }

        StringBuilder fetchTrace = null;
        if (Http.LOG.isTraceEnabled()) {
            // Trace message
            fetchTrace = new StringBuilder(
                    "url: " + url + "; status code: " + code + "; bytes received: " + content.length);
            if (getHeader(Response.CONTENT_LENGTH) != null)
                fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH));
            if (getHeader(Response.LOCATION) != null)
                fetchTrace.append("; Location: " + getHeader(Response.LOCATION));
        }
        // Extract gzip, x-gzip and deflate content
        if (content != null) {
            // check if we have to uncompress it
            String contentEncoding = headers.get(Response.CONTENT_ENCODING);
            if (contentEncoding != null && Http.LOG.isTraceEnabled())
                fetchTrace.append("; Content-Encoding: " + contentEncoding);
            if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
                content = http.processGzipEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            } else if ("deflate".equals(contentEncoding)) {
                content = http.processDeflateEncoded(content, url);
                if (Http.LOG.isTraceEnabled())
                    fetchTrace.append("; extracted to " + content.length + " bytes");
            }
        }

        // Log trace message
        if (Http.LOG.isTraceEnabled()) {
            Http.LOG.trace(fetchTrace);
        }
    } finally {
        get.releaseConnection();
    }
}

From source file:com.thoughtworks.go.agent.functional.X509CertificateTest.java

@Test
public void shouldSaveCertificateInAgentTrustStore() throws Exception {
    Protocol authhttps = new Protocol("https", protocolSocketFactory, sslPort);
    Protocol.registerProtocol("https", authhttps);
    HttpClient client = new HttpClient();
    GetMethod httpget = new GetMethod("https://localhost:" + sslPort + "/go/");
    client.executeMethod(httpget);//from  w ww . j a  v a2s  .c o m
    assertTrue("Should have created trust store", truststore.exists());
}

From source file:ca.uvic.cs.tagsea.research.GetIDWithProgress.java

public void run(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
    GetMethod getUidMethod = new GetMethod(uidScript);
    getUidMethod.setQueryString(new NameValuePair[] { first, last, email, job, size, buisness, anon });
    monitor.beginTask("Get User Id", 1);
    HttpClient client = new HttpClient();
    try {//from ww w .  j ava 2  s.com
        status = client.executeMethod(getUidMethod);

        resp = getData(getUidMethod.getResponseBodyAsStream());

        // release the connection to the server
        getUidMethod.releaseConnection();
    } catch (Exception e) {
        // there was a problem with the file upload so throw up
        // an error
        // dialog to inform the user and log the exception
        failed = true;
        throw new InvocationTargetException(e);
    }

    if (status != 200) {
        // there was a problem with the file upload so throw up an error
        // dialog to inform the user

        failed = true;

        // there was a problem with the file upload so throw up an error
        // dialog to inform the user
        MessageDialog.openError(null, "Error Getting User ID", "There was an error getting a user id: \n"
                + "HTTP Response Code " + status + "\n" + "Please try again later");
    } else {
        resp = resp.substring(resp.indexOf(":") + 1).trim();
        uid = Integer.parseInt(resp);
    }

    monitor.worked(1);
    monitor.done();
}

From source file:com.htmlhifive.tools.jslint.engine.download.AbstractDownloadEngineSupport.java

@Override
public EngineInfo getEngineInfo(IProgressMonitor monitor) throws IOException {
    IProgressMonitor actualMonitor = monitor;
    if (monitor == null) {
        actualMonitor = new NullProgressMonitor();
    }/*ww w . j a  v  a  2 s. co  m*/
    actualMonitor.setTaskName(Messages.T0009.getText());
    HttpClient client = createHttpClient(getEngineSourceUrl());
    HttpMethod getMethod = new GetMethod(getEngineSourceUrl());
    int result = client.executeMethod(getMethod);
    if (result != HttpStatus.SC_OK) {
        // TODO 
        return null;
    }
    StringBuilder licenseSb = new StringBuilder();
    StringBuilder rawSource = new StringBuilder();
    Header header = getMethod.getResponseHeader("Content-Length");
    int content = Integer.valueOf(header.getValue());
    actualMonitor.beginTask(Messages.T0010.getText(), content);
    BufferedReader reader = new BufferedReader(new InputStreamReader(getMethod.getResponseBodyAsStream()));
    String temp = reader.readLine();
    int progress = 0;
    while (!isEndLicenseLine(temp)) {
        progress += temp.length();
        actualMonitor.subTask(Messages.T0011.format(progress, content));
        actualMonitor.worked(temp.length());
        rawSource.append(temp);
        temp = StringUtils.trim(temp);
        temp = StringUtils.substring(temp, 2);
        temp = StringUtils.trim(temp);
        licenseSb.append(temp);
        licenseSb.append(System.getProperty("line.separator"));
        rawSource.append(System.getProperty("line.separator"));
        temp = reader.readLine();
    }
    EngineInfo info = new EngineInfo();
    info.setLicenseStr(licenseSb.toString());

    while ((temp = reader.readLine()) != null) {
        progress += temp.length();
        actualMonitor.subTask(Messages.T0011.format(progress, content));
        actualMonitor.worked(temp.length());
        rawSource.append(temp);
        rawSource.append(System.getProperty("line.separator"));
    }
    info.setMainSource(rawSource.toString());
    monitor.done();
    return info;
}

From source file:mapbuilder.ProxyRedirect.java

/***************************************************************************
 * Process the HTTP Get request//from w  w w . ja  v a 2 s  . co  m
 */
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException {
    try {
        if (log.isDebugEnabled()) {
            Enumeration e = request.getHeaderNames();
            while (e.hasMoreElements()) {
                String name = (String) e.nextElement();
                String value = request.getHeader(name);
                log.debug("request header:" + name + ":" + value);
            }
        }

        // Transfer bytes from in to out
        log.debug("HTTP GET: transferring...");

        //execute the GET
        String serverUrl = request.getParameter("url");
        if (serverUrl.startsWith("http://") || serverUrl.startsWith("https://")) {
            log.info("GET param serverUrl:" + serverUrl);
            HttpClient client = new HttpClient();
            GetMethod httpget = new GetMethod(serverUrl);
            client.executeMethod(httpget);

            if (log.isDebugEnabled()) {
                Header[] respHeaders = httpget.getResponseHeaders();
                for (int i = 0; i < respHeaders.length; ++i) {
                    String headerName = respHeaders[i].getName();
                    String headerValue = respHeaders[i].getValue();
                    log.debug("responseHeaders:" + headerName + "=" + headerValue);
                }
            }

            //dump response to out
            if (httpget.getStatusCode() == HttpStatus.SC_OK) {
                //force the response to have XML content type (WMS servers generally don't)
                response.setContentType("text/xml");
                String responseBody = httpget.getResponseBodyAsString().trim();
                // use encoding of the request or UTF8
                String encoding = request.getCharacterEncoding();
                if (encoding == null)
                    encoding = "UTF-8";
                response.setCharacterEncoding(encoding);
                log.info("responseEncoding:" + encoding);
                // do not set a content-length of the response (string length might not match the response byte size)
                //response.setContentLength(responseBody.length());
                log.info("responseBody:" + responseBody);
                PrintWriter out = response.getWriter();
                out.print(responseBody);
                response.flushBuffer();
            } else {
                log.error("Unexpected failure: " + httpget.getStatusLine().toString());
            }
            httpget.releaseConnection();
        } else {
            throw new ServletException("only HTTP(S) protocol supported");
        }

    } catch (Throwable e) {
        throw new ServletException(e);
    }
}

From source file:it.geosolutions.figis.requester.HTTPUtils.java

/**
* Performs an HTTP GET on the given URL.
* <BR>Basic auth is used if both username and pw are not null.
*
* @param url The URL where to connect to.
* @param username Basic auth credential. No basic auth if null.
* @param pw Basic auth credential. No basic auth if null.
* @return The HTTP response as a String if the HTTP response code was 200 (OK).
* @throws MalformedURLException/*from  w w  w .j a va  2 s.c o  m*/
*/
public static String get(String url, String username, String pw) throws MalformedURLException {
    GetMethod httpMethod = null;
    try {
        HttpClient client = new HttpClient();
        setAuth(client, url, username, pw);
        httpMethod = new GetMethod(url);
        client.getHttpConnectionManager().getParams().setConnectionTimeout(5000);

        int status = client.executeMethod(httpMethod);
        if (status == HttpStatus.SC_OK) {
            InputStream is = httpMethod.getResponseBodyAsStream();
            String response = IOUtils.toString(is);
            if (response.trim().length() == 0) // sometime gs rest fails
            {
                LOGGER.warn("ResponseBody is empty");

                return null;
            } else {
                return response;
            }
        } else {
            LOGGER.info("(" + status + ") " + HttpStatus.getStatusText(status) + " -- " + url);
        }
    } catch (ConnectException e) {
        LOGGER.error("Couldn't connect to [" + url + "]", e);
    } catch (IOException e) {
        LOGGER.error("Error talking to [" + url + "]", e);
    } finally {
        if (httpMethod != null) {
            httpMethod.releaseConnection();
        }
    }

    return null;
}

From source file:net.sourceforge.jwbf.actions.mw.editing.GetRevision.java

/**
 * TODO follow redirects.//w w  w  .  jav a 2s  .  c  om
 */
public GetRevision(final String articlename, final int property) {
    sa = new SimpleArticle();
    sa.setLabel(articlename);
    String uS = "";
    URI uri = null;
    try {
        uS = "/api.php?action=query&prop=revisions&titles="
                + URLEncoder.encode(articlename, MediaWikiBot.CHARSET) + "&rvprop="
                + getDataProperties(property) + getReversion(property) + "&rvlimit=1" + "&format=xml";
        uri = new URI(uS);
    } catch (UnsupportedEncodingException e) {
        e.printStackTrace();
    } catch (URISyntaxException e) {
        e.printStackTrace();
    }
    LOG.debug(uS);
    msgs.add(new GetMethod(uri.toString()));

}

From source file:com.tasktop.c2c.server.common.web.server.AvatarImageController.java

@RequestMapping(value = "{hash}.jpg", method = RequestMethod.GET)
public void handleRequest(HttpServletRequest request, HttpServletResponse response,
        @PathVariable("hash") String hash, @RequestParam(value = "d", required = false) String defaultUrl,
        @RequestParam(value = "s", required = false) String size) throws HttpException, IOException {
    String jsonUrl = gravatarBaseJsonUrl + hash + ".json";

    GetMethod get = new GetMethod(jsonUrl);

    int rc = -1;/*from   w w w  .jav a 2s .c o m*/

    try {
        rc = client.executeMethod(get);
    } catch (Exception e) {
        LOGGER.info("Error while contacting gravatar API", e);
    } finally {
        get.releaseConnection();
    }

    if (rc == 200) {
        if (size == null) {
            size = "40";
        }
        response.sendRedirect(gravatarBaseImageUrl + hash + ".jpg?s=" + size);
    } else {
        if (defaultUrl != null) {
            response.sendRedirect(defaultUrl);
        } else {
            response.sendError(404);
        }
    }
}

From source file:com.adobe.translation.google.impl.TranslationServiceImpl.java

public String translate(String text, String src, String dst) {
    GetMethod get = new GetMethod(GOOGLE_API_V2_URI);
    List<NameValuePair> query = new LinkedList<NameValuePair>();
    query.add(new NameValuePair("key", key));
    if (src != null && src.length() > 0) {
        query.add(new NameValuePair("source", src));
    }/*ww w . j a v a2s  .  c  o m*/
    query.add(new NameValuePair("target", dst));
    query.add(new NameValuePair("q", text));
    get.setQueryString(query.toArray(new NameValuePair[query.size()]));
    try {
        int code = httpClient.executeMethod(get);
        if (code != 200) {
            log.error("Unable to translate text. Server responded {}.", code);
            log.error("Response body: {}", get.getResponseBodyAsString());
        } else {
            // we just concatenate the texts
            StringBuilder ret = new StringBuilder();
            JSONObject json = new JSONObject(get.getResponseBodyAsString());
            JSONObject data = json.getJSONObject("data");
            JSONArray translations = data.getJSONArray("translations");
            for (int i = 0; i < translations.length(); i++) {
                JSONObject translation = translations.getJSONObject(i);
                ret.append(translation.getString("translatedText"));
            }
            return ret.toString();
        }
    } catch (HttpException e) {
        log.error("Error while translating.", e);
    } catch (JSONException e) {
        log.error("Error while translating.", e);
    } catch (IOException e) {
        log.error("Error while translating.", e);
    } finally {
        get.releaseConnection();
    }
    return "";
}