Example usage for org.apache.commons.httpclient.params HttpMethodParams RETRY_HANDLER

Introduction

In this page you can find the example usage for org.apache.commons.httpclient.params HttpMethodParams RETRY_HANDLER.

Prototype

String RETRY_HANDLER

To view the source code for org.apache.commons.httpclient.params HttpMethodParams RETRY_HANDLER.

Click Source Link

Usage

From source file:org.apdplat.search.GoogleAjaxSearcher.java

@Override
public SearchResult search(String keyword, int page) {
    int pageSize = 8;
    //???8start?//w  ww. ja  v a2  s  .  c  o  m
    //?start=0start=10start=20?(page-1)*pageSize
    String url = "http://ajax.googleapis.com/ajax/services/search/web?start=" + (page - 1) * pageSize
            + "&rsz=large&v=1.0&q=" + keyword;

    SearchResult searchResult = new SearchResult();
    searchResult.setPage(page);
    List<Webpage> webpages = new ArrayList<>();
    try {
        HttpClient httpClient = new HttpClient();
        GetMethod getMethod = new GetMethod(url);

        httpClient.executeMethod(getMethod);
        getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());

        int statusCode = httpClient.executeMethod(getMethod);
        if (statusCode != HttpStatus.SC_OK) {
            LOG.error("?: " + getMethod.getStatusLine());
            return null;
        }
        InputStream in = getMethod.getResponseBodyAsStream();
        byte[] responseBody = Tools.readAll(in);
        String response = new String(responseBody, "UTF-8");
        LOG.debug("??" + response);
        JSONObject json = new JSONObject(response);
        String totalResult = json.getJSONObject("responseData").getJSONObject("cursor")
                .getString("estimatedResultCount");
        int totalResultCount = Integer.parseInt(totalResult);
        LOG.info("? " + totalResultCount);
        searchResult.setTotal(totalResultCount);

        JSONArray results = json.getJSONObject("responseData").getJSONArray("results");

        LOG.debug("?:");
        for (int i = 0; i < results.length(); i++) {
            Webpage webpage = new Webpage();
            JSONObject result = results.getJSONObject(i);
            //???
            String title = result.getString("titleNoFormatting");
            LOG.debug("" + title);
            webpage.setTitle(title);
            //????
            String summary = result.get("content").toString();
            summary = summary.replaceAll("<b>", "");
            summary = summary.replaceAll("</b>", "");
            summary = summary.replaceAll("\\.\\.\\.", "");
            LOG.debug("?" + summary);
            webpage.setSummary(summary);
            //URL???
            String _url = result.get("url").toString();
            webpage.setUrl(_url);
            String content = Tools.getHTMLContent(_url);
            LOG.debug("" + content);
            webpage.setContent(content);
            webpages.add(webpage);
        }
    } catch (IOException | JSONException | NumberFormatException e) {
        LOG.error("?", e);
    }
    searchResult.setWebpages(webpages);
    return searchResult;
}

---

From source file:org.araqne.pkg.HttpWagon.java

public static InputStream openDownloadStream(URL url, boolean useAuth, String username, String password)
        throws IOException {
    Logger logger = LoggerFactory.getLogger(HttpWagon.class.getName());
    logger.trace("http wagon: downloading {}", url);

    int socketTimeout = getSocketTimeout();
    int connectionTimeout = getConnectTimeout();

    HttpClient client = new HttpClient();

    client.getParams().setParameter("http.socket.timeout", socketTimeout);
    client.getParams().setParameter("http.connection.timeout", connectionTimeout);
    client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, null);

    String realm = "";
    if (useAuth) {
        client.getParams().setAuthenticationPreemptive(true);
        if (realmCache.containsKey(getRealmCacheKey(url)))
            realm = realmCache.get(getRealmCacheKey(url));
        setClientAuth(url, username, password, realm, client);
    }//from  ww w  .j  ava 2 s. c  om

    HttpMethod method = new GetMethod(url.toString());

    int statusCode = client.executeMethod(method);

    if (useAuth && statusCode == HttpStatus.SC_UNAUTHORIZED) {
        realm = getRealm(method);
        setClientAuth(url, username, password, realm, client);
        method = new GetMethod(url.toString());
        statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new IOException("digest auth failed: " + method.getStatusLine());
        } else {
            realmCache.put(getRealmCacheKey(url), realm);
        }
    }

    if (statusCode != HttpStatus.SC_OK) {
        throw new IOException("method failed: " + method.getStatusLine());
    }

    return method.getResponseBodyAsStream();
}

---

From source file:org.archive.crawler.fetcher.FetchHTTP.java

/**
 * Configure the HttpMethod setting options and headers.
 *
 * @param curi CrawlURI from which we pull configuration.
 * @param method The Method to configure.
 * @return HostConfiguration copy customized for this CrawlURI
 */// ww w.  j av  a  2 s  .  c o m
protected HostConfiguration configureMethod(CrawlURI curi, HttpMethod method) {
    // Don't auto-follow redirects
    method.setFollowRedirects(false);

    //        // set soTimeout
    //        method.getParams().setSoTimeout(
    //                ((Integer) getUncheckedAttribute(curi, ATTR_SOTIMEOUT_MS))
    //                        .intValue());

    // Set cookie policy.
    method.getParams()
            .setCookiePolicy((((Boolean) getUncheckedAttribute(curi, ATTR_IGNORE_COOKIES)).booleanValue())
                    ? CookiePolicy.IGNORE_COOKIES
                    : CookiePolicy.BROWSER_COMPATIBILITY);

    // Use only HTTP/1.0 (to avoid receiving chunked responses)
    method.getParams().setVersion(HttpVersion.HTTP_1_0);

    CrawlOrder order = getSettingsHandler().getOrder();
    String userAgent = curi.getUserAgent();
    if (userAgent == null) {
        userAgent = order.getUserAgent(curi);
    }
    method.setRequestHeader("User-Agent", userAgent);
    method.setRequestHeader("From", order.getFrom(curi));

    // Set retry handler.
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new HeritrixHttpMethodRetryHandler());

    final long maxLength = getMaxLength(curi);
    if (maxLength > 0 && ((Boolean) getUncheckedAttribute(curi, ATTR_SEND_RANGE)).booleanValue()) {
        method.addRequestHeader(RANGE, RANGE_PREFIX.concat(Long.toString(maxLength - 1)));
    }

    if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_CONNECTION_CLOSE)).booleanValue()) {
        method.addRequestHeader(HEADER_SEND_CONNECTION_CLOSE);
    }

    if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_REFERER)).booleanValue()
            && (curi.getViaContext() == null || !Link.PREREQ_MISC.equals(curi.getViaContext().toString()))) {
        // RFC2616 says no referer header if referer is https and the url
        // is not
        String via = curi.flattenVia();
        if (via != null && via.length() > 0
                && !(via.startsWith(HTTPS_SCHEME) && curi.getUURI().getScheme().equals(HTTP_SCHEME))) {
            method.setRequestHeader(REFERER, via);
        }
    }

    if (!curi.isPrerequisite()) {
        setConditionalGetHeader(curi, method, ATTR_SEND_IF_MODIFIED_SINCE,
                CoreAttributeConstants.A_LAST_MODIFIED_HEADER, "If-Modified-Since");
        setConditionalGetHeader(curi, method, ATTR_SEND_IF_NONE_MATCH, CoreAttributeConstants.A_ETAG_HEADER,
                "If-None-Match");
    }

    // TODO: What happens if below method adds a header already
    // added above: e.g. Connection, Range, or Referer?
    setAcceptHeaders(curi, method);

    HostConfiguration config = new HostConfiguration(http.getHostConfiguration());
    configureProxy(curi, config);
    configureBindAddress(curi, config);
    return config;
}

---

From source file:org.archive.crawler.fetcher.OptimizeFetchHTTP.java

/**
 * Configure the HttpMethod setting options and headers.
 *
 * @param curi CrawlURI from which we pull configuration.
 * @param method The Method to configure.
 * @return HostConfiguration copy customized for this CrawlURI
 *///from www .j  av  a2s . c  o  m
protected HostConfiguration configureMethod(CrawlURI curi, HttpMethod method) {
    // Don't auto-follow redirects
    method.setFollowRedirects(false);

    //        // set soTimeout
    //        method.getParams().setSoTimeout(
    //                ((Integer) getUncheckedAttribute(curi, ATTR_SOTIMEOUT_MS))
    //                        .intValue());

    // Set cookie policy.
    method.getParams()
            .setCookiePolicy((((Boolean) getUncheckedAttribute(curi, ATTR_IGNORE_COOKIES)).booleanValue())
                    ? CookiePolicy.IGNORE_COOKIES
                    : CookiePolicy.BROWSER_COMPATIBILITY);

    // Use only HTTP/1.0 (to avoid receiving chunked responses)
    method.getParams().setVersion(HttpVersion.HTTP_1_0);

    CrawlOrder order = getSettingsHandler().getOrder();
    String userAgent = curi.getUserAgent();
    if (userAgent == null) {
        userAgent = order.getUserAgent(curi);
    }
    method.setRequestHeader("User-Agent", userAgent);
    method.setRequestHeader("From", order.getFrom(curi));

    // Set retry handler.
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new HeritrixHttpMethodRetryHandler());

    final long maxLength = getMaxLength(curi);
    if (maxLength > 0 && ((Boolean) getUncheckedAttribute(curi, ATTR_SEND_RANGE)).booleanValue()) {
        method.addRequestHeader(RANGE, RANGE_PREFIX.concat(Long.toString(maxLength - 1)));
    }

    if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_CONNECTION_CLOSE)).booleanValue()) {
        method.addRequestHeader(HEADER_SEND_CONNECTION_CLOSE);
    }

    if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_REFERER)).booleanValue()
            && (curi.getViaContext() == null || !Link.PREREQ_MISC.equals(curi.getViaContext().toString()))) {
        // RFC2616 says no referer header if referer is https and the url
        // is not
        String via = curi.flattenVia();
        if (via != null && via.length() > 0
                && !(via.startsWith(HTTPS_SCHEME) && curi.getUURI().getScheme().equals(HTTP_SCHEME))) {
            method.setRequestHeader(REFERER, via);
        }
    }

    /*if(!curi.isPrerequisite() && curi.containsKey(URLInfo.MODIFY_TIME) &&
       (Boolean)getUncheckedAttribute(curi, ATTR_SEND_IF_MODIFIED_SINCE)) {
      long modifyTime = curi.getLong(URLInfo.MODIFY_TIME);
      if (modifyTime != 0) {
      Date date = new Date(modifyTime);
      method.setRequestHeader("If-Modified-Since", date.toString());
      logger.debug(curi.getUURI().toString() + " send header modifyTime:" + date.toGMTString());
      }
              
      setConditionalGetHeader(curi, method, ATTR_SEND_IF_MODIFIED_SINCE, 
           CoreAttributeConstants.A_LAST_MODIFIED_HEADER, "If-Modified-Since");
      setConditionalGetHeader(curi, method, ATTR_SEND_IF_NONE_MATCH, 
        CoreAttributeConstants.A_ETAG_HEADER, "If-None-Match");
    }*/

    // TODO: What happens if below method adds a header already
    // added above: e.g. Connection, Range, or Referer?
    setAcceptHeaders(curi, method);

    HttpClient http = getClient();
    HostConfiguration config = new HostConfiguration(http.getHostConfiguration());
    configureProxy(curi, config);
    configureBindAddress(curi, config);
    return config;
}

---

From source file:org.bench4Q.agent.rbe.HttpClientFactory.java

/**
 * get a HttpClient./*from   www  . j a  v  a2s.  com*/
 * 
 * @return HttpClient
 */
public static HttpClient getInstance() {
    if (m_client == null) {
        MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
        connectionManager.setMaxConnectionsPerHost(10000);
        connectionManager.setMaxTotalConnections(10000);
        m_client = new HttpClient(connectionManager);

        DefaultMethodRetryHandler retryhandler = new DefaultMethodRetryHandler();
        retryhandler.setRetryCount(DEFAULT_RETRY_COUNT);
        m_client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler);
    }
    return m_client;

}

---

From source file:org.bench4Q.agent.rbe.HttpClientFactory.java

/**
 * set retry count.//www .  j a v  a 2  s. c  om
 * 
 * @param count
 */
public static void setRetryCount(int count) {
    if (m_client != null) {
        DefaultMethodRetryHandler retryhandler = new DefaultMethodRetryHandler();
        retryhandler.setRetryCount(count);
        m_client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler);
    }

}

---

From source file:org.biomart.martservice.MartServiceUtils.java

/**
 * @param martServiceLocation//from   w  w w .j  av  a2  s.co  m
 * @param data
 * @return
 * @throws MartServiceException
 */
private static InputStream executeMethod(HttpMethod method, String martServiceLocation)
        throws MartServiceException {
    HttpClient client = new HttpClient();
    if (isProxyHost(martServiceLocation)) {
        setProxy(client);
    }

    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    //      method.getParams().setSoTimeout(60000);
    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw constructException(method, martServiceLocation, null);
        }
        return method.getResponseBodyAsStream();
    } catch (IOException e) {
        throw constructException(method, martServiceLocation, e);
    }
}

---

From source file:org.blackdog.lyrics.type.LeoLyrics.java

/** return the page containing the result of the search
 *  @param searchArtist true to search for artist
 *  @param artistTokens a list of String representing the critera for artist
 *  @param searchSong true to search for song
 *  @param songTokens a list of String representing the critera for the song
 *  @return the id of the page containing the lyrics of the first candidate or null if not found
 *//*  www.  j a  v a2  s  . co m*/
private String getResultPage(boolean searchArtist, List<String> artistTokens, boolean searchSong,
        List<String> songTokens) throws HttpException, IOException {
    String id = null;

    if ((searchArtist || searchSong) && ((artistTokens == null ? false : artistTokens.size() > 0)
            || (songTokens == null ? false : songTokens.size() > 0))) {
        HttpClient client = new HttpClient();

        StringBuffer uri = new StringBuffer();

        // create uri : http://www.leoslyrics.com/search.php?search=zero+smashing&sartist=1&ssongtitle=1

        uri.append(LEO_URL + SEARCH_SUFFIX);

        boolean tokenAddedForSong = false;

        if (searchSong && songTokens != null) {
            for (int i = 0; i < songTokens.size(); i++) {
                String currentToken = songTokens.get(i);

                if (currentToken != null) {
                    currentToken = currentToken.trim();

                    if (currentToken.length() > 0) {
                        if (tokenAddedForSong) {
                            uri.append("+");
                        }

                        uri.append(currentToken);

                        tokenAddedForSong = true;
                    }
                }
            }
        }
        if (searchSong && artistTokens != null) {
            for (int i = 0; i < artistTokens.size(); i++) {
                String currentToken = artistTokens.get(i);

                if (currentToken != null) {
                    currentToken = currentToken.trim();

                    if (currentToken.length() > 0) {
                        if (tokenAddedForSong) {
                            uri.append("+");
                        }

                        uri.append(currentToken);

                        tokenAddedForSong = true;
                    }
                }
            }
        }

        if (searchArtist) {
            uri.append("&");
            uri.append("sartist=1");
        }

        if (searchSong) {
            uri.append("&");
            uri.append("ssongtitle=1");
        }

        logger.info("leo lyrics request : " + uri);

        GetMethod method = new GetMethod(uri.toString());

        method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
                new DefaultHttpMethodRetryHandler(5, false));

        try {
            int statusCode = client.executeMethod(method);

            if (statusCode == HttpStatus.SC_OK) {
                InputStream stream = method.getResponseBodyAsStream();

                if (stream != null) { /* search first occurrence of '?hid=' */
                    String hidString = "?hid=";
                    StringBuffer bufferId = new StringBuffer();
                    StringBuffer bufferTmp = new StringBuffer(hidString.length());
                    boolean idFound = false;

                    byte[] buffer = new byte[128];

                    try {
                        int byteRead = stream.read(buffer);

                        while (byteRead != -1) {
                            if (!idFound) {
                                for (int i = 0; i < byteRead; i++) {
                                    byte b = buffer[i];

                                    if (!idFound) {
                                        if (bufferTmp.length() >= hidString.length()) {
                                            if (b != '"') {
                                                bufferId.append((char) b);
                                            } else {
                                                idFound = true;
                                            }
                                        } else {
                                            if (b == hidString.charAt(bufferTmp.length())) {
                                                bufferTmp.append((char) b);
                                            } else if (bufferTmp.length() > 0) {
                                                bufferTmp.delete(0, bufferTmp.length() - 1);
                                            }
                                        }
                                    }
                                }
                            }

                            byteRead = stream.read(buffer);
                        }
                    } finally {
                        if (stream != null) {
                            try {
                                stream.close();
                            } catch (IOException e) {
                                logger.warn("unable to close the input stream from request " + uri, e);
                            }
                        }
                    }

                    if (bufferId.length() > 0) {
                        id = bufferId.toString();
                    }
                }
            } else {
                logger.error("getting error while trying to connect to : " + uri + " (error code=" + statusCode
                        + ")");
            }
        } finally {
            method.releaseConnection();
        }
    }

    return id;
}

---

From source file:org.blackdog.lyrics.type.LeoLyrics.java

/** return an html String representing the lyrics content of the page with the given uri
 *  @param uri the uri of the page containing the song lyrics
 *  @return a String or null if failed//from w w w.j  a v  a  2s .  c om
 */
private String getLyricsContent(String uri) throws HttpException, IOException {
    String content = null;

    if (uri != null) {
        HttpClient client = new HttpClient();

        GetMethod method = new GetMethod(uri);
        //       method.addRequestHeader(Header.)

        method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
                new DefaultHttpMethodRetryHandler(5, false));

        try {
            int statusCode = client.executeMethod(method);
            if (statusCode == HttpStatus.SC_OK) {
                String response = method.getResponseBodyAsString();

                if (response != null) {
                    /* search first occurrence of 'This song is on the following albums'
                     *
                     *   if not found, search <font face="Trebuchet MS, Verdana, Arial" size=-1>
                     *   take to next </font>
                     */
                    String headerString = "This song is on the following albums";
                    String headerString2 = "<font face=\"Trebuchet MS, Verdana, Arial\" size=-1>";
                    String paragraphString = "<p>";
                    String paragraphEndString = "</p>";
                    StringBuffer bufferContent = new StringBuffer(800);
                    StringBuffer bufferHeaderTmp = new StringBuffer(headerString.length());
                    StringBuffer bufferParagraphTmp = new StringBuffer(paragraphString.length());
                    StringBuffer bufferParagraphEndTmp = new StringBuffer(paragraphEndString.length());
                    boolean headerFound = false;
                    boolean paragraphFound = false;
                    boolean contentFound = false;
                    //          
                    for (int i = 0; i < response.length() && !contentFound; i++) {
                        char b = response.charAt(i);

                        /** header found */
                        if (headerFound) {
                            if (paragraphFound) {
                                bufferContent.append((char) b);

                                if (b == paragraphEndString.charAt(bufferParagraphEndTmp.length())) {
                                    bufferParagraphEndTmp.append((char) b);
                                } else if (bufferParagraphEndTmp.length() > 0) {
                                    bufferParagraphEndTmp.delete(0, bufferParagraphEndTmp.length() - 1);
                                }

                                /* did we find the end of the paragraph ? */
                                if (bufferParagraphEndTmp.length() >= paragraphEndString.length()) {
                                    contentFound = true;
                                }
                            } else {
                                //                                                System.out.println("  <<");
                                if (b == paragraphString.charAt(bufferParagraphTmp.length())) {
                                    bufferParagraphTmp.append((char) b);
                                } else if (bufferParagraphTmp.length() > 0) {
                                    bufferParagraphTmp.delete(0, bufferParagraphTmp.length() - 1);
                                }

                                if (bufferParagraphTmp.length() >= paragraphString.length()) {
                                    paragraphFound = true;
                                    bufferHeaderTmp.append(headerString);
                                }
                            }
                        } else {
                            //                                            System.out.println("  <<");
                            if (b == headerString.charAt(bufferHeaderTmp.length())) {
                                bufferHeaderTmp.append((char) b);
                            } else if (bufferHeaderTmp.length() > 0) {
                                bufferHeaderTmp.delete(0, bufferHeaderTmp.length() - 1);
                            }

                            if (bufferHeaderTmp.length() >= headerString.length()) {
                                headerFound = true;
                            }
                        }
                    }

                    if (!contentFound) {
                        int headerString2Index = response.indexOf(headerString2);

                        if (headerString2Index > -1) {
                            String truncatedResponse = response.substring(headerString2Index);

                            int lastEndfont = truncatedResponse.indexOf("</font>");

                            if (lastEndfont > -1) {
                                bufferContent.append(truncatedResponse.substring(0, lastEndfont));
                                contentFound = true;
                            }
                        }
                    }

                    if (bufferContent.length() > 0) {
                        bufferContent.insert(0, "<html>");
                        bufferContent.append("</html>");

                        try {
                            content = new String(bufferContent.toString().getBytes(),
                                    method.getResponseCharSet()); // "UTF-8"
                        } catch (UnsupportedEncodingException e) {
                            logger.warn("unable to encode lyrics content in UTF-8");
                            content = bufferContent.toString();
                        }
                    }
                }
            } else {
                logger.error("getting error while trying to connect to : " + uri + " (error code=" + statusCode
                        + ")");
            }
        } finally {
            method.releaseConnection();
        }
    }

    return content;
}

---

From source file:org.bonitasoft.console.common.application.ApplicationURLUtils.java

/**
 * Check if the dedicated application URL is reachable
 * /* w w w . j a v  a  2s  .c  o m*/
 * @param anUrlToTest the URL to check
 * @return true if the URL is reacheable, false otherwise
 */
private boolean checkURLConnection(final URL anUrlToTest) {

    final HttpClient client = new HttpClient();

    // Create a method instance.
    final GetMethod method = new GetMethod(anUrlToTest.toString());

    // Provide custom retry handler is necessary
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(0, false));

    boolean urlReachable = false;
    try {
        // Execute the method.
        final int statusCode = client.executeMethod(method);

        urlReachable = (statusCode == HttpStatus.SC_OK);

    } catch (final HttpException e) {
        if (LOGGER.isLoggable(Level.WARNING)) {
            LOGGER.warning("Fatal protocol violation: " + e.getMessage());
        }
    } catch (final IOException e) {
        if (LOGGER.isLoggable(Level.WARNING)) {
            LOGGER.warning("Fatal transport error: " + e.getMessage());
        }
    } finally {
        // Release the connection.
        method.releaseConnection();
    }
    return urlReachable;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• »