Example usage for org.apache.commons.httpclient.params HttpMethodParams RETRY_HANDLER

List of usage examples for org.apache.commons.httpclient.params HttpMethodParams RETRY_HANDLER

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.httpclient.params.*
  5. HttpMethodParams
  6. RETRY_HANDLER

Introduction

In this page you can find the example usage for org.apache.commons.httpclient.params HttpMethodParams RETRY_HANDLER.

Prototype

String RETRY_HANDLER

To view the source code for org.apache.commons.httpclient.params HttpMethodParams RETRY_HANDLER.

Click Source Link

Usage

From source file:org.hammer.santamaria.mapper.dataset.CKANDataSetInput.java

@SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] pArgs) throws Exception {
    String id = "proportion-of-children-under-5-years-who-have-ever-breastfed-by-county-xls-2005-6";
    String sId = EncodeURIComponent(id);
    String url = "https://africaopendata.org/api/action";

    BSONObject dataset = new BasicBSONObject();
    dataset.put("datasource", "Test");
    dataset.put("id", id);

    LOG.info("---> id " + id + " - " + sId);

    HttpClient client = new HttpClient();
    client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false);
    LOG.info(//from   ww  w.  ja v a 2  s .c  o  m
            "******************************************************************************************************");
    LOG.info(" ");
    LOG.info(url + PACKAGE_GET + sId);
    LOG.info(" ");
    LOG.info(
            "******************************************************************************************************");

    GetMethod method = new GetMethod(url + PACKAGE_GET + sId);

    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));

    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));

        if (doc != null && doc.containsKey("result")) {
            Document result = new Document();
            LOG.info(doc.get("result").getClass().toString());
            if (doc.get("result") instanceof Document) {
                LOG.info("!!! Document result !!!!");
                result = (Document) doc.get("result");
            } else if (doc.get("result") instanceof ArrayList) {
                LOG.info("!!! Document list !!!!");

                result = (Document) (((ArrayList) doc.get("result")).get(0));
            } else {
                LOG.info("!!! NOT FOUND !!!!");
                result = null;
            }
            LOG.info("result find!");
            if (result != null) {
                dataset.put("title", result.get("title"));
                dataset.put("author", result.get("author"));
                dataset.put("author_email", result.get("author_email"));
                dataset.put("license_id", result.get("license_id"));
            }

            ArrayList<String> tags = new ArrayList<String>();
            ArrayList<String> meta = new ArrayList<String>();
            ArrayList<String> other_tags = new ArrayList<String>();

            if (result.containsKey("author") && result.get("author") != null)
                other_tags.add(result.get("author").toString());
            if (result.containsKey("title") && result.get("title") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString()));
            if (result.containsKey("description") && result.get("description") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString()));

            ArrayList<Document> resources = new ArrayList<Document>();
            if (result != null && result.containsKey("resources")) {
                resources = (ArrayList<Document>) result.get("resources");
                for (Document resource : resources) {
                    if (resource.getString("format").toUpperCase().equals("JSON")) {
                        dataset.put("dataset-type", "JSON");
                        dataset.put("url", resource.get("url"));
                        dataset.put("created", resource.get("created"));
                        dataset.put("description", resource.get("description"));
                        dataset.put("revision_timestamp", resource.get("revision_timestamp"));
                        meta = DSSUtils.GetMetaByResource(resource.get("url").toString());
                    }
                }
            }

            if (result != null && result.containsKey("tags")) {
                ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags");
                for (Document tag : tagsFromCKAN) {
                    if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    } else if (tag.containsKey("display_name")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    }
                }

            }

            dataset.put("tags", tags);
            dataset.put("meta", meta);
            dataset.put("resources", resources);
            dataset.put("other_tags", other_tags);

        }

    } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }

    //GetMetaByDocument("http://catalog.data.gov/api/action/package_show?id=1e68f387-5f1c-46c0-a0d1-46044ffef5bf");
}

From source file:org.hammer.santamaria.mapper.dataset.INPSDataSetInput.java

@SuppressWarnings("unchecked")
public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) {
    BSONObject dataset = new BasicBSONObject();

    dataset.put("datasource", datasource);
    dataset.put("id", id);

    LOG.info(datasource + id);/*from w  w  w  . j a  v  a2 s  .c o  m*/

    HttpClient client = new HttpClient();
    GetMethod method = new GetMethod(url);
    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");

    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));
        dataset.put("title", doc.getString("name"));
        dataset.put("author", doc.getString("author"));
        dataset.put("author_email", doc.getString("author_email"));
        dataset.put("license_id", doc.getString("license_id"));
        dataset.put("description", doc.getString("notes"));

        ArrayList<String> tags = new ArrayList<String>();
        ArrayList<String> meta = new ArrayList<String>();
        ArrayList<String> other_tags = new ArrayList<String>();

        if (doc.containsKey("author"))
            other_tags.add(doc.get("author").toString());
        if (doc.containsKey("title"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("name").toString()));
        if (doc.containsKey("notes"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("notes").toString()));

        ArrayList<Document> resources = (ArrayList<Document>) doc.get("resources");
        for (Document resource : resources) {
            if (resource.getString("format").toUpperCase().equals("JSON")) {
                dataset.put("dataset-type", "JSON");
                dataset.put("url", resource.get("url"));
                dataset.put("created", resource.get("created"));
                dataset.put("revision_timestamp", resource.get("last_modified"));

                meta = this.getMetaByDocument(resource.get("url").toString());

            }
        }

        tags = (ArrayList<String>) doc.get("tags");

        dataset.put("tags", tags);
        dataset.put("meta", meta);
        dataset.put("resources", resources);
        dataset.put("other_tags", other_tags);

    } catch (Exception e) {
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }
    LOG.debug(dataset.get("title"));

    return dataset;
}

From source file:org.hammer.santamaria.mapper.dataset.INPSDataSetInput.java

/**
 * Return meta from document for CKAN implementation
 * /*  ww  w.ja va2s .c o m*/
 * @param url
 * @return
 */
public ArrayList<String> getMetaByDocument(String url) {
    ArrayList<String> meta = new ArrayList<String>();
    HttpClient client = new HttpClient();
    GetMethod method = new GetMethod(url);
    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");

    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();

        if ((JSON.parse(new String(responseBody))) instanceof BasicBSONList) {
            BasicBSONList pList = (BasicBSONList) JSON.parse(new String(responseBody));
            if (pList.size() > 0 && pList.get(0) != null) {
                BSONObject temp = (BSONObject) pList.get(0);
                for (String metaKey : temp.keySet()) {
                    if (!meta.contains(metaKey.toLowerCase())) {
                        meta.add(metaKey.toLowerCase());
                    }
                }
            }
        } else if ((JSON.parse(new String(responseBody))) instanceof BasicDBObject) {
            BasicDBObject temp = (BasicDBObject) JSON.parse(new String(responseBody));
            for (String metaKey : temp.keySet()) {
                if (!meta.contains(metaKey.toLowerCase())) {
                    meta.add(metaKey.toLowerCase());
                }
            }

        }

    } catch (Exception e) {
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }

    return meta;
}

From source file:org.hammer.santamaria.mapper.dataset.StatPortalDataSetInput.java

@SuppressWarnings("unchecked")
public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) {
    BSONObject dataset = new BasicBSONObject();

    dataset.put("datasource", datasource);
    dataset.put("id", id);

    LOG.info(datasource + id);/* w w  w.ja v  a  2 s  . c om*/

    HttpClient client = new HttpClient();
    GetMethod method = new GetMethod(url);
    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");

    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));
        dataset.put("title", doc.getString("title"));
        dataset.put("author", doc.getString("author"));
        dataset.put("author_email", doc.getString("author_email"));
        dataset.put("license_id", doc.getString("license_id"));
        dataset.put("description", doc.getString("notes"));

        ArrayList<String> tags = new ArrayList<String>();
        ArrayList<String> meta = new ArrayList<String>();
        ArrayList<String> other_tags = new ArrayList<String>();

        if (doc.containsKey("author"))
            other_tags.add(doc.get("author").toString());
        if (doc.containsKey("title"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("title").toString()));
        if (doc.containsKey("notes"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("notes").toString()));

        ArrayList<Document> resources = (ArrayList<Document>) doc.get("resources");
        for (Document resource : resources) {
            if (resource.getString("format").toUpperCase().equals("JSON")) {
                dataset.put("dataset-type", "JSON");
                dataset.put("url", resource.get("url"));
                dataset.put("created", resource.get("created"));
                dataset.put("revision_timestamp", resource.get("last_modified"));

                meta = this.getMetaByDocument(resource.get("url").toString());

            }
        }

        tags = (ArrayList<String>) doc.get("tags");

        dataset.put("tags", tags);
        dataset.put("meta", meta);
        dataset.put("resources", resources);
        dataset.put("other_tags", other_tags);

    } catch (Exception e) {
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }
    LOG.debug(dataset.get("title"));

    return dataset;
}

From source file:org.infoscoop.request.ProxyRequest.java

public int executeGet() throws Exception {
    GetMethod method = null;/*from  ww w .j  a v a  2  s.  c o  m*/
    try {
        HttpClient client = this.newHttpClient();
        method = new GetMethod(this.getTargetURL());
        method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
                new DefaultHttpMethodRetryHandler(1, false));
        ProxyFilterContainer filterContainer = (ProxyFilterContainer) SpringUtil.getBean(filterType);
        return filterContainer.invoke(client, method, this);
    } catch (ProxyAuthenticationException e) {
        if (e.isTraceOn()) {
            log.error(this.getTargetURL());
            log.error("", e);
        } else {
            log.warn(this.getTargetURL() + " : " + e.getMessage());
        }
        return 401;
    }
}

From source file:org.infoscoop.request.ProxyRequest.java

public int executePost() throws Exception {
    PostMethod method = null;/*from   ww w .  j a  v  a2s  .c  o m*/
    try {
        HttpClient client = this.newHttpClient();
        method = new PostMethod(this.getTargetURL());
        method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
                new DefaultHttpMethodRetryHandler(1, false));
        if (this.getRequestBody() != null)
            method.setRequestEntity(new InputStreamRequestEntity(this.getRequestBody()));
        ProxyFilterContainer filterContainer = (ProxyFilterContainer) SpringUtil.getBean(filterType);
        return filterContainer.invoke(client, method, this);
    } catch (ProxyAuthenticationException e) {
        if (e.isTraceOn()) {
            log.error(this.getTargetURL());
            log.error("", e);
        } else {
            log.warn(this.getTargetURL() + " : " + e.getMessage());
        }
        return 401;
    }
}

From source file:org.infoscoop.request.ProxyRequest.java

public int executePut() throws Exception {
    PutMethod method = null;//from  w  w  w.j a  v  a2  s.  co m
    try {
        HttpClient client = this.newHttpClient();
        method = new PutMethod(this.getTargetURL());
        method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
                new DefaultHttpMethodRetryHandler(1, false));
        if (this.getRequestBody() != null)
            method.setRequestEntity(new InputStreamRequestEntity(this.getRequestBody()));
        ProxyFilterContainer filterContainer = (ProxyFilterContainer) SpringUtil.getBean(filterType);
        return filterContainer.invoke(client, method, this);
    } catch (ProxyAuthenticationException e) {
        if (e.isTraceOn()) {
            log.error(this.getTargetURL());
            log.error("", e);
        } else {
            log.warn(this.getTargetURL() + " : " + e.getMessage());
        }
        return 401;
    }
}

From source file:org.infoscoop.request.ProxyRequest.java

public int executeReport() throws Exception {
    ReportMethod method = null;/*  w  ww.  ja  v a 2 s .  c  om*/
    try {
        HttpClient client = this.newHttpClient();
        method = new ReportMethod(this.getTargetURL());
        method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
                new DefaultHttpMethodRetryHandler(1, false));
        if (this.getRequestBody() != null)
            method.setRequestEntity(new InputStreamRequestEntity(this.getRequestBody()));
        ProxyFilterContainer filterContainer = (ProxyFilterContainer) SpringUtil.getBean(filterType);
        return filterContainer.invoke(client, method, this);
    } catch (ProxyAuthenticationException e) {
        if (e.isTraceOn()) {
            log.error(this.getTargetURL());
            log.error("", e);
        } else {
            log.warn(this.getTargetURL() + " : " + e.getMessage());
        }
        return 401;
    }
}

From source file:org.infoscoop.request.ProxyRequest.java

public int executeDelete() throws Exception {
    DeleteMethod method = null;//from   www.  j a  va2 s .c  om
    try {
        HttpClient client = this.newHttpClient();
        method = new DeleteMethod(this.getTargetURL());
        method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
                new DefaultHttpMethodRetryHandler(1, false));
        ProxyFilterContainer filterContainer = (ProxyFilterContainer) SpringUtil.getBean(filterType);
        return filterContainer.invoke(client, method, this);
    } catch (ProxyAuthenticationException e) {
        if (e.isTraceOn()) {
            log.error(this.getTargetURL());
            log.error("", e);
        } else {
            log.warn(this.getTargetURL() + " : " + e.getMessage());
        }
        return 401;
    }
}

From source file:org.infoscoop.request.ProxyRequest.java

public int executePropFind() throws Exception {
    PropFindMethod method = null;//from   ww  w  .j a v  a  2  s .  co m
    try {
        HttpClient client = this.newHttpClient();
        method = new PropFindMethod(this.getTargetURL());
        method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
                new DefaultHttpMethodRetryHandler(1, false));
        if (this.getRequestBody() != null)
            method.setRequestEntity(new InputStreamRequestEntity(this.getRequestBody()));
        ProxyFilterContainer filterContainer = (ProxyFilterContainer) SpringUtil.getBean(filterType);
        return filterContainer.invoke(client, method, this);
    } catch (ProxyAuthenticationException e) {
        if (e.isTraceOn()) {
            log.error(this.getTargetURL());
            log.error("", e);
        } else {
            log.warn(this.getTargetURL() + " : " + e.getMessage());
        }
        return 401;
    }
}