Example usage for org.apache.commons.httpclient DefaultHttpMethodRetryHandler DefaultHttpMethodRetryHandler

List of usage examples for org.apache.commons.httpclient DefaultHttpMethodRetryHandler DefaultHttpMethodRetryHandler

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.httpclient.*
  5. DefaultHttpMethodRetryHandler
  6. DefaultHttpMethodRetryHandler

Introduction

In this page you can find the example usage for org.apache.commons.httpclient DefaultHttpMethodRetryHandler DefaultHttpMethodRetryHandler.

Prototype

public DefaultHttpMethodRetryHandler(int paramInt, boolean paramBoolean)

Source Link

Usage

From source file:org.esgf.solr.proxy.SolrProxyController.java

private String relay(HttpServletRequest request, HttpServletResponse response) {
    System.out.println("--TIME MEASUREMENT FOR LOADING RESULTS--");

    long beginTime = System.nanoTime();

    String queryString = request.getQueryString();
    LOG.debug("queryString=" + queryString);
    String requestUri = request.getRequestURI();
    LOG.debug("requestUri=" + requestUri);

    String urlString = solrURL + "select?" + queryString + "&wt=json";

    String responseBody = null;/*  w  ww.j a  va  2 s  . c  o  m*/

    // create an http client
    HttpClient client = new HttpClient();

    // create a method instance
    GetMethod method = new GetMethod(urlString);

    // custom retry
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    try {
        // execute the method
        int statusCode = client.executeMethod(method);

        if (statusCode != HttpStatus.SC_OK) {
            LOG.error("Method failed: " + method.getStatusLine());

        }

        // read the response
        responseBody = method.getResponseBodyAsString();

    } catch (HTTPException e) {
        LOG.error("Fatal protocol violation");
        e.printStackTrace();
    } catch (IOException e) {
        LOG.error("Fatal transport error");
        e.printStackTrace();
    } finally {
        method.releaseConnection();
    }

    LOG.debug("Solr URL = " + urlString);
    //LOG.debug("responseBody = " + responseBody);

    long endTime = System.nanoTime();

    System.out.println("\tTotal Time: " + ((int) (endTime - beginTime)) + "ns");

    System.out.println("-----------------------------------------");

    return responseBody;
}

From source file:org.eurekastreams.server.service.utility.http.HttpDocumentFetcherImpl.java

/**
 * Retrieves an XML document from a given URL.
 *
 * @param url//from w  ww. jav  a  2 s . c  om
 *            The URL from which to get the document.
 * @param httpHeaders
 *            HTTP headers to add to the request.
 * @param proxyHost
 *            host name to use (if desired) for proxying http requests.
 * @param proxyPort
 *            port for http proxy server.
 * @param timeout
 *            the timeout period to wait for the request to return (in ms).
 * @param domFactory
 *            Factory for creating document builders.
 * @return The document.
 * @throws IOException
 *             On error.
 * @throws ParserConfigurationException
 *             On error.
 * @throws SAXException
 *             On error.
 */
public Document fetchDocument(final String url, final Map<String, String> httpHeaders, final String proxyHost,
        final String proxyPort, final int timeout, final DocumentBuilderFactory domFactory)
        throws IOException, ParserConfigurationException, SAXException {
    HttpConnectionManagerParams managerParams = new HttpConnectionManagerParams();
    managerParams.setSoTimeout(timeout);
    managerParams.setConnectionTimeout(timeout);

    HttpConnectionManager manager = new SimpleHttpConnectionManager();
    manager.setParams(managerParams);

    HttpClientParams params = new HttpClientParams();
    params.setConnectionManagerTimeout(timeout);
    params.setSoTimeout(timeout);

    HttpClient client = new HttpClient(params, manager);
    HttpMethodRetryHandler retryHandler = new DefaultHttpMethodRetryHandler(1, true);
    client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler);

    if (!proxyHost.isEmpty()) {
        client.getHostConfiguration().setProxy(proxyHost, Integer.parseInt(proxyPort));
    }

    GetMethod get = new GetMethod(url);

    if (httpHeaders != null) {
        for (Map.Entry<String, String> header : httpHeaders.entrySet()) {
            get.setRequestHeader(header.getKey(), header.getValue());
        }
    }

    try {
        client.executeMethod(get);

        DocumentBuilder builder = domFactory.newDocumentBuilder();

        return builder.parse(get.getResponseBodyAsStream());
    } finally {
        get.releaseConnection();
    }
}

From source file:org.fao.geonet.notifier.MetadataNotifierClient.java

/**
 * Uses the notifier update service to handle insertion and updates of metadata.
 *
 * @param metadata/*from  w ww . j av  a  2 s .  c o m*/
 * @param metadataUuid
 * @throws MetadataNotifierClientException
 */
public void webUpdate(String serviceUrl, String username, String password, String metadata, String metadataUuid,
        GeonetContext gc) throws MetadataNotifierClientException {
    HttpClient client;

    Protocol.registerProtocol("https", new Protocol("https", new EasySSLProtocolSocketFactory(), 443));

    // Create a method instance.
    PostMethod method = new PostMethod(serviceUrl);

    // Provide custom retry handler is necessary
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));

    NameValuePair[] data = { new NameValuePair("action", "update"), new NameValuePair("uuid", metadataUuid),
            new NameValuePair("XMLFile", metadata) };

    method.setRequestBody(data);

    //RequestEntity requestEntity = new InputStreamRequestEntity(isoDocumentInputStream);

    //method.setRequestEntity(requestEntity);
    try {
        // Create an instance of HttpClient.
        client = new HttpClient();

        if (StringUtils.isNotBlank(username) && StringUtils.isNotBlank(password)) {
            System.out.println("webUpdate: SET USER");
            client.getState().setCredentials(AuthScope.ANY,
                    new UsernamePasswordCredentials(username, password));

            method.setDoAuthentication(true);
        }

        System.out.println("settingMan: " + (gc.getSettingManager() != null));
        if (gc.getSettingManager() != null)
            Lib.net.setupProxy(gc.getSettingManager(), client);

        // Execute the method.
        int statusCode = client.executeMethod(method);

        if (statusCode != HttpStatus.SC_OK) {
            throw new MetadataNotifierClientException("Method failed: " + method.getStatusLine());
        }

        // Read the response body.
        // byte[] responseBody = method.getResponseBody();

        // Deal with the response.
        // Use caution: ensure correct character encoding and is not binary data
        // System.out.println(new String(responseBody));

    } catch (HttpException e) {
        throw new MetadataNotifierClientException(e);
    } catch (IOException e) {
        throw new MetadataNotifierClientException(e);
    } finally {
        // Release the connection.
        method.releaseConnection();
        client = null;
    }
}

From source file:org.fao.geonet.notifier.MetadataNotifierClient.java

/**
 * Uses the notifier delete service to handle deletion of metadata.
 *
 * @param metadataUuid// w  ww  .  j  a v a  2 s.  c  o  m
 * @throws MetadataNotifierClientException
 */
public void webDelete(String serviceUrl, String username, String password, String metadataUuid,
        GeonetContext gc) throws MetadataNotifierClientException {
    HttpClient client;

    Protocol.registerProtocol("https", new Protocol("https", new EasySSLProtocolSocketFactory(), 443));

    // Create a method instance.
    PostMethod method = new PostMethod(serviceUrl);

    // Provide custom retry handler is necessary
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));

    try {
        // Create an instance of HttpClient.
        client = new HttpClient();

        if (StringUtils.isNotBlank(username) && StringUtils.isNotBlank(password)) {
            client.getState().setCredentials(AuthScope.ANY,
                    new UsernamePasswordCredentials(username, password));
            method.setDoAuthentication(true);
        }

        if (gc.getSettingManager() != null)
            Lib.net.setupProxy(gc.getSettingManager(), client);

        NameValuePair[] data = { new NameValuePair("action", "delete"), new NameValuePair("uuid", metadataUuid),
                new NameValuePair("XMLFile", "") };

        method.setRequestBody(data);

        //RequestEntity requestEntity = new StringRequestEntity(metadataUuid, null, null);

        //method.setRequestEntity(requestEntity);

        // Execute the method.
        int statusCode = client.executeMethod(method);

        if (statusCode != HttpStatus.SC_OK) {
            throw new MetadataNotifierClientException("Method failed: " + method.getStatusLine());
        }

        // Read the response body.
        // byte[] responseBody = method.getResponseBody();

        // Deal with the response.
        // Use caution: ensure correct character encoding and is not binary data
        // System.out.println(new String(responseBody));

    } catch (HttpException e) {
        throw new MetadataNotifierClientException(e);
    } catch (IOException e) {
        throw new MetadataNotifierClientException(e);
    } finally {
        // Release the connection.
        method.releaseConnection();
        client = null;
    }
}

From source file:org.geworkbench.engine.ccm.ComponentConfigurationManagerWindow2.java

private void findAvailableUpdatesFromTomcat() {

    installedRow = initialRow;/*from   w  w w .  j  ava  2  s . c  o m*/
    //      String url = System.getProperty("remote_components_config_url");
    //remote_components_config_url=http://califano11.cgc.cpmc.columbia.edu:8080/componentRepository/deploycomponents.txt

    GlobalPreferences prefs = GlobalPreferences.getInstance();
    String url = prefs.getRCM_URL().trim();
    if (url == null || url == "") {
        log.info("No Remote Component Manager URL configured.");
        return;
    }
    url += "/deploycomponents.txt";

    HttpClient client = new HttpClient();
    client.getHttpConnectionManager().getParams().setConnectionTimeout(3000);
    GetMethod method = new GetMethod(url);
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    method.setFollowRedirects(true);

    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            JOptionPane.showMessageDialog(null,
                    "No updates are available at this time.\nPlease try again later.",
                    "Remote Component Update", JOptionPane.PLAIN_MESSAGE);
            return;
        }

        String deploycomponents = method.getResponseBodyAsString();
        //         String[] rows = deploycomponents.split("\\r\\n");
        String[] folderNameVersions = deploycomponents.split("\\r\\n");

        for (int i = 0; i < ccmTableModel.getModelRowCount(); i++) {
            String localFolderName = ccmTableModel.getResourceFolder(i);

            String localVersion = ((String) ccmTableModel.getModelValueAt(i, CCMTableModel2.VERSION_INDEX));
            Double dLocalVersion = new Double(localVersion);

            for (int j = 0; j < folderNameVersions.length; j++) {
                String[] cols = folderNameVersions[j].split(",");
                String remoteFolderName = cols[0];
                String remoteVersion = cols[1];
                Double dRemoteVersion = new Double(remoteVersion);

                if (localFolderName.equalsIgnoreCase(remoteFolderName)) {
                    if (dRemoteVersion.compareTo(dLocalVersion) > 0) {
                        ccmTableModel.setModelValueAt(remoteVersion, i, CCMTableModel2.AVAILABLE_UPDATE_INDEX,
                                CCMTableModel2.NO_VALIDATION);
                    }
                }
            }
        }
    } catch (HttpException e) {
        JOptionPane.showMessageDialog(null, "No updates are available at this time.\nPlease try again later.",
                "Remote Component Update", JOptionPane.PLAIN_MESSAGE);
        //e.printStackTrace();
        return;
    } catch (IOException e) {
        JOptionPane.showMessageDialog(null,
                e.getMessage() + ".\n(" + e.getClass().getName() + ")\nPlease try again later.",
                "No Update Available", JOptionPane.PLAIN_MESSAGE);
        //e.printStackTrace();
        return;
    } catch (Exception e) { // IllegalArgumentException
        JOptionPane.showMessageDialog(null, e.getMessage() + ".", "No Update Available",
                JOptionPane.PLAIN_MESSAGE);
    }
}

From source file:org.hammer.santamaria.mapper.dataset.CKAN2DataSetInput.java

@SuppressWarnings("unchecked")
public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) {
    BSONObject dataset = new BasicBSONObject();
    dataset.put("datasource", datasource);
    dataset.put("id", id);

    String sId = EncodeURIComponent(id);
    LOG.info("---> id " + id + " - " + sId);

    HttpClient client = new HttpClient();
    client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false);

    LOG.info(/*from w ww  . j a v  a2 s  .  com*/
            "******************************************************************************************************");
    LOG.info(" ");
    LOG.info(url + PACKAGE_GET + sId);
    LOG.info(" ");
    LOG.info(
            "******************************************************************************************************");

    GetMethod method = new GetMethod(url + PACKAGE_GET + sId);

    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));

    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document result = Document.parse(new String(responseBody));

        if (result != null) {
            dataset.put("title", result.get("title"));
            dataset.put("author", result.get("author"));
            dataset.put("author_email", result.get("author_email"));
            dataset.put("license_id", result.get("license"));
        }

        ArrayList<String> tags = new ArrayList<String>();
        ArrayList<String> meta = new ArrayList<String>();
        ArrayList<String> other_tags = new ArrayList<String>();

        if (result.containsKey("author"))
            other_tags.add(result.get("author").toString());
        if (result.containsKey("title"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString()));
        if (result.containsKey("description"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString()));

        ArrayList<Document> resources = new ArrayList<Document>();
        if (result != null && result.containsKey("resources")) {
            resources = (ArrayList<Document>) result.get("resources");
            for (Document resource : resources) {
                if (resource.getString("format").toUpperCase().equals("JSON")) {
                    dataset.put("dataset-type", "JSON");
                    dataset.put("url", resource.get("url"));
                    dataset.put("created", resource.get("created"));
                    dataset.put("description", resource.get("description"));
                    dataset.put("revision_timestamp", resource.get("revision_timestamp"));
                    meta = DSSUtils.GetMetaByResource(resource.get("url").toString());

                }
            }
        }

        if (result != null && result.containsKey("tags")) {
            ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags");
            for (Document tag : tagsFromCKAN) {
                if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) {
                    tags.add(tag.getString("display_name").trim().toLowerCase());
                } else if (tag.containsKey("display_name")) {
                    tags.add(tag.getString("display_name").trim().toLowerCase());
                }
            }

        }

        dataset.put("tags", tags);
        dataset.put("meta", meta);
        dataset.put("resources", resources);
        dataset.put("other_tags", other_tags);

    } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }
    return dataset;
}

From source file:org.hammer.santamaria.mapper.dataset.CKANDataSetInput.java

@SuppressWarnings({ "unchecked", "rawtypes" })
public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) {
    BSONObject dataset = new BasicBSONObject();
    dataset.put("datasource", datasource);
    dataset.put("id", id);

    String sId = EncodeURIComponent(id);
    LOG.info("---> id " + id + " - " + sId);

    HttpClient client = new HttpClient();

    // some ckan site doesn't allow connection with hight timeout!!!
    // client.getHttpConnectionManager().getParams().setConnectionTimeout(3000);
    // client.getHttpConnectionManager().getParams().setSoTimeout(2000);
    //////*from ww w  .  j  a v a 2s. co  m*/

    // add to prevent redirect (?)
    client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false);

    LOG.info(
            "******************************************************************************************************");
    LOG.info(" ");
    LOG.info(url + PACKAGE_GET + sId);
    LOG.info(" ");
    LOG.info(
            "******************************************************************************************************");

    GetMethod method = new GetMethod(url + PACKAGE_GET + sId);

    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));

    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));

        if (doc != null && doc.containsKey("result") && doc.get("result") != null) {
            Document result = new Document();
            LOG.info(doc.get("result").getClass().toString());
            if (doc.get("result") instanceof Document) {
                LOG.info("!!! Document result !!!!");
                result = (Document) doc.get("result");
            } else if (doc.get("result") instanceof ArrayList) {
                LOG.info("!!! Document list !!!!");

                result = (Document) (((ArrayList) doc.get("result")).get(0));
            } else {
                LOG.info("!!! NOT FOUND !!!!");
                result = null;
            }
            LOG.info("result find!");
            if (result != null) {
                dataset.put("title", result.get("title"));
                dataset.put("author", result.get("author"));
                dataset.put("author_email", result.get("author_email"));
                dataset.put("license_id", result.get("license_id"));
            }

            ArrayList<String> tags = new ArrayList<String>();
            ArrayList<String> meta = new ArrayList<String>();
            ArrayList<String> other_tags = new ArrayList<String>();

            if (result.containsKey("author") && result.get("author") != null)
                other_tags.add(result.get("author").toString());
            if (result.containsKey("title") && result.get("title") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString()));
            if (result.containsKey("description") && result.get("description") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString()));
            if (result.containsKey("notes") && result.get("notes") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("notes").toString()));

            ArrayList<Document> resources = new ArrayList<Document>();
            if (result != null && result.containsKey("resources")) {
                resources = (ArrayList<Document>) result.get("resources");
                for (Document resource : resources) {
                    if (resource.getString("format").toUpperCase().equals("JSON")) {
                        dataset.put("dataset-type", "JSON");
                        dataset.put("url", resource.get("url"));
                        dataset.put("created", resource.get("created"));
                        dataset.put("description", resource.get("description"));
                        dataset.put("revision_timestamp", resource.get("revision_timestamp"));
                        meta = DSSUtils.GetMetaByResource(resource.get("url").toString());
                    }
                }
            }

            if (result != null && result.containsKey("tags")) {
                ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags");
                for (Document tag : tagsFromCKAN) {
                    if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    } else if (tag.containsKey("display_name")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    }
                }

            }

            dataset.put("tags", tags);
            dataset.put("meta", meta);
            dataset.put("resources", resources);
            dataset.put("other_tags", other_tags);

        }
    } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }
    return dataset;
}

From source file:org.hammer.santamaria.mapper.dataset.CKANDataSetInput.java

@SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] pArgs) throws Exception {
    String id = "proportion-of-children-under-5-years-who-have-ever-breastfed-by-county-xls-2005-6";
    String sId = EncodeURIComponent(id);
    String url = "https://africaopendata.org/api/action";

    BSONObject dataset = new BasicBSONObject();
    dataset.put("datasource", "Test");
    dataset.put("id", id);

    LOG.info("---> id " + id + " - " + sId);

    HttpClient client = new HttpClient();
    client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false);
    LOG.info(/* w w  w  .  j a  va 2 s  .  co  m*/
            "******************************************************************************************************");
    LOG.info(" ");
    LOG.info(url + PACKAGE_GET + sId);
    LOG.info(" ");
    LOG.info(
            "******************************************************************************************************");

    GetMethod method = new GetMethod(url + PACKAGE_GET + sId);

    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));

    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));

        if (doc != null && doc.containsKey("result")) {
            Document result = new Document();
            LOG.info(doc.get("result").getClass().toString());
            if (doc.get("result") instanceof Document) {
                LOG.info("!!! Document result !!!!");
                result = (Document) doc.get("result");
            } else if (doc.get("result") instanceof ArrayList) {
                LOG.info("!!! Document list !!!!");

                result = (Document) (((ArrayList) doc.get("result")).get(0));
            } else {
                LOG.info("!!! NOT FOUND !!!!");
                result = null;
            }
            LOG.info("result find!");
            if (result != null) {
                dataset.put("title", result.get("title"));
                dataset.put("author", result.get("author"));
                dataset.put("author_email", result.get("author_email"));
                dataset.put("license_id", result.get("license_id"));
            }

            ArrayList<String> tags = new ArrayList<String>();
            ArrayList<String> meta = new ArrayList<String>();
            ArrayList<String> other_tags = new ArrayList<String>();

            if (result.containsKey("author") && result.get("author") != null)
                other_tags.add(result.get("author").toString());
            if (result.containsKey("title") && result.get("title") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString()));
            if (result.containsKey("description") && result.get("description") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString()));

            ArrayList<Document> resources = new ArrayList<Document>();
            if (result != null && result.containsKey("resources")) {
                resources = (ArrayList<Document>) result.get("resources");
                for (Document resource : resources) {
                    if (resource.getString("format").toUpperCase().equals("JSON")) {
                        dataset.put("dataset-type", "JSON");
                        dataset.put("url", resource.get("url"));
                        dataset.put("created", resource.get("created"));
                        dataset.put("description", resource.get("description"));
                        dataset.put("revision_timestamp", resource.get("revision_timestamp"));
                        meta = DSSUtils.GetMetaByResource(resource.get("url").toString());
                    }
                }
            }

            if (result != null && result.containsKey("tags")) {
                ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags");
                for (Document tag : tagsFromCKAN) {
                    if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    } else if (tag.containsKey("display_name")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    }
                }

            }

            dataset.put("tags", tags);
            dataset.put("meta", meta);
            dataset.put("resources", resources);
            dataset.put("other_tags", other_tags);

        }

    } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }

    //GetMetaByDocument("http://catalog.data.gov/api/action/package_show?id=1e68f387-5f1c-46c0-a0d1-46044ffef5bf");
}

From source file:org.hammer.santamaria.mapper.dataset.INPSDataSetInput.java

@SuppressWarnings("unchecked")
public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) {
    BSONObject dataset = new BasicBSONObject();

    dataset.put("datasource", datasource);
    dataset.put("id", id);

    LOG.info(datasource + id);//ww w  .ja v  a  2s.co m

    HttpClient client = new HttpClient();
    GetMethod method = new GetMethod(url);
    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");

    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));
        dataset.put("title", doc.getString("name"));
        dataset.put("author", doc.getString("author"));
        dataset.put("author_email", doc.getString("author_email"));
        dataset.put("license_id", doc.getString("license_id"));
        dataset.put("description", doc.getString("notes"));

        ArrayList<String> tags = new ArrayList<String>();
        ArrayList<String> meta = new ArrayList<String>();
        ArrayList<String> other_tags = new ArrayList<String>();

        if (doc.containsKey("author"))
            other_tags.add(doc.get("author").toString());
        if (doc.containsKey("title"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("name").toString()));
        if (doc.containsKey("notes"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("notes").toString()));

        ArrayList<Document> resources = (ArrayList<Document>) doc.get("resources");
        for (Document resource : resources) {
            if (resource.getString("format").toUpperCase().equals("JSON")) {
                dataset.put("dataset-type", "JSON");
                dataset.put("url", resource.get("url"));
                dataset.put("created", resource.get("created"));
                dataset.put("revision_timestamp", resource.get("last_modified"));

                meta = this.getMetaByDocument(resource.get("url").toString());

            }
        }

        tags = (ArrayList<String>) doc.get("tags");

        dataset.put("tags", tags);
        dataset.put("meta", meta);
        dataset.put("resources", resources);
        dataset.put("other_tags", other_tags);

    } catch (Exception e) {
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }
    LOG.debug(dataset.get("title"));

    return dataset;
}

From source file:org.hammer.santamaria.mapper.dataset.INPSDataSetInput.java

/**
 * Return meta from document for CKAN implementation
 * /*from w  w w  .j  a  va  2 s .  co m*/
 * @param url
 * @return
 */
public ArrayList<String> getMetaByDocument(String url) {
    ArrayList<String> meta = new ArrayList<String>();
    HttpClient client = new HttpClient();
    GetMethod method = new GetMethod(url);
    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");

    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();

        if ((JSON.parse(new String(responseBody))) instanceof BasicBSONList) {
            BasicBSONList pList = (BasicBSONList) JSON.parse(new String(responseBody));
            if (pList.size() > 0 && pList.get(0) != null) {
                BSONObject temp = (BSONObject) pList.get(0);
                for (String metaKey : temp.keySet()) {
                    if (!meta.contains(metaKey.toLowerCase())) {
                        meta.add(metaKey.toLowerCase());
                    }
                }
            }
        } else if ((JSON.parse(new String(responseBody))) instanceof BasicDBObject) {
            BasicDBObject temp = (BasicDBObject) JSON.parse(new String(responseBody));
            for (String metaKey : temp.keySet()) {
                if (!meta.contains(metaKey.toLowerCase())) {
                    meta.add(metaKey.toLowerCase());
                }
            }

        }

    } catch (Exception e) {
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }

    return meta;
}