Example usage for org.apache.commons.httpclient.params HttpMethodParams USER_AGENT

List of usage examples for org.apache.commons.httpclient.params HttpMethodParams USER_AGENT

Introduction

In this page you can find the example usage for org.apache.commons.httpclient.params HttpMethodParams USER_AGENT.

Prototype

String USER_AGENT

To view the source code for org.apache.commons.httpclient.params HttpMethodParams USER_AGENT.

Click Source Link

Usage

From source file:org.colombbus.tangara.net.TConnection.java

/**
 * Create a new connection//from   www . j  a va 2s. com
 *
 * @param userAgent
 *            the user-agent of the HTTP header
 * @param charset
 *            the character set encoding of the HTTP communication
 * @param serverURL
 *            the address of the server as an URL
 * @param retryCount
 * @param timeout
 */
public TConnection(String userAgent, String charset, String serverURL, int retryCount, int timeout) {
    super();
    this.serverURL = serverURL;
    this.encodingCharset = charset;
    this.retryHandler = new HttpConnRetryHandler(retryCount);
    params.setParameter(HttpMethodParams.USER_AGENT, userAgent);
    params.setContentCharset(charset);
    params.setConnectionManagerTimeout(timeout);
    params.setSoTimeout(timeout);
    params.setParameter(HttpMethodParams.RETRY_HANDLER, this.retryHandler);
}

From source file:org.eclipse.epp.internal.mpc.core.service.DefaultMarketplaceService.java

public void reportInstallError(IProgressMonitor monitor, IStatus result, Set<Node> nodes,
        Set<String> iuIdsAndVersions, String resolutionDetails) throws CoreException {
    HttpClient client = new HttpClient();
    client.getParams().setParameter(HttpMethodParams.USER_AGENT, MarketplaceClientCore.BUNDLE_ID);

    URL location;/*w  w  w.  j a  va2  s.  c om*/
    PostMethod method;
    try {
        location = new URL(baseUrl, "install/error/report"); //$NON-NLS-1$
        method = new PostMethod(location.toURI().toString());
    } catch (URISyntaxException e) {
        throw new IllegalStateException(e);
    } catch (MalformedURLException e) {
        throw new IllegalStateException(e);
    }
    try {
        List<NameValuePair> parameters = new ArrayList<NameValuePair>();
        parameters.add(new NameValuePair("status", Integer.toString(result.getSeverity()))); //$NON-NLS-1$
        parameters.add(new NameValuePair("statusMessage", result.getMessage())); //$NON-NLS-1$
        for (Node node : nodes) {
            parameters.add(new NameValuePair("node", node.getId())); //$NON-NLS-1$
        }
        if (iuIdsAndVersions != null && !iuIdsAndVersions.isEmpty()) {
            for (String iuAndVersion : iuIdsAndVersions) {
                parameters.add(new NameValuePair("iu", iuAndVersion)); //$NON-NLS-1$
            }
        }
        parameters.add(new NameValuePair("detailedMessage", resolutionDetails)); //$NON-NLS-1$
        if (!parameters.isEmpty()) {
            method.setRequestBody(parameters.toArray(new NameValuePair[parameters.size()]));
            client.executeMethod(method);
        }
    } catch (IOException e) {
        String message = NLS.bind(Messages.DefaultMarketplaceService_cannotCompleteRequest_reason,
                location.toString(), e.getMessage());
        throw new CoreException(createErrorStatus(message, e));
    } finally {
        method.releaseConnection();
    }
}

From source file:org.eclipse.mylyn.commons.net.WebUtil.java

/**
 * @since 3.0/*from w  w  w .ja v a 2 s  .c  o  m*/
 */
public static void configureHttpClient(HttpClient client, String userAgent) {
    client.getParams().setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, true);
    client.getParams().setParameter(HttpMethodParams.USER_AGENT, getUserAgent(userAgent));
    client.getParams().setConnectionManagerTimeout(CONNECTION_TIMEOUT_INTERVAL);
    // TODO consider setting this as the default
    //client.getParams().setCookiePolicy(CookiePolicy.RFC_2109);
    configureHttpClientConnectionManager(client);
}

From source file:org.eclipse.mylyn.commons.tests.net.WebUtilTest.java

public void testConfigureHttpClient() {
    HttpClient client = new HttpClient();

    WebUtil.configureHttpClient(client, "");
    assertEquals(WebUtil.getUserAgent(""), client.getParams().getParameter(HttpMethodParams.USER_AGENT));

    WebUtil.configureHttpClient(client, null);
    assertEquals(WebUtil.getUserAgent(""), client.getParams().getParameter(HttpMethodParams.USER_AGENT));

    WebUtil.configureHttpClient(client, "myagent");
    assertTrue(//from  w  w w . j a v  a  2s .  com
            -1 != client.getParams().getParameter(HttpMethodParams.USER_AGENT).toString().indexOf("myagent"));

    // TODO test timeouts
}

From source file:org.hammer.santamaria.mapper.dataset.CKAN2DataSetInput.java

@SuppressWarnings("unchecked")
public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) {
    BSONObject dataset = new BasicBSONObject();
    dataset.put("datasource", datasource);
    dataset.put("id", id);

    String sId = EncodeURIComponent(id);
    LOG.info("---> id " + id + " - " + sId);

    HttpClient client = new HttpClient();
    client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false);

    LOG.info(//ww  w  . jav  a2  s  . c om
            "******************************************************************************************************");
    LOG.info(" ");
    LOG.info(url + PACKAGE_GET + sId);
    LOG.info(" ");
    LOG.info(
            "******************************************************************************************************");

    GetMethod method = new GetMethod(url + PACKAGE_GET + sId);

    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));

    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document result = Document.parse(new String(responseBody));

        if (result != null) {
            dataset.put("title", result.get("title"));
            dataset.put("author", result.get("author"));
            dataset.put("author_email", result.get("author_email"));
            dataset.put("license_id", result.get("license"));
        }

        ArrayList<String> tags = new ArrayList<String>();
        ArrayList<String> meta = new ArrayList<String>();
        ArrayList<String> other_tags = new ArrayList<String>();

        if (result.containsKey("author"))
            other_tags.add(result.get("author").toString());
        if (result.containsKey("title"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString()));
        if (result.containsKey("description"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString()));

        ArrayList<Document> resources = new ArrayList<Document>();
        if (result != null && result.containsKey("resources")) {
            resources = (ArrayList<Document>) result.get("resources");
            for (Document resource : resources) {
                if (resource.getString("format").toUpperCase().equals("JSON")) {
                    dataset.put("dataset-type", "JSON");
                    dataset.put("url", resource.get("url"));
                    dataset.put("created", resource.get("created"));
                    dataset.put("description", resource.get("description"));
                    dataset.put("revision_timestamp", resource.get("revision_timestamp"));
                    meta = DSSUtils.GetMetaByResource(resource.get("url").toString());

                }
            }
        }

        if (result != null && result.containsKey("tags")) {
            ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags");
            for (Document tag : tagsFromCKAN) {
                if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) {
                    tags.add(tag.getString("display_name").trim().toLowerCase());
                } else if (tag.containsKey("display_name")) {
                    tags.add(tag.getString("display_name").trim().toLowerCase());
                }
            }

        }

        dataset.put("tags", tags);
        dataset.put("meta", meta);
        dataset.put("resources", resources);
        dataset.put("other_tags", other_tags);

    } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }
    return dataset;
}

From source file:org.hammer.santamaria.mapper.dataset.CKANDataSetInput.java

@SuppressWarnings({ "unchecked", "rawtypes" })
public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) {
    BSONObject dataset = new BasicBSONObject();
    dataset.put("datasource", datasource);
    dataset.put("id", id);

    String sId = EncodeURIComponent(id);
    LOG.info("---> id " + id + " - " + sId);

    HttpClient client = new HttpClient();

    // some ckan site doesn't allow connection with hight timeout!!!
    // client.getHttpConnectionManager().getParams().setConnectionTimeout(3000);
    // client.getHttpConnectionManager().getParams().setSoTimeout(2000);
    ///////  w  w  w  .  j  a  va2  s . c  o  m

    // add to prevent redirect (?)
    client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false);

    LOG.info(
            "******************************************************************************************************");
    LOG.info(" ");
    LOG.info(url + PACKAGE_GET + sId);
    LOG.info(" ");
    LOG.info(
            "******************************************************************************************************");

    GetMethod method = new GetMethod(url + PACKAGE_GET + sId);

    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));

    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));

        if (doc != null && doc.containsKey("result") && doc.get("result") != null) {
            Document result = new Document();
            LOG.info(doc.get("result").getClass().toString());
            if (doc.get("result") instanceof Document) {
                LOG.info("!!! Document result !!!!");
                result = (Document) doc.get("result");
            } else if (doc.get("result") instanceof ArrayList) {
                LOG.info("!!! Document list !!!!");

                result = (Document) (((ArrayList) doc.get("result")).get(0));
            } else {
                LOG.info("!!! NOT FOUND !!!!");
                result = null;
            }
            LOG.info("result find!");
            if (result != null) {
                dataset.put("title", result.get("title"));
                dataset.put("author", result.get("author"));
                dataset.put("author_email", result.get("author_email"));
                dataset.put("license_id", result.get("license_id"));
            }

            ArrayList<String> tags = new ArrayList<String>();
            ArrayList<String> meta = new ArrayList<String>();
            ArrayList<String> other_tags = new ArrayList<String>();

            if (result.containsKey("author") && result.get("author") != null)
                other_tags.add(result.get("author").toString());
            if (result.containsKey("title") && result.get("title") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString()));
            if (result.containsKey("description") && result.get("description") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString()));
            if (result.containsKey("notes") && result.get("notes") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("notes").toString()));

            ArrayList<Document> resources = new ArrayList<Document>();
            if (result != null && result.containsKey("resources")) {
                resources = (ArrayList<Document>) result.get("resources");
                for (Document resource : resources) {
                    if (resource.getString("format").toUpperCase().equals("JSON")) {
                        dataset.put("dataset-type", "JSON");
                        dataset.put("url", resource.get("url"));
                        dataset.put("created", resource.get("created"));
                        dataset.put("description", resource.get("description"));
                        dataset.put("revision_timestamp", resource.get("revision_timestamp"));
                        meta = DSSUtils.GetMetaByResource(resource.get("url").toString());
                    }
                }
            }

            if (result != null && result.containsKey("tags")) {
                ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags");
                for (Document tag : tagsFromCKAN) {
                    if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    } else if (tag.containsKey("display_name")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    }
                }

            }

            dataset.put("tags", tags);
            dataset.put("meta", meta);
            dataset.put("resources", resources);
            dataset.put("other_tags", other_tags);

        }
    } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }
    return dataset;
}

From source file:org.hammer.santamaria.mapper.dataset.CKANDataSetInput.java

@SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] pArgs) throws Exception {
    String id = "proportion-of-children-under-5-years-who-have-ever-breastfed-by-county-xls-2005-6";
    String sId = EncodeURIComponent(id);
    String url = "https://africaopendata.org/api/action";

    BSONObject dataset = new BasicBSONObject();
    dataset.put("datasource", "Test");
    dataset.put("id", id);

    LOG.info("---> id " + id + " - " + sId);

    HttpClient client = new HttpClient();
    client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false);
    LOG.info(/* w  w w  . java 2  s  .c om*/
            "******************************************************************************************************");
    LOG.info(" ");
    LOG.info(url + PACKAGE_GET + sId);
    LOG.info(" ");
    LOG.info(
            "******************************************************************************************************");

    GetMethod method = new GetMethod(url + PACKAGE_GET + sId);

    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));

    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));

        if (doc != null && doc.containsKey("result")) {
            Document result = new Document();
            LOG.info(doc.get("result").getClass().toString());
            if (doc.get("result") instanceof Document) {
                LOG.info("!!! Document result !!!!");
                result = (Document) doc.get("result");
            } else if (doc.get("result") instanceof ArrayList) {
                LOG.info("!!! Document list !!!!");

                result = (Document) (((ArrayList) doc.get("result")).get(0));
            } else {
                LOG.info("!!! NOT FOUND !!!!");
                result = null;
            }
            LOG.info("result find!");
            if (result != null) {
                dataset.put("title", result.get("title"));
                dataset.put("author", result.get("author"));
                dataset.put("author_email", result.get("author_email"));
                dataset.put("license_id", result.get("license_id"));
            }

            ArrayList<String> tags = new ArrayList<String>();
            ArrayList<String> meta = new ArrayList<String>();
            ArrayList<String> other_tags = new ArrayList<String>();

            if (result.containsKey("author") && result.get("author") != null)
                other_tags.add(result.get("author").toString());
            if (result.containsKey("title") && result.get("title") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString()));
            if (result.containsKey("description") && result.get("description") != null)
                other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString()));

            ArrayList<Document> resources = new ArrayList<Document>();
            if (result != null && result.containsKey("resources")) {
                resources = (ArrayList<Document>) result.get("resources");
                for (Document resource : resources) {
                    if (resource.getString("format").toUpperCase().equals("JSON")) {
                        dataset.put("dataset-type", "JSON");
                        dataset.put("url", resource.get("url"));
                        dataset.put("created", resource.get("created"));
                        dataset.put("description", resource.get("description"));
                        dataset.put("revision_timestamp", resource.get("revision_timestamp"));
                        meta = DSSUtils.GetMetaByResource(resource.get("url").toString());
                    }
                }
            }

            if (result != null && result.containsKey("tags")) {
                ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags");
                for (Document tag : tagsFromCKAN) {
                    if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    } else if (tag.containsKey("display_name")) {
                        tags.add(tag.getString("display_name").trim().toLowerCase());
                    }
                }

            }

            dataset.put("tags", tags);
            dataset.put("meta", meta);
            dataset.put("resources", resources);
            dataset.put("other_tags", other_tags);

        }

    } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }

    //GetMetaByDocument("http://catalog.data.gov/api/action/package_show?id=1e68f387-5f1c-46c0-a0d1-46044ffef5bf");
}

From source file:org.hammer.santamaria.mapper.dataset.INPSDataSetInput.java

@SuppressWarnings("unchecked")
public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) {
    BSONObject dataset = new BasicBSONObject();

    dataset.put("datasource", datasource);
    dataset.put("id", id);

    LOG.info(datasource + id);//from   www.  j  a v  a  2s  . c  om

    HttpClient client = new HttpClient();
    GetMethod method = new GetMethod(url);
    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");

    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));
        dataset.put("title", doc.getString("name"));
        dataset.put("author", doc.getString("author"));
        dataset.put("author_email", doc.getString("author_email"));
        dataset.put("license_id", doc.getString("license_id"));
        dataset.put("description", doc.getString("notes"));

        ArrayList<String> tags = new ArrayList<String>();
        ArrayList<String> meta = new ArrayList<String>();
        ArrayList<String> other_tags = new ArrayList<String>();

        if (doc.containsKey("author"))
            other_tags.add(doc.get("author").toString());
        if (doc.containsKey("title"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("name").toString()));
        if (doc.containsKey("notes"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("notes").toString()));

        ArrayList<Document> resources = (ArrayList<Document>) doc.get("resources");
        for (Document resource : resources) {
            if (resource.getString("format").toUpperCase().equals("JSON")) {
                dataset.put("dataset-type", "JSON");
                dataset.put("url", resource.get("url"));
                dataset.put("created", resource.get("created"));
                dataset.put("revision_timestamp", resource.get("last_modified"));

                meta = this.getMetaByDocument(resource.get("url").toString());

            }
        }

        tags = (ArrayList<String>) doc.get("tags");

        dataset.put("tags", tags);
        dataset.put("meta", meta);
        dataset.put("resources", resources);
        dataset.put("other_tags", other_tags);

    } catch (Exception e) {
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }
    LOG.debug(dataset.get("title"));

    return dataset;
}

From source file:org.hammer.santamaria.mapper.dataset.INPSDataSetInput.java

/**
 * Return meta from document for CKAN implementation
 * //from www .  j  a v  a 2s.co m
 * @param url
 * @return
 */
public ArrayList<String> getMetaByDocument(String url) {
    ArrayList<String> meta = new ArrayList<String>();
    HttpClient client = new HttpClient();
    GetMethod method = new GetMethod(url);
    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");

    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();

        if ((JSON.parse(new String(responseBody))) instanceof BasicBSONList) {
            BasicBSONList pList = (BasicBSONList) JSON.parse(new String(responseBody));
            if (pList.size() > 0 && pList.get(0) != null) {
                BSONObject temp = (BSONObject) pList.get(0);
                for (String metaKey : temp.keySet()) {
                    if (!meta.contains(metaKey.toLowerCase())) {
                        meta.add(metaKey.toLowerCase());
                    }
                }
            }
        } else if ((JSON.parse(new String(responseBody))) instanceof BasicDBObject) {
            BasicDBObject temp = (BasicDBObject) JSON.parse(new String(responseBody));
            for (String metaKey : temp.keySet()) {
                if (!meta.contains(metaKey.toLowerCase())) {
                    meta.add(metaKey.toLowerCase());
                }
            }

        }

    } catch (Exception e) {
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }

    return meta;
}

From source file:org.hammer.santamaria.mapper.dataset.StatPortalDataSetInput.java

@SuppressWarnings("unchecked")
public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) {
    BSONObject dataset = new BasicBSONObject();

    dataset.put("datasource", datasource);
    dataset.put("id", id);

    LOG.info(datasource + id);/*from   w w w .j av a2 s .  co  m*/

    HttpClient client = new HttpClient();
    GetMethod method = new GetMethod(url);
    method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
    method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");

    method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
            new DefaultHttpMethodRetryHandler(3, false));
    try {
        int statusCode = client.executeMethod(method);
        if (statusCode != HttpStatus.SC_OK) {
            throw new Exception("Method failed: " + method.getStatusLine());
        }
        byte[] responseBody = method.getResponseBody();
        LOG.debug(new String(responseBody));
        Document doc = Document.parse(new String(responseBody));
        dataset.put("title", doc.getString("title"));
        dataset.put("author", doc.getString("author"));
        dataset.put("author_email", doc.getString("author_email"));
        dataset.put("license_id", doc.getString("license_id"));
        dataset.put("description", doc.getString("notes"));

        ArrayList<String> tags = new ArrayList<String>();
        ArrayList<String> meta = new ArrayList<String>();
        ArrayList<String> other_tags = new ArrayList<String>();

        if (doc.containsKey("author"))
            other_tags.add(doc.get("author").toString());
        if (doc.containsKey("title"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("title").toString()));
        if (doc.containsKey("notes"))
            other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("notes").toString()));

        ArrayList<Document> resources = (ArrayList<Document>) doc.get("resources");
        for (Document resource : resources) {
            if (resource.getString("format").toUpperCase().equals("JSON")) {
                dataset.put("dataset-type", "JSON");
                dataset.put("url", resource.get("url"));
                dataset.put("created", resource.get("created"));
                dataset.put("revision_timestamp", resource.get("last_modified"));

                meta = this.getMetaByDocument(resource.get("url").toString());

            }
        }

        tags = (ArrayList<String>) doc.get("tags");

        dataset.put("tags", tags);
        dataset.put("meta", meta);
        dataset.put("resources", resources);
        dataset.put("other_tags", other_tags);

    } catch (Exception e) {
        LOG.error(e);
    } finally {
        method.releaseConnection();
    }
    LOG.debug(dataset.get("title"));

    return dataset;
}