List of usage examples for org.apache.commons.httpclient.params HttpMethodParams USER_AGENT
String USER_AGENT
To view the source code for org.apache.commons.httpclient.params HttpMethodParams USER_AGENT.
Click Source Link
From source file:org.colombbus.tangara.net.TConnection.java
/** * Create a new connection//from www . j a va 2s. com * * @param userAgent * the user-agent of the HTTP header * @param charset * the character set encoding of the HTTP communication * @param serverURL * the address of the server as an URL * @param retryCount * @param timeout */ public TConnection(String userAgent, String charset, String serverURL, int retryCount, int timeout) { super(); this.serverURL = serverURL; this.encodingCharset = charset; this.retryHandler = new HttpConnRetryHandler(retryCount); params.setParameter(HttpMethodParams.USER_AGENT, userAgent); params.setContentCharset(charset); params.setConnectionManagerTimeout(timeout); params.setSoTimeout(timeout); params.setParameter(HttpMethodParams.RETRY_HANDLER, this.retryHandler); }
From source file:org.eclipse.epp.internal.mpc.core.service.DefaultMarketplaceService.java
public void reportInstallError(IProgressMonitor monitor, IStatus result, Set<Node> nodes, Set<String> iuIdsAndVersions, String resolutionDetails) throws CoreException { HttpClient client = new HttpClient(); client.getParams().setParameter(HttpMethodParams.USER_AGENT, MarketplaceClientCore.BUNDLE_ID); URL location;/*w w w. j a va2 s. c om*/ PostMethod method; try { location = new URL(baseUrl, "install/error/report"); //$NON-NLS-1$ method = new PostMethod(location.toURI().toString()); } catch (URISyntaxException e) { throw new IllegalStateException(e); } catch (MalformedURLException e) { throw new IllegalStateException(e); } try { List<NameValuePair> parameters = new ArrayList<NameValuePair>(); parameters.add(new NameValuePair("status", Integer.toString(result.getSeverity()))); //$NON-NLS-1$ parameters.add(new NameValuePair("statusMessage", result.getMessage())); //$NON-NLS-1$ for (Node node : nodes) { parameters.add(new NameValuePair("node", node.getId())); //$NON-NLS-1$ } if (iuIdsAndVersions != null && !iuIdsAndVersions.isEmpty()) { for (String iuAndVersion : iuIdsAndVersions) { parameters.add(new NameValuePair("iu", iuAndVersion)); //$NON-NLS-1$ } } parameters.add(new NameValuePair("detailedMessage", resolutionDetails)); //$NON-NLS-1$ if (!parameters.isEmpty()) { method.setRequestBody(parameters.toArray(new NameValuePair[parameters.size()])); client.executeMethod(method); } } catch (IOException e) { String message = NLS.bind(Messages.DefaultMarketplaceService_cannotCompleteRequest_reason, location.toString(), e.getMessage()); throw new CoreException(createErrorStatus(message, e)); } finally { method.releaseConnection(); } }
From source file:org.eclipse.mylyn.commons.net.WebUtil.java
/** * @since 3.0/*from w w w .ja v a 2 s .c o m*/ */ public static void configureHttpClient(HttpClient client, String userAgent) { client.getParams().setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, true); client.getParams().setParameter(HttpMethodParams.USER_AGENT, getUserAgent(userAgent)); client.getParams().setConnectionManagerTimeout(CONNECTION_TIMEOUT_INTERVAL); // TODO consider setting this as the default //client.getParams().setCookiePolicy(CookiePolicy.RFC_2109); configureHttpClientConnectionManager(client); }
From source file:org.eclipse.mylyn.commons.tests.net.WebUtilTest.java
public void testConfigureHttpClient() { HttpClient client = new HttpClient(); WebUtil.configureHttpClient(client, ""); assertEquals(WebUtil.getUserAgent(""), client.getParams().getParameter(HttpMethodParams.USER_AGENT)); WebUtil.configureHttpClient(client, null); assertEquals(WebUtil.getUserAgent(""), client.getParams().getParameter(HttpMethodParams.USER_AGENT)); WebUtil.configureHttpClient(client, "myagent"); assertTrue(//from w w w . j a v a 2s . com -1 != client.getParams().getParameter(HttpMethodParams.USER_AGENT).toString().indexOf("myagent")); // TODO test timeouts }
From source file:org.hammer.santamaria.mapper.dataset.CKAN2DataSetInput.java
@SuppressWarnings("unchecked") public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) { BSONObject dataset = new BasicBSONObject(); dataset.put("datasource", datasource); dataset.put("id", id); String sId = EncodeURIComponent(id); LOG.info("---> id " + id + " - " + sId); HttpClient client = new HttpClient(); client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false); LOG.info(//ww w . jav a2 s . c om "******************************************************************************************************"); LOG.info(" "); LOG.info(url + PACKAGE_GET + sId); LOG.info(" "); LOG.info( "******************************************************************************************************"); GetMethod method = new GetMethod(url + PACKAGE_GET + sId); method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { throw new Exception("Method failed: " + method.getStatusLine()); } byte[] responseBody = method.getResponseBody(); LOG.debug(new String(responseBody)); Document result = Document.parse(new String(responseBody)); if (result != null) { dataset.put("title", result.get("title")); dataset.put("author", result.get("author")); dataset.put("author_email", result.get("author_email")); dataset.put("license_id", result.get("license")); } ArrayList<String> tags = new ArrayList<String>(); ArrayList<String> meta = new ArrayList<String>(); ArrayList<String> other_tags = new ArrayList<String>(); if (result.containsKey("author")) other_tags.add(result.get("author").toString()); if (result.containsKey("title")) other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString())); if (result.containsKey("description")) other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString())); ArrayList<Document> resources = new ArrayList<Document>(); if (result != null && result.containsKey("resources")) { resources = (ArrayList<Document>) result.get("resources"); for (Document resource : resources) { if (resource.getString("format").toUpperCase().equals("JSON")) { dataset.put("dataset-type", "JSON"); dataset.put("url", resource.get("url")); dataset.put("created", resource.get("created")); dataset.put("description", resource.get("description")); dataset.put("revision_timestamp", resource.get("revision_timestamp")); meta = DSSUtils.GetMetaByResource(resource.get("url").toString()); } } } if (result != null && result.containsKey("tags")) { ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags"); for (Document tag : tagsFromCKAN) { if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) { tags.add(tag.getString("display_name").trim().toLowerCase()); } else if (tag.containsKey("display_name")) { tags.add(tag.getString("display_name").trim().toLowerCase()); } } } dataset.put("tags", tags); dataset.put("meta", meta); dataset.put("resources", resources); dataset.put("other_tags", other_tags); } catch (Exception e) { e.printStackTrace(); LOG.error(e); } finally { method.releaseConnection(); } return dataset; }
From source file:org.hammer.santamaria.mapper.dataset.CKANDataSetInput.java
@SuppressWarnings({ "unchecked", "rawtypes" }) public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) { BSONObject dataset = new BasicBSONObject(); dataset.put("datasource", datasource); dataset.put("id", id); String sId = EncodeURIComponent(id); LOG.info("---> id " + id + " - " + sId); HttpClient client = new HttpClient(); // some ckan site doesn't allow connection with hight timeout!!! // client.getHttpConnectionManager().getParams().setConnectionTimeout(3000); // client.getHttpConnectionManager().getParams().setSoTimeout(2000); /////// w w w . j a va2 s . c o m // add to prevent redirect (?) client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false); LOG.info( "******************************************************************************************************"); LOG.info(" "); LOG.info(url + PACKAGE_GET + sId); LOG.info(" "); LOG.info( "******************************************************************************************************"); GetMethod method = new GetMethod(url + PACKAGE_GET + sId); method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { throw new Exception("Method failed: " + method.getStatusLine()); } byte[] responseBody = method.getResponseBody(); LOG.debug(new String(responseBody)); Document doc = Document.parse(new String(responseBody)); if (doc != null && doc.containsKey("result") && doc.get("result") != null) { Document result = new Document(); LOG.info(doc.get("result").getClass().toString()); if (doc.get("result") instanceof Document) { LOG.info("!!! Document result !!!!"); result = (Document) doc.get("result"); } else if (doc.get("result") instanceof ArrayList) { LOG.info("!!! Document list !!!!"); result = (Document) (((ArrayList) doc.get("result")).get(0)); } else { LOG.info("!!! NOT FOUND !!!!"); result = null; } LOG.info("result find!"); if (result != null) { dataset.put("title", result.get("title")); dataset.put("author", result.get("author")); dataset.put("author_email", result.get("author_email")); dataset.put("license_id", result.get("license_id")); } ArrayList<String> tags = new ArrayList<String>(); ArrayList<String> meta = new ArrayList<String>(); ArrayList<String> other_tags = new ArrayList<String>(); if (result.containsKey("author") && result.get("author") != null) other_tags.add(result.get("author").toString()); if (result.containsKey("title") && result.get("title") != null) other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString())); if (result.containsKey("description") && result.get("description") != null) other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString())); if (result.containsKey("notes") && result.get("notes") != null) other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("notes").toString())); ArrayList<Document> resources = new ArrayList<Document>(); if (result != null && result.containsKey("resources")) { resources = (ArrayList<Document>) result.get("resources"); for (Document resource : resources) { if (resource.getString("format").toUpperCase().equals("JSON")) { dataset.put("dataset-type", "JSON"); dataset.put("url", resource.get("url")); dataset.put("created", resource.get("created")); dataset.put("description", resource.get("description")); dataset.put("revision_timestamp", resource.get("revision_timestamp")); meta = DSSUtils.GetMetaByResource(resource.get("url").toString()); } } } if (result != null && result.containsKey("tags")) { ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags"); for (Document tag : tagsFromCKAN) { if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) { tags.add(tag.getString("display_name").trim().toLowerCase()); } else if (tag.containsKey("display_name")) { tags.add(tag.getString("display_name").trim().toLowerCase()); } } } dataset.put("tags", tags); dataset.put("meta", meta); dataset.put("resources", resources); dataset.put("other_tags", other_tags); } } catch (Exception e) { e.printStackTrace(); LOG.error(e); } finally { method.releaseConnection(); } return dataset; }
From source file:org.hammer.santamaria.mapper.dataset.CKANDataSetInput.java
@SuppressWarnings({ "rawtypes", "unchecked" }) public static void main(String[] pArgs) throws Exception { String id = "proportion-of-children-under-5-years-who-have-ever-breastfed-by-county-xls-2005-6"; String sId = EncodeURIComponent(id); String url = "https://africaopendata.org/api/action"; BSONObject dataset = new BasicBSONObject(); dataset.put("datasource", "Test"); dataset.put("id", id); LOG.info("---> id " + id + " - " + sId); HttpClient client = new HttpClient(); client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false); LOG.info(/* w w w . java 2 s .c om*/ "******************************************************************************************************"); LOG.info(" "); LOG.info(url + PACKAGE_GET + sId); LOG.info(" "); LOG.info( "******************************************************************************************************"); GetMethod method = new GetMethod(url + PACKAGE_GET + sId); method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { throw new Exception("Method failed: " + method.getStatusLine()); } byte[] responseBody = method.getResponseBody(); LOG.debug(new String(responseBody)); Document doc = Document.parse(new String(responseBody)); if (doc != null && doc.containsKey("result")) { Document result = new Document(); LOG.info(doc.get("result").getClass().toString()); if (doc.get("result") instanceof Document) { LOG.info("!!! Document result !!!!"); result = (Document) doc.get("result"); } else if (doc.get("result") instanceof ArrayList) { LOG.info("!!! Document list !!!!"); result = (Document) (((ArrayList) doc.get("result")).get(0)); } else { LOG.info("!!! NOT FOUND !!!!"); result = null; } LOG.info("result find!"); if (result != null) { dataset.put("title", result.get("title")); dataset.put("author", result.get("author")); dataset.put("author_email", result.get("author_email")); dataset.put("license_id", result.get("license_id")); } ArrayList<String> tags = new ArrayList<String>(); ArrayList<String> meta = new ArrayList<String>(); ArrayList<String> other_tags = new ArrayList<String>(); if (result.containsKey("author") && result.get("author") != null) other_tags.add(result.get("author").toString()); if (result.containsKey("title") && result.get("title") != null) other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString())); if (result.containsKey("description") && result.get("description") != null) other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString())); ArrayList<Document> resources = new ArrayList<Document>(); if (result != null && result.containsKey("resources")) { resources = (ArrayList<Document>) result.get("resources"); for (Document resource : resources) { if (resource.getString("format").toUpperCase().equals("JSON")) { dataset.put("dataset-type", "JSON"); dataset.put("url", resource.get("url")); dataset.put("created", resource.get("created")); dataset.put("description", resource.get("description")); dataset.put("revision_timestamp", resource.get("revision_timestamp")); meta = DSSUtils.GetMetaByResource(resource.get("url").toString()); } } } if (result != null && result.containsKey("tags")) { ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags"); for (Document tag : tagsFromCKAN) { if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) { tags.add(tag.getString("display_name").trim().toLowerCase()); } else if (tag.containsKey("display_name")) { tags.add(tag.getString("display_name").trim().toLowerCase()); } } } dataset.put("tags", tags); dataset.put("meta", meta); dataset.put("resources", resources); dataset.put("other_tags", other_tags); } } catch (Exception e) { e.printStackTrace(); LOG.error(e); } finally { method.releaseConnection(); } //GetMetaByDocument("http://catalog.data.gov/api/action/package_show?id=1e68f387-5f1c-46c0-a0d1-46044ffef5bf"); }
From source file:org.hammer.santamaria.mapper.dataset.INPSDataSetInput.java
@SuppressWarnings("unchecked") public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) { BSONObject dataset = new BasicBSONObject(); dataset.put("datasource", datasource); dataset.put("id", id); LOG.info(datasource + id);//from www. j a v a 2s . c om HttpClient client = new HttpClient(); GetMethod method = new GetMethod(url); method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { throw new Exception("Method failed: " + method.getStatusLine()); } byte[] responseBody = method.getResponseBody(); LOG.debug(new String(responseBody)); Document doc = Document.parse(new String(responseBody)); dataset.put("title", doc.getString("name")); dataset.put("author", doc.getString("author")); dataset.put("author_email", doc.getString("author_email")); dataset.put("license_id", doc.getString("license_id")); dataset.put("description", doc.getString("notes")); ArrayList<String> tags = new ArrayList<String>(); ArrayList<String> meta = new ArrayList<String>(); ArrayList<String> other_tags = new ArrayList<String>(); if (doc.containsKey("author")) other_tags.add(doc.get("author").toString()); if (doc.containsKey("title")) other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("name").toString())); if (doc.containsKey("notes")) other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("notes").toString())); ArrayList<Document> resources = (ArrayList<Document>) doc.get("resources"); for (Document resource : resources) { if (resource.getString("format").toUpperCase().equals("JSON")) { dataset.put("dataset-type", "JSON"); dataset.put("url", resource.get("url")); dataset.put("created", resource.get("created")); dataset.put("revision_timestamp", resource.get("last_modified")); meta = this.getMetaByDocument(resource.get("url").toString()); } } tags = (ArrayList<String>) doc.get("tags"); dataset.put("tags", tags); dataset.put("meta", meta); dataset.put("resources", resources); dataset.put("other_tags", other_tags); } catch (Exception e) { LOG.error(e); } finally { method.releaseConnection(); } LOG.debug(dataset.get("title")); return dataset; }
From source file:org.hammer.santamaria.mapper.dataset.INPSDataSetInput.java
/** * Return meta from document for CKAN implementation * //from www . j a v a 2s.co m * @param url * @return */ public ArrayList<String> getMetaByDocument(String url) { ArrayList<String> meta = new ArrayList<String>(); HttpClient client = new HttpClient(); GetMethod method = new GetMethod(url); method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { throw new Exception("Method failed: " + method.getStatusLine()); } byte[] responseBody = method.getResponseBody(); if ((JSON.parse(new String(responseBody))) instanceof BasicBSONList) { BasicBSONList pList = (BasicBSONList) JSON.parse(new String(responseBody)); if (pList.size() > 0 && pList.get(0) != null) { BSONObject temp = (BSONObject) pList.get(0); for (String metaKey : temp.keySet()) { if (!meta.contains(metaKey.toLowerCase())) { meta.add(metaKey.toLowerCase()); } } } } else if ((JSON.parse(new String(responseBody))) instanceof BasicDBObject) { BasicDBObject temp = (BasicDBObject) JSON.parse(new String(responseBody)); for (String metaKey : temp.keySet()) { if (!meta.contains(metaKey.toLowerCase())) { meta.add(metaKey.toLowerCase()); } } } } catch (Exception e) { LOG.error(e); } finally { method.releaseConnection(); } return meta; }
From source file:org.hammer.santamaria.mapper.dataset.StatPortalDataSetInput.java
@SuppressWarnings("unchecked") public BSONObject getDataSet(String url, String datasource, String id, BSONObject c) { BSONObject dataset = new BasicBSONObject(); dataset.put("datasource", datasource); dataset.put("id", id); LOG.info(datasource + id);/*from w w w .j av a2 s . co m*/ HttpClient client = new HttpClient(); GetMethod method = new GetMethod(url); method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler"); method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { throw new Exception("Method failed: " + method.getStatusLine()); } byte[] responseBody = method.getResponseBody(); LOG.debug(new String(responseBody)); Document doc = Document.parse(new String(responseBody)); dataset.put("title", doc.getString("title")); dataset.put("author", doc.getString("author")); dataset.put("author_email", doc.getString("author_email")); dataset.put("license_id", doc.getString("license_id")); dataset.put("description", doc.getString("notes")); ArrayList<String> tags = new ArrayList<String>(); ArrayList<String> meta = new ArrayList<String>(); ArrayList<String> other_tags = new ArrayList<String>(); if (doc.containsKey("author")) other_tags.add(doc.get("author").toString()); if (doc.containsKey("title")) other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("title").toString())); if (doc.containsKey("notes")) other_tags.addAll(DSSUtils.GetKeyWordsFromText(doc.get("notes").toString())); ArrayList<Document> resources = (ArrayList<Document>) doc.get("resources"); for (Document resource : resources) { if (resource.getString("format").toUpperCase().equals("JSON")) { dataset.put("dataset-type", "JSON"); dataset.put("url", resource.get("url")); dataset.put("created", resource.get("created")); dataset.put("revision_timestamp", resource.get("last_modified")); meta = this.getMetaByDocument(resource.get("url").toString()); } } tags = (ArrayList<String>) doc.get("tags"); dataset.put("tags", tags); dataset.put("meta", meta); dataset.put("resources", resources); dataset.put("other_tags", other_tags); } catch (Exception e) { LOG.error(e); } finally { method.releaseConnection(); } LOG.debug(dataset.get("title")); return dataset; }