List of usage examples for org.apache.commons.httpclient.params HttpMethodParams HTTP_CONTENT_CHARSET
String HTTP_CONTENT_CHARSET
To view the source code for org.apache.commons.httpclient.params HttpMethodParams HTTP_CONTENT_CHARSET.
Click Source Link
From source file:org.ala.harvester.BlueTierHarvester.java
private byte[] getContent(String url) throws Exception { String contentStr = null;/*from w ww . j a va 2s .c o m*/ // Create an instance of HttpClient. HttpClient client = new HttpClient(); // Create a method instance. GetMethod method = new GetMethod(url); // Provide custom retry handler is necessary method.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_ELEMENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_URI_CHARSET, "UTF-8"); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { String errMsg = "HTTP GET to " + "`" + url + "`" + " returned non HTTP OK code. " + "Returned code " + statusCode + " and message " + method.getStatusLine() + "\n"; method.releaseConnection(); logger.error(errMsg); throw new Exception(errMsg); } InputStream responseStream = method.getResponseBodyAsStream(); contentStr = inputStream2String(responseStream); } catch (Exception domCreationErr) { throw new Exception(domCreationErr); } finally { // Release the connection. method.releaseConnection(); } return contentStr.getBytes(); }
From source file:org.ala.harvester.BlueTierHarvester.java
private String getIndexPageStr(String url) throws Exception { // Constructs the GET URL to search. // `woe_id` is Yahoo! Where On Earth ID. // Issue//from w w w. j av a 2 s. c o m // http://api.Morphbank.com/services/rest/?method=Morphbank.places.find&api_key=08f5318120189e9d12669465c0113351&query=australia // to find Australia. // `woe_id` here is country level code, as opposed to continent code. String urlToSearch = url; System.out.println("Search URL: " + urlToSearch); /* * // Default parameters if not supplied. if (this.MorphbankApiKeySupplied * == false) { urlToSearch += "&" + "api_key=" + this.MorphbankApiKey; } if * (this.currentPageNumSupplied == false) { urlToSearch += "&" + "page=" * + this.currentPageNum; } if (this.recordsPerPageSupplied == false) { * urlToSearch += "&" + "per_page=" + this.recordsPerPage; } */ // System.out.println(urlToSearch); logger.debug("URL to search is: " + "`" + urlToSearch + "`" + "\n"); // Create an instance of HttpClient. HttpClient client = new HttpClient(); // Create a method instance. GetMethod method = new GetMethod(urlToSearch); // Provide custom retry handler is necessary method.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_ELEMENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_URI_CHARSET, "UTF-8"); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { String errMsg = "HTTP GET to " + "`" + urlToSearch + "`" + " returned non HTTP OK code. Returned code " + statusCode + " and message " + method.getStatusLine() + "\n"; method.releaseConnection(); throw new Exception(errMsg); } // InputStream responseStream = method.getResponseBodyAsStream(); // // // Instantiates a DOM builder to create a DOM of the response. // DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); // DocumentBuilder builder = factory.newDocumentBuilder(); // return a parsed Document return method.getResponseBodyAsString(); } catch (Exception httpErr) { String errMsg = "HTTP GET to `" + urlToSearch + "` returned HTTP error."; throw new Exception(errMsg, httpErr); } finally { // Release the connection. method.releaseConnection(); } }
From source file:org.ala.harvester.FlickrHarvester.java
/** * Retrieves a map of licences.//from w w w. j av a 2s . c om * * @return */ private Map<String, Licence> getLicencesMap() throws Exception { final String flickrMethodUri = "flickr.photos.licenses.getInfo"; String urlToSearch = this.flickrRestBaseUrl + "/" + "?method=" + flickrMethodUri + "&api_key=" + this.flickrApiKey; logger.info(urlToSearch); logger.debug("URL to search is: " + "`" + urlToSearch + "`" + "\n"); // Create an instance of HttpClient. HttpClient client = new HttpClient(); // Create a method instance. GetMethod method = new GetMethod(urlToSearch); // Provide custom retry handler is necessary method.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_ELEMENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_URI_CHARSET, "UTF-8"); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { String errMsg = "HTTP GET to " + "`" + urlToSearch + "`" + " returned non HTTP OK code. Returned code " + statusCode + " and message " + method.getStatusLine() + "\n"; method.releaseConnection(); throw new Exception(errMsg); } //parse the response InputStream responseStream = method.getResponseBodyAsStream(); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = factory.newDocumentBuilder(); Document doc = builder.parse(responseStream); XPathFactory xfactory = XPathFactory.newInstance(); XPath xpath = xfactory.newXPath(); XPathExpression xe = xpath.compile("/rsp/licenses/license"); NodeList nodeSet = (NodeList) xe.evaluate(doc, XPathConstants.NODESET); Map<String, Licence> licencesMap = new HashMap<String, Licence>(); for (int i = 0; i < nodeSet.getLength(); i++) { NamedNodeMap map = nodeSet.item(i).getAttributes(); String id = map.getNamedItem("id").getNodeValue(); Licence licence = new Licence(); licence.setName(map.getNamedItem("name").getNodeValue()); licence.setUrl(map.getNamedItem("url").getNodeValue()); licencesMap.put(id, licence); } return licencesMap; } catch (Exception httpErr) { String errMsg = "HTTP GET to `" + urlToSearch + "` returned HTTP error."; throw new Exception(errMsg, httpErr); } finally { // Release the connection. method.releaseConnection(); } }
From source file:org.ala.harvester.FlickrHarvester.java
/** * Process a single image, do the document mapping etc * // w w w .ja v a2s.c om * @param infosourceId * @param imageIndex * @param currentResDom * @throws Exception */ private boolean processSingleImage(int infosourceId, int imageIndex, Document currentResDom) throws Exception { XPathFactory factory = XPathFactory.newInstance(); XPath xpath = factory.newXPath(); String xPathToPhotoId = "/rsp/photos/photo[" + imageIndex + "]/@id"; String photoId = (String) xpath.evaluate(xPathToPhotoId, currentResDom, XPathConstants.STRING); logger.info("Handling photo ID: " + photoId); final String flickrMethod = "flickr.photos.getInfo"; // Calls the Flickr's Photo Info API to determine whether the photo // comes from Australia or not. String photoInfoFlickrUrl = this.flickrRestBaseUrl + "/" + "?method=" + flickrMethod + "&" + "api_key=" + this.flickrApiKey + "&" + "photo_id=" + photoId; System.out.println("PHOTO URL:" + photoInfoFlickrUrl); org.w3c.dom.Document photoInfoDom = null; // Create an instance of HttpClient. HttpClient client = new HttpClient(); // Create a method instance. GetMethod method = new GetMethod(photoInfoFlickrUrl); // Provide custom retry handler is necessary method.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_ELEMENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_URI_CHARSET, "UTF-8"); logger.debug( "Fetching info. for photo with ID " + photoId + " from " + "`" + photoInfoFlickrUrl + "`" + "\n"); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { String errMsg = "HTTP GET to " + "`" + photoInfoFlickrUrl + "`" + " returned non HTTP OK code. " + "Returned code " + statusCode + " and message " + method.getStatusLine() + "\n"; method.releaseConnection(); logger.error(errMsg); throw new Exception(errMsg); } InputStream responseStream = method.getResponseBodyAsStream(); // Instantiates a DOM builder to create a DOM of the response. DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder domBuilder = domFactory.newDocumentBuilder(); photoInfoDom = domBuilder.parse(responseStream); } catch (Exception domCreationErr) { throw new Exception("Failed to create DOM representation of GET response.", domCreationErr); } finally { // Release the connection. method.releaseConnection(); } // Check for Flickr's error. if (!isDocExtractionSuccessful(photoInfoDom)) { throw new Exception("Flickr error response for fetching single image information."); } if (System.getProperty("overwrite") != null && "false".equals(System.getProperty("overwrite"))) { String photoPageUrl = (String) xpath.evaluate("/rsp/photo/urls/url[@type=\"photopage\"]/text()", photoInfoDom, XPathConstants.STRING); logger.debug("photo page URL: " + photoPageUrl); org.ala.model.Document doc = this.repository.getDocumentByGuid(photoPageUrl); if (doc != null) { logger.debug("Document with URI already harvested. Skipping: " + photoPageUrl); return true; } } // Determines whether photo has geo-coded tag from Australia. // If so, pass onto DocumentMapper. if (isPhotoFromAustralia(photoInfoDom)) { try { String document = (DOMUtils.domToString(photoInfoDom)); // FIXME flickr GUID ??? List<ParsedDocument> parsedDocs = documentMapper.map("", document.getBytes()); for (ParsedDocument parsedDoc : parsedDocs) { this.repository.storeDocument(infosourceId, parsedDoc); debugParsedDoc(parsedDoc); } return false; } catch (Exception docMapperErr) { // Skipping over errors here and proceeding to next document. logger.error("Problem processing image. " + docMapperErr.toString() + ", Problem processing: " + photoInfoFlickrUrl, docMapperErr); } } else { logger.debug("Photo is unAustralian: " + photoInfoFlickrUrl); } return false; }
From source file:org.ala.harvester.FlickrHarvester.java
private org.w3c.dom.Document getIndexPage(int pageNumber, Date startDate, Date endDate) throws Exception { final String flickrMethodUri = "flickr.photos.search"; // Constructs the GET URL to search. // `woe_id` is Yahoo! Where On Earth ID. // Issue//ww w .j a v a2s . com // http://api.flickr.com/services/rest/?method=flickr.places.find&api_key=08f5318120189e9d12669465c0113351&query=australia // to find Australia. // `woe_id` here is country level code, as opposed to continent code. SimpleDateFormat mysqlDateTime = new SimpleDateFormat("yyyy-MM-dd"); String minUpdateDate = mysqlDateTime.format(startDate); String maxUpdateDate = mysqlDateTime.format(endDate); String urlToSearch = this.flickrRestBaseUrl + "/" + "?method=" + flickrMethodUri + "&content_type=1" // + "&sort=date-posted-asc" // + "&machine_tag_mode=any" // + "&group_id=" + this.eolGroupId + "&user_id=" + this.eolGroupId // + "&accuracy=3" + "&privacy_filter=1" // + "&machine_tags=%22geo:country=Australia%22" // + "&machine_tags=%22taxonomy:binomial=Pogona%20barbata%22" // + "&tags=geo:country=Australia&country=Australia" // + "&has_geo=1" // + "&accuracy=3" // + "&woe_id=23424748" // MYSQL date time + "&min_upload_date=" + minUpdateDate //startDate + "&max_upload_date=" + maxUpdateDate //endDate + "&api_key=" + this.flickrApiKey + "&page=" + pageNumber + "&per_page=" + this.recordsPerPage; // String urlToSearch = // "http://api.flickr.com/services/rest/?method=flickr.photos.search" + // "&api_key=08f5318120189e9d12669465c0113351" + // "&page=1" + // "&per_page=50" + // "&machine_tag_mode=any" + // "&content_type=1" + // "&group_id=806927@N20&privacy_filter=1" + // "&machine_tags=%22taxonomy:binomial=Pogona%20barbata%22"; logger.info("Search URL: " + urlToSearch); /* * // Default parameters if not supplied. if (this.flickrApiKeySupplied * == false) { urlToSearch += "&" + "api_key=" + this.flickrApiKey; } if * (this.currentPageNumSupplied == false) { urlToSearch += "&" + "page=" * + this.currentPageNum; } if (this.recordsPerPageSupplied == false) { * urlToSearch += "&" + "per_page=" + this.recordsPerPage; } */ // logger.info(urlToSearch); logger.debug("URL to search is: " + "`" + urlToSearch + "`" + "\n"); // Create an instance of HttpClient. HttpClient client = new HttpClient(); // Create a method instance. GetMethod method = new GetMethod(urlToSearch); // Provide custom retry handler is necessary method.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_ELEMENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_URI_CHARSET, "UTF-8"); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { String errMsg = "HTTP GET to " + "`" + urlToSearch + "`" + " returned non HTTP OK code. Returned code " + statusCode + " and message " + method.getStatusLine() + "\n"; method.releaseConnection(); throw new Exception(errMsg); } InputStream responseStream = method.getResponseBodyAsStream(); // Instantiates a DOM builder to create a DOM of the response. DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = factory.newDocumentBuilder(); // return a parsed Document return builder.parse(responseStream); } catch (Exception httpErr) { String errMsg = "HTTP GET to `" + urlToSearch + "` returned HTTP error."; throw new Exception(errMsg, httpErr); } finally { // Release the connection. method.releaseConnection(); } }
From source file:org.ala.harvester.MaHarvester.java
private String getIndexPageStr() throws Exception { // Constructs the GET URL to search. // `woe_id` is Yahoo! Where On Earth ID. // Issue/* ww w. java 2s.co m*/ // http://api.Morphbank.com/services/rest/?method=Morphbank.places.find&api_key=08f5318120189e9d12669465c0113351&query=australia // to find Australia. // `woe_id` here is country level code, as opposed to continent code. String urlToSearch = this.endpoint; System.out.println("Search URL: " + urlToSearch); /* * // Default parameters if not supplied. if (this.MorphbankApiKeySupplied * == false) { urlToSearch += "&" + "api_key=" + this.MorphbankApiKey; } if * (this.currentPageNumSupplied == false) { urlToSearch += "&" + "page=" * + this.currentPageNum; } if (this.recordsPerPageSupplied == false) { * urlToSearch += "&" + "per_page=" + this.recordsPerPage; } */ // System.out.println(urlToSearch); logger.debug("URL to search is: " + "`" + urlToSearch + "`" + "\n"); // Create an instance of HttpClient. HttpClient client = new HttpClient(); // Create a method instance. GetMethod method = new GetMethod(urlToSearch); // Provide custom retry handler is necessary method.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_ELEMENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_URI_CHARSET, "UTF-8"); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { String errMsg = "HTTP GET to " + "`" + urlToSearch + "`" + " returned non HTTP OK code. Returned code " + statusCode + " and message " + method.getStatusLine() + "\n"; method.releaseConnection(); throw new Exception(errMsg); } // InputStream responseStream = method.getResponseBodyAsStream(); // // // Instantiates a DOM builder to create a DOM of the response. // DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); // DocumentBuilder builder = factory.newDocumentBuilder(); // return a parsed Document return method.getResponseBodyAsString(); } catch (Exception httpErr) { String errMsg = "HTTP GET to `" + urlToSearch + "` returned HTTP error."; throw new Exception(errMsg, httpErr); } finally { // Release the connection. method.releaseConnection(); } }
From source file:org.ala.harvester.MorphbankHarvester.java
private org.w3c.dom.Document getIndexPage() throws Exception { final String MorphbankMethodUri = "Morphbank.photos.search"; // Constructs the GET URL to search. // `woe_id` is Yahoo! Where On Earth ID. // Issue//w w w .ja va 2 s . c om // http://api.Morphbank.com/services/rest/?method=Morphbank.places.find&api_key=08f5318120189e9d12669465c0113351&query=australia // to find Australia. // `woe_id` here is country level code, as opposed to continent code. String urlToSearch = this.endpoint; System.out.println("Search URL: " + urlToSearch); /* * // Default parameters if not supplied. if (this.MorphbankApiKeySupplied * == false) { urlToSearch += "&" + "api_key=" + this.MorphbankApiKey; } if * (this.currentPageNumSupplied == false) { urlToSearch += "&" + "page=" * + this.currentPageNum; } if (this.recordsPerPageSupplied == false) { * urlToSearch += "&" + "per_page=" + this.recordsPerPage; } */ // System.out.println(urlToSearch); logger.debug("URL to search is: " + "`" + urlToSearch + "`" + "\n"); // Create an instance of HttpClient. HttpClient client = new HttpClient(); // Create a method instance. GetMethod method = new GetMethod(urlToSearch); // Provide custom retry handler is necessary method.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_ELEMENT_CHARSET, "UTF-8"); method.getParams().setParameter(HttpMethodParams.HTTP_URI_CHARSET, "UTF-8"); try { int statusCode = client.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { String errMsg = "HTTP GET to " + "`" + urlToSearch + "`" + " returned non HTTP OK code. Returned code " + statusCode + " and message " + method.getStatusLine() + "\n"; method.releaseConnection(); throw new Exception(errMsg); } String inputStr = method.getResponseBodyAsString(); // inputStr = inputStr.replaceAll("[^\\x00-\\x7f]*", ""); inputStr = inputStr.replaceAll("/dwcg:VerbatimLongitude>", "</dwcg:VerbatimLongitude>"); inputStr = inputStr.replaceAll("/dwcg:VerbatimLatitude>", "</dwcg:VerbatimLatitude>"); inputStr = inputStr.replaceAll("<</", "</"); // Pattern p = Pattern.compile("[^<]{1}/[a-zA-Z]{1,}:[a-zA-Z]{1,}>"); // // Matcher m = p.matcher(inputStr); // // int searchIdx = 0; // // while (m.find(searchIdx)) { // int endIdx = m.end(); // // // // searchIdx = endIdx; // } // System.out.println(inputStr); InputSource is = new InputSource(new StringReader(new String(inputStr))); // Instantiates a DOM builder to create a DOM of the response. DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = factory.newDocumentBuilder(); // return a parsed Document return builder.parse(is); } catch (Exception httpErr) { String errMsg = "HTTP GET to `" + urlToSearch + "` returned HTTP error."; throw new Exception(errMsg, httpErr); } finally { // Release the connection. method.releaseConnection(); } }
From source file:org.easyframework.core.crawl.GetSample.java
public static Map<String, String> get(String url) { Map<String, String> result = new HashMap<String, String>(); HttpClient client = new HttpClient(); GetMethod method = new GetMethod(url); method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); try {/*from w w w . ja v a2s . c om*/ int statusCode = client.executeMethod(method); String charset = method.getResponseCharSet(); method.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, charset); if (statusCode != HttpStatus.SC_OK) { System.err.println("Method failed: " + method.getStatusLine()); } byte[] responseBody = method.getResponseBody(); result.put(GetSample.RESPONSE_TEXT, new String(responseBody, charset)); result.put(GetSample.CHARSET, charset); } catch (HttpException e) { System.err.println("Fatal protocol violation: " + e.getMessage()); e.printStackTrace(); } catch (IOException e) { System.err.println("Fatal transport error: " + e.getMessage()); e.printStackTrace(); } finally { method.releaseConnection(); return result; } }
From source file:org.eclipse.ecf.remoteservice.rest.client.RestClientService.java
protected HttpMethod createAndPrepareHttpMethod(String uri, IRemoteCall call, IRemoteCallable callable) throws RestException { HttpMethod httpMethod = null;// w w w . j a v a 2 s . c o m IRemoteCallableRequestType requestType = callable.getRequestType(); if (requestType == null) throw new RestException("Request type for call cannot be null"); //$NON-NLS-1$ try { if (requestType instanceof HttpGetRequestType) { httpMethod = prepareGetMethod(uri, call, callable); } else if (requestType instanceof HttpPostRequestType) { httpMethod = preparePostMethod(uri, call, callable); } else if (requestType instanceof HttpPutRequestType) { httpMethod = preparePutMethod(uri, call, callable); } else if (requestType instanceof HttpDeleteRequestType) { httpMethod = prepareDeleteMethod(uri, call, callable); } else { throw new RestException("HTTP method " + requestType + " not supported"); //$NON-NLS-1$ //$NON-NLS-2$ } } catch (NotSerializableException e) { String message = "Could not serialize parameters for uri=" + uri + " call=" + call + " callable=" //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$ + callable; logException(message, e); throw new RestException(message); } // add additional request headers addRequestHeaders(httpMethod, call, callable); // handle authentication setupAuthenticaton(httpClient, httpMethod); // needed because a resource can link to another resource httpClient.getParams().setParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, new Boolean(true)); httpClient.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, DEFAULT_HTTP_CONTENT_CHARSET); setupTimeouts(httpClient, call, callable); return httpMethod; }
From source file:org.openo.nfvo.monitor.dac.util.APIHttpClient.java
public static String doPut(String uri, String jsonObj, String token) { String resStr = null;/*from w ww.java2 s . c om*/ HttpClient htpClient = new HttpClient(); PutMethod putMethod = new PutMethod(uri); putMethod.addRequestHeader("Content-Type", "application/json"); putMethod.addRequestHeader("X-Auth-Token", token); putMethod.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "utf-8"); putMethod.setRequestBody(jsonObj); try { int statusCode = htpClient.executeMethod(putMethod); if (statusCode != HttpStatus.SC_OK) { return null; } byte[] responseBody = putMethod.getResponseBody(); resStr = new String(responseBody, "utf-8"); } catch (Exception e) { e.printStackTrace(); } finally { putMethod.releaseConnection(); } return resStr; }