List of usage examples for org.apache.http.entity ContentType getOrDefault
public static ContentType getOrDefault(HttpEntity httpEntity) throws ParseException, UnsupportedCharsetException
From source file:com.jaeksoft.searchlib.crawler.web.spider.HttpAbstract.java
public String getContentTypeCharset() { synchronized (this) { if (httpEntity == null) return null; try {/* w w w .j a v a 2s . c o m*/ ContentType ct = ContentType.getOrDefault(httpEntity); if (ct == null) return null; Charset charset = ct.getCharset(); if (charset == null) return null; return charset.name(); } catch (UnsupportedCharsetException e) { Logging.warn(e); return null; } } }
From source file:act.installer.bing.BingSearchResults.java
/** This function issues a Bing search API call and gets the JSONObject containing the relevant results * (including TotalCounts and SearchResults) * @param queryTerm (String) the term to query for. * @param count (int) URL parameter indicating how many results to return. Max value is 100. * @param offset (int) URL parameter indicating the offset for results. * @return a JSONObject containing the response. * @throws IOException/*from ww w . j a va 2 s. c o m*/ */ private JsonNode fetchBingSearchAPIResponse(String queryTerm, Integer count, Integer offset) throws IOException { if (count <= 0) { LOGGER.error( "Bing Search API was called with \"count\" URL parameter = 0. Please request at least one result."); return null; } URI uri = null; try { // Bing URL pattern. Note that we use composite queries to allow retrieval of the total results count. // Transaction cost is [count] bings, where [count] is the value of the URL parameter "count". // In other words, we can make 5M calls with [count]=1 per month. // Example: https://api.cognitive.microsoft.com/bing/v5.0/search?q=porsche&responseFilter=webpages uri = new URIBuilder().setScheme("https").setHost(BING_API_HOST).setPath(BING_API_PATH) // Wrap the query term (%s) with double quotes (%%22) for exact search .setParameter("q", String.format("%s", queryTerm)) // Restrict response to Web Pages only .setParameter("responseFilter", "webpages") // "count" parameter. .setParameter("count", count.toString()) // "offset" parameter. .setParameter("offset", offset.toString()).build(); } catch (URISyntaxException e) { LOGGER.error("An error occurred when trying to build the Bing Search API URI", e); } JsonNode results; HttpGet httpget = new HttpGet(uri); // Yay for un-encrypted account key! // TODO: actually is there a way to encrypt it? httpget.setHeader("Ocp-Apim-Subscription-Key", accountKey); CloseableHttpClient httpclient = HttpClients.custom().setConnectionManager(basicConnManager).build(); try (CloseableHttpResponse response = httpclient.execute(httpget)) { Integer statusCode = response.getStatusLine().getStatusCode(); // TODO: The Web Search API returns useful error messages, we could use them to have better insights on failures. // See: https://dev.cognitive.microsoft.com/docs/services/56b43eeccf5ff8098cef3807/operations/56b4447dcf5ff8098cef380d if (!statusCode.equals(HttpStatus.SC_OK)) { LOGGER.error("Bing Search API call returned an unexpected status code (%d) for URI: %s", statusCode, uri); return null; } HttpEntity entity = response.getEntity(); ContentType contentType = ContentType.getOrDefault(entity); Charset charset = contentType.getCharset(); if (charset == null) { charset = StandardCharsets.UTF_8; } try (final BufferedReader in = new BufferedReader( new InputStreamReader(entity.getContent(), charset))) { String inputLine; final StringBuilder stringResponse = new StringBuilder(); while ((inputLine = in.readLine()) != null) { stringResponse.append(inputLine); } JsonNode rootNode = mapper.readValue(stringResponse.toString(), JsonNode.class); results = rootNode.path("webPages"); } } return results; }
From source file:org.apache.manifoldcf.agents.output.solr.ModifiedHttpSolrServer.java
@Override public NamedList<Object> request(final SolrRequest request, final ResponseParser processor) throws SolrServerException, IOException { HttpRequestBase method = null;//from ww w . ja va 2 s. c o m InputStream is = null; SolrParams params = request.getParams(); Collection<ContentStream> streams = requestWriter.getContentStreams(request); String path = requestWriter.getPath(request); if (path == null || !path.startsWith("/")) { path = DEFAULT_PATH; } ResponseParser parser = request.getResponseParser(); if (parser == null) { parser = this.parser; } // The parser 'wt=' and 'version=' params are used instead of the original // params ModifiableSolrParams wparams = new ModifiableSolrParams(params); if (parser != null) { wparams.set(CommonParams.WT, parser.getWriterType()); wparams.set(CommonParams.VERSION, parser.getVersion()); } if (invariantParams != null) { wparams.add(invariantParams); } params = wparams; int tries = maxRetries + 1; try { while (tries-- > 0) { // Note: since we aren't do intermittent time keeping // ourselves, the potential non-timeout latency could be as // much as tries-times (plus scheduling effects) the given // timeAllowed. try { if (SolrRequest.METHOD.GET == request.getMethod()) { if (streams != null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!"); } method = new HttpGet(baseUrl + path + toQueryString(params, false)); } else if (SolrRequest.METHOD.POST == request.getMethod()) { String url = baseUrl + path; boolean hasNullStreamName = false; if (streams != null) { for (ContentStream cs : streams) { if (cs.getName() == null) { hasNullStreamName = true; break; } } } boolean isMultipart = (this.useMultiPartPost || (streams != null && streams.size() > 1)) && !hasNullStreamName; LinkedList<NameValuePair> postParams = new LinkedList<NameValuePair>(); if (streams == null || isMultipart) { HttpPost post = new HttpPost(url); post.setHeader("Content-Charset", "UTF-8"); if (!isMultipart) { post.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } List<FormBodyPart> parts = new LinkedList<FormBodyPart>(); Iterator<String> iter = params.getParameterNamesIterator(); while (iter.hasNext()) { String p = iter.next(); String[] vals = params.getParams(p); if (vals != null) { for (String v : vals) { if (isMultipart) { parts.add( new FormBodyPart(p, new StringBody(v, StandardCharsets.UTF_8))); } else { postParams.add(new BasicNameValuePair(p, v)); } } } } if (isMultipart && streams != null) { for (ContentStream content : streams) { String contentType = content.getContentType(); if (contentType == null) { contentType = "application/octet-stream"; // default } String contentName = content.getName(); parts.add(new FormBodyPart(contentName, new InputStreamBody(content.getStream(), contentType, content.getName()))); } } if (parts.size() > 0) { ModifiedMultipartEntity entity = new ModifiedMultipartEntity( HttpMultipartMode.STRICT, null, StandardCharsets.UTF_8); for (FormBodyPart p : parts) { entity.addPart(p); } post.setEntity(entity); } else { //not using multipart post.setEntity(new UrlEncodedFormEntity(postParams, StandardCharsets.UTF_8)); } method = post; } // It is has one stream, it is the post body, put the params in the URL else { String pstr = toQueryString(params, false); HttpPost post = new HttpPost(url + pstr); // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; for (ContentStream content : streams) { contentStream[0] = content; break; } if (contentStream[0] instanceof RequestWriter.LazyContentStream) { post.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { @Override public Header getContentType() { return new BasicHeader("Content-Type", contentStream[0].getContentType()); } @Override public boolean isRepeatable() { return false; } }); } else { post.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { @Override public Header getContentType() { return new BasicHeader("Content-Type", contentStream[0].getContentType()); } @Override public boolean isRepeatable() { return false; } }); } method = post; } } else { throw new SolrServerException("Unsupported method: " + request.getMethod()); } } catch (NoHttpResponseException r) { method = null; if (is != null) { is.close(); } // If out of tries then just rethrow (as normal error). if (tries < 1) { throw r; } } } } catch (IOException ex) { throw new SolrServerException("error reading streams", ex); } // XXX client already has this set, is this needed? //method.getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, // followRedirects); method.addHeader("User-Agent", AGENT); InputStream respBody = null; boolean shouldClose = true; try { // Execute the method. final HttpResponse response = httpClient.execute(method); int httpStatus = response.getStatusLine().getStatusCode(); // Read the contents respBody = response.getEntity().getContent(); // handle some http level checks before trying to parse the response switch (httpStatus) { case HttpStatus.SC_OK: case HttpStatus.SC_BAD_REQUEST: case HttpStatus.SC_CONFLICT: // 409 break; case HttpStatus.SC_MOVED_PERMANENTLY: case HttpStatus.SC_MOVED_TEMPORARILY: if (!followRedirects) { throw new SolrServerException( "Server at " + getBaseURL() + " sent back a redirect (" + httpStatus + ")."); } break; default: throw new SolrException(SolrException.ErrorCode.getErrorCode(httpStatus), "Server at " + getBaseURL() + " returned non ok status:" + httpStatus + ", message:" + response.getStatusLine().getReasonPhrase()); } if (processor == null) { // no processor specified, return raw stream NamedList<Object> rsp = new NamedList<Object>(); rsp.add("stream", respBody); // Only case where stream should not be closed shouldClose = false; return rsp; } Charset charsetObject = ContentType.getOrDefault(response.getEntity()).getCharset(); String charset; if (charsetObject != null) charset = charsetObject.name(); else charset = "utf-8"; NamedList<Object> rsp = processor.processResponse(respBody, charset); if (httpStatus != HttpStatus.SC_OK) { String reason = null; try { NamedList err = (NamedList) rsp.get("error"); if (err != null) { reason = (String) err.get("msg"); // TODO? get the trace? } } catch (Exception ex) { } if (reason == null) { StringBuilder msg = new StringBuilder(); msg.append(response.getStatusLine().getReasonPhrase()); msg.append("\n\n"); msg.append("request: " + method.getURI()); reason = URLDecoder.decode(msg.toString()); } throw new SolrException(SolrException.ErrorCode.getErrorCode(httpStatus), reason); } return rsp; } catch (ConnectException e) { throw new SolrServerException("Server refused connection at: " + getBaseURL(), e); } catch (SocketTimeoutException e) { throw new SolrServerException("Timeout occured while waiting response from server at: " + getBaseURL(), e); } catch (IOException e) { throw new SolrServerException("IOException occured when talking to server at: " + getBaseURL(), e); } finally { if (respBody != null && shouldClose) { try { respBody.close(); } catch (Throwable t) { } // ignore } } }
From source file:org.duniter.core.client.service.HttpServiceImpl.java
protected Object parseResponse(HttpUriRequest request, HttpResponse response, Class<?> ResultClass) throws IOException { Object result = null;//from www . j av a2s . c o m boolean isStreamContent = ResultClass == null || ResultClass.equals(InputStream.class); boolean isStringContent = !isStreamContent && ResultClass != null && ResultClass.equals(String.class); InputStream content = response.getEntity().getContent(); // If should return an inputstream if (isStreamContent) { result = content; // must be close by caller } // If should return String else if (isStringContent) { try { String stringContent = getContentAsString(content); // Add a debug before returning the result if (log.isDebugEnabled()) { log.debug("Parsing response:\n" + stringContent); } return stringContent; } finally { if (content != null) { content.close(); } } } // deserialize Json else { try { result = readValue(content, ResultClass); } catch (Exception e) { String requestPath = request.getURI().toString(); // Check if content-type error ContentType contentType = ContentType.getOrDefault(response.getEntity()); String actualMimeType = contentType.getMimeType(); if (!ObjectUtils.equals(ContentType.APPLICATION_JSON.getMimeType(), actualMimeType)) { throw new TechnicalException(I18n.t("duniter4j.client.core.invalidResponseContentType", requestPath, ContentType.APPLICATION_JSON.getMimeType(), actualMimeType)); } // throw a generic error throw new TechnicalException(I18n.t("duniter4j.client.core.invalidResponse", requestPath), e); } finally { if (content != null) { content.close(); } } } if (result == null) { throw new TechnicalException( I18n.t("duniter4j.client.core.emptyResponse", request.getURI().toString())); } return result; }
From source file:de.elomagic.carafile.client.CaraFileClient.java
/** * Returns status of the registry.//from w w w . ja v a 2 s . c om * * @return Status * @throws IOException Thrown when unable to request status of the registry */ public RegistryStatus getRegistryStatus() throws IOException { if (registryURI == null) { throw new IllegalArgumentException("Parameter 'registryURI' must not be null!"); } LOG.debug("Getting registry status"); URI uri = CaraFileUtils.buildURI(registryURI, "registry", "status"); HttpResponse response = executeRequest(Request.Get(uri)).returnResponse(); Charset charset = ContentType.getOrDefault(response.getEntity()).getCharset(); RegistryStatus status = JsonUtil.read(new InputStreamReader(response.getEntity().getContent(), charset), RegistryStatus.class); LOG.debug("Registry status responsed"); return status; }
From source file:org.duniter.core.client.service.HttpServiceImpl.java
private boolean isContentType(HttpResponse response, ContentType expectedContentType) { if (response.getEntity() == null) return false; ContentType contentType = ContentType.getOrDefault(response.getEntity()); String actualMimeType = contentType.getMimeType(); return ObjectUtils.equals(expectedContentType.getMimeType(), actualMimeType); }
From source file:de.elomagic.carafile.client.CaraFileClient.java
private Set<PeerData> downloadPeerSet() throws IOException { if (registryURI == null) { throw new IllegalArgumentException("Parameter 'registryURI' must not be null!"); }/*from ww w .j a va 2 s .c om*/ LOG.debug("Download set of peers"); URI uri = CaraFileUtils.buildURI(registryURI, "registry", "listPeers"); HttpResponse response = executeRequest(Request.Get(uri)).returnResponse(); if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { Charset charset = ContentType.getOrDefault(response.getEntity()).getCharset(); Set<PeerData> peerSet = JsonUtil.read(new InputStreamReader(response.getEntity().getContent(), charset), PeerDataSet.class); LOG.debug("Registry response set of " + peerSet.size() + " peer(s)"); return peerSet; } throw new IOException("HTTP responce code " + response.getStatusLine().getStatusCode() + " " + response.getStatusLine().getReasonPhrase()); }
From source file:com.buffalokiwi.api.API.java
/** * Prepare the response entity for usage * @param response HTTP Response/* w w w . j a v a2 s . c o m*/ * @param get HTTP Get * @return The response results * @throws BrowserException * @throws RedirectException if a redirect needs to happen */ private IAPIResponse processResponse(final HttpResponse response, final HttpUriRequest get) throws APIException { if (response == null) throw new APIException("Endpoint response was null"); final HttpEntity entity = response.getEntity(); try { if (entity != null) { //..Get the charset String charset = ""; try { java.nio.charset.Charset cs = ContentType.getOrDefault(entity).getCharset(); if (cs != null) charset = cs.displayName(); } catch (ParseException | UnsupportedCharsetException e) { //..No nothing, use defaults } if ((charset == null) || (charset.isEmpty())) { charset = "UTF-8"; final Header[] headers = response.getHeaders("Content-Type"); if (headers.length > 0) { if (headers[0].getValue().equals("application/octet-stream")) charset = ""; } } //..Get content length header final Header[] clen = response.getHeaders("Content-Length"); final int contentLength = (clen.length > 0) ? Integer.valueOf(clen[0].getValue()) : 0; //..Process the stream try (final InputStream in = entity.getContent()) { final byte[] content = processEntity(in, contentLength); //..set the character set used to create the htmlBuffer if (LOG.isTraceEnabled()) { if (!charset.isEmpty()) APILog.trace(LOG, new String(content, 0, content.length, charset)); else APILog.trace(LOG, new String(content)); } final IAPIResponse res = createResponseObject(response, content, charset); APILog.debug(LOG, String.valueOf(res.getStatusLine().getStatusCode()), res.getStatusLine().getReasonPhrase(), "for", get.getURI().toString()); return res; } catch (RuntimeException e) { APILog.error(LOG, e); //..Abort get.abort(); throw new APIException(e.getMessage(), e); } } else { final IAPIResponse res = createResponseObject(response, null, ""); APILog.debug(LOG, String.valueOf(res.getStatusLine().getStatusCode()), res.getStatusLine().getReasonPhrase(), "for", get.getURI().toString()); return res; } } catch (IOException e) { APILog.error(LOG, e); throw new APIException("Failed to retrieve entity content (IOException)", e); } finally { try { EntityUtils.consume(entity); } catch (IOException e) { } } }
From source file:de.elomagic.carafile.client.CaraFileClient.java
MetaData getMetaDataFromResponse(final HttpResponse response) throws IOException { Charset charset = ContentType.getOrDefault(response.getEntity()).getCharset(); return JsonUtil.readFromReader(new InputStreamReader(response.getEntity().getContent(), charset)); }