List of usage examples for org.apache.http.impl.client RedirectLocations getAll
public List<URI> getAll()
From source file:com.frochr123.helper.CachedFileDownloader.java
public static SimpleEntry<String, ByteArrayOutputStream> getResultFromURL(String url) throws IOException { SimpleEntry<String, ByteArrayOutputStream> result = null; String finalUrl = null;/* ww w . j ava2 s . co m*/ ByteArrayOutputStream outputStream = null; if (url == null || url.isEmpty()) { return null; } // Create HTTP client and cusomized config for timeouts CloseableHttpClient httpClient = HttpClients.createDefault(); RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(CACHE_DOWNLOADER_DEFAULT_TIMEOUT) .setConnectTimeout(CACHE_DOWNLOADER_DEFAULT_TIMEOUT) .setConnectionRequestTimeout(CACHE_DOWNLOADER_DEFAULT_TIMEOUT).build(); // Create HTTP Get request HttpGet httpGet = new HttpGet(url); httpGet.setConfig(requestConfig); // Create context HttpContext context = new BasicHttpContext(); // Check for temporary FabQR download of own configured private FabQR instance // In that case Authorization needs to be added if (FabQRFunctions.getFabqrPrivateURL() != null && !FabQRFunctions.getFabqrPrivateURL().isEmpty() && url.startsWith(FabQRFunctions.getFabqrPrivateURL()) && url.contains("/" + FabQRFunctions.FABQR_TEMPORARY_MARKER + "/")) { // Set authentication information String encodedCredentials = Helper.getEncodedCredentials(FabQRFunctions.getFabqrPrivateUser(), FabQRFunctions.getFabqrPrivatePassword()); if (!encodedCredentials.isEmpty()) { httpGet.addHeader("Authorization", "Basic " + encodedCredentials); } } // Send request CloseableHttpResponse response = httpClient.execute(httpGet, context); // Get all redirected locations from context, if there are any RedirectLocations redirectLocations = (RedirectLocations) (context .getAttribute(HttpClientContext.REDIRECT_LOCATIONS)); if (redirectLocations != null) { finalUrl = redirectLocations.getAll().get(redirectLocations.getAll().size() - 1).toString(); } else { finalUrl = url; } // Check response valid and max file size if (response.getEntity() == null || response.getEntity().getContentLength() > CACHE_DOWNLOADER_MAX_FILESIZE_BYTES) { return null; } // Get data outputStream = new ByteArrayOutputStream(); response.getEntity().writeTo(outputStream); // Return result result = new SimpleEntry<String, ByteArrayOutputStream>(finalUrl, outputStream); return result; }
From source file:com.sangupta.jerry.http.WebResponseHandler.java
/** * @see org.apache.http.client.ResponseHandler#handleResponse(org.apache.http.HttpResponse) *///from ww w . jav a2s. c om @Override public WebResponse handleResponse(HttpResponse response, HttpContext localHttpContext) throws ClientProtocolException, IOException { StatusLine statusLine = response.getStatusLine(); HttpEntity entity = response.getEntity(); byte[] bytes = null; if (entity != null) { bytes = EntityUtils.toByteArray(entity); } final WebResponse webResponse = new WebResponse(bytes); // decipher from status line webResponse.responseCode = statusLine.getStatusCode(); webResponse.message = statusLine.getReasonPhrase(); // set size if (entity != null) { webResponse.size = entity.getContentLength(); } else { long value = 0; Header header = response.getFirstHeader(HttpHeaders.CONTENT_LENGTH); if (header != null) { String headerValue = header.getValue(); if (AssertUtils.isNotEmpty(headerValue)) { try { value = Long.parseLong(headerValue); } catch (Exception e) { // eat the exception } } } webResponse.size = value; } // content type if (entity != null && entity.getContentType() != null) { webResponse.contentType = entity.getContentType().getValue(); } // response headers final Header[] responseHeaders = response.getAllHeaders(); if (AssertUtils.isNotEmpty(responseHeaders)) { for (Header header : responseHeaders) { webResponse.headers.put(header.getName(), header.getValue()); } } // charset try { ContentType type = ContentType.get(entity); if (type != null) { webResponse.charSet = type.getCharset(); } } catch (UnsupportedCharsetException e) { // we are unable to find the charset for the content // let's leave it to be considered binary } // fill in the redirect uri chain RedirectLocations locations = (RedirectLocations) localHttpContext .getAttribute(HttpClientContext.REDIRECT_LOCATIONS); if (AssertUtils.isNotEmpty(locations)) { webResponse.setRedirectChain(locations.getAll()); } // return the object finally return webResponse; }
From source file:fr.ippon.wip.http.hc.TransformerResponseInterceptor.java
/** * If httpResponse must be transformed, creates an instance of * WIPTransformer, executes WIPTransformer#transform on the response content * and updates the response entity accordingly. * //w ww .j a v a 2 s . co m * @param httpResponse * @param context * @throws HttpException * @throws IOException */ public void process(HttpResponse httpResponse, HttpContext context) throws HttpException, IOException { PortletRequest portletRequest = HttpClientResourceManager.getInstance().getCurrentPortletRequest(); PortletResponse portletResponse = HttpClientResourceManager.getInstance().getCurrentPortletResponse(); WIPConfiguration config = WIPUtil.getConfiguration(portletRequest); RequestBuilder request = HttpClientResourceManager.getInstance().getCurrentRequest(); if (httpResponse == null) { // No response -> no transformation LOG.warning("No response to transform."); return; } HttpEntity entity = httpResponse.getEntity(); if (entity == null) { // No entity -> no transformation return; } ContentType contentType = ContentType.getOrDefault(entity); String mimeType = contentType.getMimeType(); String actualURL; RedirectLocations redirectLocations = (RedirectLocations) context .getAttribute("http.protocol.redirect-locations"); if (redirectLocations != null) actualURL = Iterables.getLast(redirectLocations.getAll()).toString(); else if (context.getAttribute(CachingHttpClient.CACHE_RESPONSE_STATUS) == CacheResponseStatus.CACHE_HIT) { actualURL = request.getRequestedURL(); } else { HttpRequest actualRequest = (HttpRequest) context.getAttribute(ExecutionContext.HTTP_REQUEST); HttpHost actualHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST); actualURL = actualHost.toURI() + actualRequest.getRequestLine().getUri(); } // Check if actual URI must be transformed if (!config.isProxyURI(actualURL)) return; // a builder for creating a WIPTransformer instance TransformerBuilder transformerBuilder = new TransformerBuilder().setActualURL(actualURL) .setMimeType(mimeType).setPortletRequest(portletRequest).setPortletResponse(portletResponse) .setResourceType(request.getResourceType()).setXmlReaderPool(xmlReaderPool); // Creates an instance of Transformer depending on ResourceType and // MimeType int status = transformerBuilder.build(); if (status == TransformerBuilder.STATUS_NO_TRANSFORMATION) return; WIPTransformer transformer = transformerBuilder.getTransformer(); // Call WIPTransformer#transform method and update the response Entity // object try { String content = EntityUtils.toString(entity); String transformedContent = ((AbstractTransformer) transformer).transform(content); StringEntity transformedEntity; if (contentType.getCharset() != null) { transformedEntity = new StringEntity(transformedContent, contentType); } else { transformedEntity = new StringEntity(transformedContent); } transformedEntity.setContentType(contentType.toString()); httpResponse.setEntity(transformedEntity); } catch (SAXException e) { LOG.log(Level.SEVERE, "Could not transform HTML", e); throw new IllegalArgumentException(e); } catch (TransformerException e) { LOG.log(Level.SEVERE, "Could not transform HTML", e); throw new IllegalArgumentException(e); } }
From source file:org.tallison.cc.CCGetter.java
private void fetch(CCIndexRecord r, Path rootDir, BufferedWriter writer) throws IOException { Path targFile = rootDir.resolve(r.getDigest().substring(0, 2) + "/" + r.getDigest()); if (Files.isRegularFile(targFile)) { writeStatus(r, FETCH_STATUS.ALREADY_IN_REPOSITORY, writer); logger.info("already retrieved:" + targFile.toAbsolutePath()); return;/*from w w w.j av a 2 s . co m*/ } String url = AWS_BASE + r.getFilename(); URI uri = null; try { uri = new URI(url); } catch (URISyntaxException e) { logger.warn("Bad url: " + url); writeStatus(r, FETCH_STATUS.BAD_URL, writer); return; } CloseableHttpClient httpClient = HttpClients.createDefault(); HttpHost target = new HttpHost(uri.getHost()); String urlPath = uri.getRawPath(); if (uri.getRawQuery() != null) { urlPath += "?" + uri.getRawQuery(); } HttpGet httpGet = null; try { httpGet = new HttpGet(urlPath); } catch (Exception e) { logger.warn("bad path " + uri.toString(), e); writeStatus(r, FETCH_STATUS.BAD_URL, writer); return; } if (proxyHost != null && proxyPort > -1) { HttpHost proxy = new HttpHost(proxyHost, proxyPort, "http"); RequestConfig requestConfig = RequestConfig.custom().setProxy(proxy).build(); httpGet.setConfig(requestConfig); } httpGet.addHeader("Range", r.getOffsetHeader()); HttpCoreContext coreContext = new HttpCoreContext(); CloseableHttpResponse httpResponse = null; URI lastURI = null; try { httpResponse = httpClient.execute(target, httpGet, coreContext); RedirectLocations redirectLocations = (RedirectLocations) coreContext .getAttribute(DefaultRedirectStrategy.REDIRECT_LOCATIONS); if (redirectLocations != null) { for (URI redirectURI : redirectLocations.getAll()) { lastURI = redirectURI; } } else { lastURI = httpGet.getURI(); } } catch (IOException e) { logger.warn("IOException for " + uri.toString(), e); writeStatus(r, FETCH_STATUS.FETCHED_IO_EXCEPTION, writer); return; } lastURI = uri.resolve(lastURI); if (httpResponse.getStatusLine().getStatusCode() != 200 && httpResponse.getStatusLine().getStatusCode() != 206) { logger.warn("Bad status for " + uri.toString() + " : " + httpResponse.getStatusLine().getStatusCode()); writeStatus(r, FETCH_STATUS.FETCHED_NOT_200, writer); return; } Path tmp = null; Header[] headers = null; boolean isTruncated = false; try { //this among other parts is plagiarized from centic9's CommonCrawlDocumentDownload //probably saved me hours. Thank you, Dominik! tmp = Files.createTempFile("cc-getter", ""); try (InputStream is = new GZIPInputStream(httpResponse.getEntity().getContent())) { WARCRecord warcRecord = new WARCRecord(new FastBufferedInputStream(is), "", 0); ArchiveRecordHeader archiveRecordHeader = warcRecord.getHeader(); if (archiveRecordHeader.getHeaderFields().containsKey(WARCConstants.HEADER_KEY_TRUNCATED)) { isTruncated = true; } headers = LaxHttpParser.parseHeaders(warcRecord, "UTF-8"); Files.copy(warcRecord, tmp, StandardCopyOption.REPLACE_EXISTING); } } catch (IOException e) { writeStatus(r, null, headers, 0L, isTruncated, FETCH_STATUS.FETCHED_IO_EXCEPTION_READING_ENTITY, writer); deleteTmp(tmp); return; } String digest = null; long tmpLength = 0l; try (InputStream is = Files.newInputStream(tmp)) { digest = base32.encodeAsString(DigestUtils.sha1(is)); tmpLength = Files.size(tmp); } catch (IOException e) { writeStatus(r, null, headers, tmpLength, isTruncated, FETCH_STATUS.FETCHED_IO_EXCEPTION_SHA1, writer); logger.warn("IOException during digesting: " + tmp.toAbsolutePath()); deleteTmp(tmp); return; } if (Files.exists(targFile)) { writeStatus(r, digest, headers, tmpLength, isTruncated, FETCH_STATUS.ALREADY_IN_REPOSITORY, writer); deleteTmp(tmp); return; } try { Files.createDirectories(targFile.getParent()); Files.copy(tmp, targFile); } catch (IOException e) { writeStatus(r, digest, headers, tmpLength, isTruncated, FETCH_STATUS.FETCHED_EXCEPTION_COPYING_TO_REPOSITORY, writer); deleteTmp(tmp); } writeStatus(r, digest, headers, tmpLength, isTruncated, FETCH_STATUS.ADDED_TO_REPOSITORY, writer); deleteTmp(tmp); }
From source file:com.buffalokiwi.api.API.java
/** * Retrieves the status and version number information from the response * @param response Response to pull data from *//* w ww.j av a 2 s .c o m*/ private IAPIResponse createResponseObject(final HttpResponse response, final byte[] content, final String charset) { final RedirectLocations locations = ((RedirectLocations) context .getAttribute(HttpClientContext.REDIRECT_LOCATIONS)); final List<URI> redirectLocations = new ArrayList<>(); if (locations != null) { redirectLocations.addAll(locations.getAll()); } return new APIResponse(response.getProtocolVersion(), response.getStatusLine(), new ArrayList<>(Arrays.asList(response.getAllHeaders())), redirectLocations, content, charset); }