List of usage examples for org.apache.http.nio.client.methods ZeroCopyConsumer ZeroCopyConsumer
public ZeroCopyConsumer(final File file) throws FileNotFoundException
From source file:httpasync.ZeroCopyHttpExchange.java
public static void main(final String[] args) throws Exception { CloseableHttpAsyncClient httpclient = HttpAsyncClients.createDefault(); try {//from w ww . ja v a 2 s . c o m httpclient.start(); File upload = new File(args[0]); File download = new File(args[1]); ZeroCopyPost httpost = new ZeroCopyPost("http://localhost:8080/", upload, ContentType.create("text/plain")); ZeroCopyConsumer<File> consumer = new ZeroCopyConsumer<File>(download) { @Override protected File process(final HttpResponse response, final File file, final ContentType contentType) throws Exception { if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { throw new ClientProtocolException("Upload failed: " + response.getStatusLine()); } return file; } }; Future<File> future = httpclient.execute(httpost, consumer, null); File result = future.get(); System.out.println("Response file length: " + result.length()); System.out.println("Shutting down"); } finally { httpclient.close(); } System.out.println("Done"); }
From source file:com.boonya.http.async.examples.nio.client.ZeroCopyHttpExchange.java
public static void main(final String[] args) throws Exception { CloseableHttpAsyncClient httpclient = HttpAsyncClients.createDefault(); try {/*ww w. java 2 s . c o m*/ httpclient.start(); File upload = new File(args[0]); File download = new File(args[1]); ZeroCopyPost httpost = new ZeroCopyPost("http://localhost:8080/", upload, ContentType.create("text/plain")); ZeroCopyConsumer<File> consumer = new ZeroCopyConsumer<File>(download) { @Override protected File process(final HttpResponse response, final File file, final ContentType contentType) throws Exception { if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { throw new ClientProtocolException("Upload failed: " + response.getStatusLine()); } return file; } }; Future<File> future = httpclient.execute(httpost, consumer, null); File result = future.get(); System.out.println("Response file length: " + result.length()); System.out.println("Shutting down"); } finally { httpclient.close(); } System.out.println("Done"); }
From source file:org.grycap.gpf4med.DownloadService.java
/** * Uses a group of URIs to retrieve objects and writes them to the same number of files. This method will do * its best effort to optimally handle the downloads, opening a pool of connections to the servers and reusing * them as much as possible. Also, it will create several concurrent threads in the JVM in order to perform * simultaneous downloads.// w w w . j av a 2 s.c o m * @param requests a key-value map with the list of requests to handle. The source of the object is the key of * the map, while the value is the destination file. * @param validator checks the file for correctness. * @param config download settings. * @param encryptionProvider an optional encryption provider that, when available, is used to encrypt the * files after download. * @param task an optional task that will be executed passing each individual file as parameter, when the download * of the file ends. * @return the requests that could not be served after exhausting the individual retries. * @throws IOException if an error occurs in the execution of the operation. */ public ImmutableMap<URI, File> download(final ImmutableMap<URI, File> requests, final @Nullable FileValidator validator, final DownloadConfiguration config, final @Nullable FileEncryptionProvider encryptionProvider, final @Nullable PostProcessTask<File> task) throws IOException { checkArgument(requests != null, "Uninitialized request"); checkArgument(config != null, "Uninitialized configuration"); ImmutableMap<URI, File> pending = ImmutableMap.copyOf(requests); final List<URI> cancelled = new ArrayList<URI>(); try { for (int attempt = 0; attempt < config.getRetries() && !pending.isEmpty() && pending.size() > cancelled.size(); attempt++) { LOGGER.info("Attempt " + (attempt + 1) + " to download " + requests.size() + " files"); // create connection manager final PoolingNHttpClientConnectionManager connectionManager = createConnectionManager(); // create HTTP asynchronous client int eSoTimeoutMs = config.soToMs + (int) (config.soToMs * attempt * (config.toIncPercent >= 0.0d && config.toIncPercent <= 1.0d ? config.toIncPercent : 0.0d)); int eConnTimeoutMs = config.connToMs + (int) (config.connToMs * attempt * (config.toIncPercent >= 0.0d && config.toIncPercent <= 1.0d ? config.toIncPercent : 0.0d)); final RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(eConnTimeoutMs) .setConnectionRequestTimeout(eConnTimeoutMs).setSocketTimeout(eSoTimeoutMs).build(); final CloseableHttpAsyncClient httpclient = HttpAsyncClients.custom() .setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).build(); httpclient.start(); // attempt to perform download try { final CountDownLatch latch = new CountDownLatch(pending.size()); for (final Map.Entry<URI, File> entry : pending.entrySet()) { final URI uri = entry.getKey(); if (cancelled.contains(uri)) { continue; } final File file = entry.getValue(); FileUtils.forceMkdir(file.getParentFile()); final HttpGet request = new HttpGet(uri); final HttpAsyncRequestProducer producer = new BasicAsyncRequestProducer( new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme()), request); final ZeroCopyConsumer<File> consumer = new ZeroCopyConsumer<File>(file) { @Override protected File process(final HttpResponse response, final File file, final ContentType contentType) throws Exception { releaseResources(); if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { FileUtils.deleteQuietly(file); throw new ClientProtocolException( "Download failed: " + response.getStatusLine()); } if (validator != null && !validator.isValid(file)) { FileUtils.deleteQuietly(file); cancelled.add(uri); throw new IOException( file.getCanonicalPath() + " not recognised as a supported file format"); } if (encryptionProvider != null) { try { final File cipherFile = File .createTempFile(RandomStringUtils.random(8, true, true), ".tmp"); encryptionProvider.encrypt(new FileInputStream(file), new FileOutputStream(cipherFile)); FileUtils.deleteQuietly(file); FileUtils.moveFile(cipherFile, file); LOGGER.info("File encrypted: " + file.getCanonicalPath()); } catch (Exception e) { FileUtils.deleteQuietly(file); cancelled.add(uri); LOGGER.warn("Failed to encrypt: " + file.getCanonicalPath(), e); throw new IOException("File encryption failed"); } } LOGGER.info("Download succeed to file: " + file.getCanonicalPath()); return file; } }; httpclient.execute(producer, consumer, new FutureCallback<File>() { @Override public void completed(final File result) { request.releaseConnection(); latch.countDown(); if (task != null) { task.apply(result); } LOGGER.info("Request succeed: " + request.getRequestLine() + " => Response file length: " + result.length()); } @Override public void failed(final Exception ex) { request.releaseConnection(); FileUtils.deleteQuietly(file); latch.countDown(); LOGGER.error("Request failed: " + request.getRequestLine() + "=>" + ex); } @Override public void cancelled() { request.releaseConnection(); FileUtils.deleteQuietly(file); latch.countDown(); LOGGER.error("Request cancelled: " + request.getRequestLine()); } }); } latch.await(); } finally { try { httpclient.close(); } catch (Exception ignore) { } try { shutdown(connectionManager, 0l); } catch (Exception ignore) { } } // populate the pending list with the files that does not exist final ImmutableMap.Builder<URI, File> builder = new ImmutableMap.Builder<URI, File>(); for (final Map.Entry<URI, File> entry : requests.entrySet()) { if (!entry.getValue().exists()) { builder.put(entry.getKey(), entry.getValue()); } } pending = builder.build(); if ((attempt + 1) < config.retries && !pending.isEmpty() && pending.size() > cancelled.size()) { final long waitingTime = (long) (config.soToMs * 0.1d); LOGGER.info("Waiting " + waitingTime + " ms before attempt " + (attempt + 2) + " to download " + requests.size() + " pending files"); Thread.sleep(waitingTime); } } } catch (IOException ioe) { throw ioe; } catch (Exception e) { throw new IOException("Download has failed", e); } return pending; }
From source file:es.upv.grycap.coreutils.fiber.http.HttpDataFetcher.java
/** * Allows fetching and saving a bunch of objects to the specified directory from a server that uses a REST or REST-like API * where each object is retrieved from the URL formed appending the object's identifier to the path of the the base URL, and * optionally from a server that uses a parameter to identify the objects. Supports additional configuration options to name * the fetched objects.//from w w w.j av a 2s. c om * @param baseUrl - base URL from where the objects will be fetched * @param queryParam - if defined, a query parameter will be appended to the base URL with the identifier of the request * @param ids - a list with the identifiers of the all requests that will be attempted * @param prefix - optionally prepend this prefix to the filenames of the saved files * @param suffix - optionally append this suffix to the filenames of the saved files * @param outdir - directory where the files will be stored * @return A {@link CompletableFuture} that allows cancellation. Once each fetch operation is completed, its status is updated * in the future with one of the possible values provided by the enumeration {@link FetchStatus}. * @throws IOException If an error occurs during the execution of the method that prevents fetching or saving the files. */ public FecthFuture fetchToDir(final URL baseUrl, final @Nullable String queryParam, final List<String> ids, final @Nullable String prefix, final @Nullable String suffix, final File outdir) throws IOException { // check mandatory parameters requireNonNull(baseUrl, "A valid URL expected"); final FecthFuture toBeCompleted = new FecthFuture( requireNonNull(ids, "A valid list of identifiers expected").stream().map(StringUtils::trimToNull) .filter(Objects::nonNull).distinct().collect(Collectors.toList())); requireNonNull(outdir, "A valid output directory expected"); checkArgument((outdir.isDirectory() && outdir.canWrite()) || outdir.mkdirs(), new StringBuilder("Cannot write to the output directory: ").append(outdir.getAbsolutePath()) .toString()); // get optional parameters final Optional<String> queryParam2 = ofNullable(trimToNull(queryParam)); final String prefix2 = ofNullable(prefix).orElse(""); final String suffix2 = ofNullable(suffix).orElse(""); try (final CloseableHttpAsyncClient asyncHttpClient = createFiberCloseableHttpAsyncClient()) { asyncHttpClient.start(); final UrlBuilder urlBuilder = getUrlBuilder(baseUrl); // an explanation is needed since this code is instrumented by Quasar and Comsat: requests are created during the first part of // this lambda expression (map), but they are not executed until the get() method is called in the second part of the expression // (forEach). Here that parallel stream is used to block and wait for the requests to complete. In case that a single stream is // used, each request will be created and executed sequentially. Therefore, the alternative to parallel stream is to separate // the lambda expression in two loops, creating the requests in the first loop and calling get() in the second one. toBeCompleted.monList.parallelStream().map(m -> { try { // create output file final File outfile = new File(outdir, new StringBuilder(prefix2).append(m.id).append(suffix2).append(".partial").toString()); checkState(outfile.createNewFile(), new StringBuilder("Cannot create the output file: ") .append(outfile.getAbsolutePath()).toString()); // create the HTTP request final HttpHost target = URIUtils.extractHost(baseUrl.toURI()); final HttpRequest request = new BasicHttpRequest("GET", urlBuilder.buildRelativeUrl(queryParam2.isPresent() ? null : m.id, queryParam2.isPresent() ? of(queryParam2.get(), m.id) : null)); final HttpAsyncRequestProducer producer = new BasicAsyncRequestProducer(target, request); // create the consumer final ZeroCopyConsumer<File> consumer = new ZeroCopyConsumer<File>(outfile) { @Override protected File process(final HttpResponse response, final File file, final ContentType contentType) throws Exception { final StatusLine status = response.getStatusLine(); if (LOGGER.isDebugEnabled()) LOGGER.debug( new StringBuilder("Got file: statusCode=").append(status.getStatusCode()) .append(", file=").append(file.getAbsolutePath()).toString()); if (status.getStatusCode() != HttpStatus.SC_OK) throw new ClientProtocolException( new StringBuilder("Object fetch failed: ").append(status).toString()); return file; } }; // prepare request m.future = asyncHttpClient.execute(producer, consumer, new FutureCallback<File>() { @Override public void cancelled() { toBeCompleted.update(m.id, FetchStatus.CANCELLED); LOGGER.info("Task cancelled"); } @Override public void completed(final File result) { try { final Path path = result.toPath(); Files.move(path, path.resolveSibling(removeEnd(result.getName(), ".partial")), REPLACE_EXISTING); toBeCompleted.update(m.id, FetchStatus.COMPLETED); } catch (IOException ex) { toBeCompleted.update(m.id, FetchStatus.FAILED); LOGGER.error("Fecth failed to move file to its final destination with error", ex); } } @Override public void failed(final Exception ex) { toBeCompleted.update(m.id, FetchStatus.FAILED); LOGGER.error("Fecth failed with error", ex); } }); } catch (Exception e) { LOGGER.error(new StringBuilder("Failed to fetch object with id: ").append(m.id).toString(), e); } return m; }).forEach(m -> { try { // submit requests and wait for completion m.future.get(); } catch (Exception ignore) { /* exceptions are handled in the callback functions */ } }); } return toBeCompleted; }