Example usage for com.google.common.util.concurrent Futures allAsList

List of usage examples for com.google.common.util.concurrent Futures allAsList

  1. HOME
  2. Java
  3. com.google.common
  4. com.google.common.util.concurrent.*
  5. Futures
  6. allAsList

Introduction

In this page you can find the example usage for com.google.common.util.concurrent Futures allAsList.

Prototype

@Beta
@CheckReturnValue
public static <V> ListenableFuture<List<V>> allAsList(
        Iterable<? extends ListenableFuture<? extends V>> futures)

Source Link

Document

Creates a new ListenableFuture whose value is a list containing the values of all its input futures, if all succeed.

Usage

From source file:org.apache.hive.ptest.execution.ExecutionPhase.java

@Override
public void execute() throws Throwable {
    long start = System.currentTimeMillis();
    List<TestBatch> testBatches = Lists.newArrayList();
    for (TestBatch batch : testBatchSupplier.get()) {
        testBatches.add(batch);/*ww  w  . j a  v  a2  s  .  co  m*/
        if (batch.isParallel()) {
            parallelWorkQueue.add(batch);
        } else {
            isolatedWorkQueue.add(batch);
        }
    }
    logger.info("ParallelWorkQueueSize={}, IsolatedWorkQueueSize={}", parallelWorkQueue.size(),
            isolatedWorkQueue.size());
    if (logger.isDebugEnabled()) {
        for (TestBatch testBatch : parallelWorkQueue) {
            logger.debug("PBatch: {}", testBatch);
        }
        for (TestBatch testBatch : isolatedWorkQueue) {
            logger.debug("IBatch: {}", testBatch);
        }
    }
    try {
        int expectedNumHosts = hostExecutors.size();
        initalizeHosts();
        do {
            replaceBadHosts(expectedNumHosts);
            List<ListenableFuture<Void>> results = Lists.newArrayList();
            for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
                results.add(hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults));
            }
            Futures.allAsList(results).get();
        } while (!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty()));
        for (TestBatch batch : testBatches) {
            File batchLogDir;
            if (failedTestResults.contains(batch)) {
                batchLogDir = new File(failedLogDir, batch.getName());
            } else {
                batchLogDir = new File(succeededLogDir, batch.getName());
            }
            JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir);
            executedTests.addAll(parser.getAllExecutedTests());
            for (String failedTest : parser.getAllFailedTests()) {
                failedTests.add(failedTest + " (batchId=" + batch.getBatchId() + ")");
            }

            // if the TEST*.xml was not generated or was corrupt, let someone know
            if (parser.getTestClassesWithReportAvailable().size() < batch.getTestClasses().size()) {
                Set<String> expTestClasses = new HashSet<>(batch.getTestClasses());
                expTestClasses.removeAll(parser.getTestClassesWithReportAvailable());
                for (String testClass : expTestClasses) {
                    StringBuilder messageBuilder = new StringBuilder();
                    messageBuilder.append(testClass)
                            .append(" - did not produce a TEST-*.xml file (likely timed out)")
                            .append(" (batchId=").append(batch.getBatchId()).append(")");
                    if (batch instanceof QFileTestBatch) {
                        Collection<String> tests = ((QFileTestBatch) batch).getTests();
                        if (tests.size() != 0) {
                            messageBuilder.append("\n\t[");
                            messageBuilder.append(Joiner.on(",").join(tests));
                            messageBuilder.append("]");
                        }
                    }
                    failedTests.add(messageBuilder.toString());
                }
            }
        }
    } finally {
        long elapsed = System.currentTimeMillis() - start;
        logger.info(
                "PERF: exec phase " + TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS) + " minutes");
    }
}

From source file:info.archinnov.achilles.internal.async.AsyncUtils.java

public ListenableFuture<List<ResultSet>> mergeResultSetFutures(
        List<ListenableFuture<ResultSet>> resultSetFutures) {
    return Futures.allAsList(resultSetFutures);
}

From source file:org.thingsboard.server.dao.timeseries.BaseTimeseriesService.java

@Override
public ListenableFuture<List<TsKvEntry>> findLatest(TenantId tenantId, EntityId entityId,
        Collection<String> keys) {
    validate(entityId);//from   w  ww  . java  2  s .  com
    List<ListenableFuture<TsKvEntry>> futures = Lists.newArrayListWithExpectedSize(keys.size());
    keys.forEach(key -> Validator.validateString(key, "Incorrect key " + key));
    if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) {
        EntityView entityView = entityViewService.findEntityViewById(tenantId, (EntityViewId) entityId);
        List<String> filteredKeys = new ArrayList<>(keys);
        if (entityView.getKeys() != null && entityView.getKeys().getTimeseries() != null
                && !entityView.getKeys().getTimeseries().isEmpty()) {
            filteredKeys.retainAll(entityView.getKeys().getTimeseries());
        }
        List<ReadTsKvQuery> queries = filteredKeys.stream().map(key -> {
            long endTs = entityView.getEndTimeMs() != 0 ? entityView.getEndTimeMs() : Long.MAX_VALUE;
            return new BaseReadTsKvQuery(key, entityView.getStartTimeMs(), endTs, 1, "DESC");
        }).collect(Collectors.toList());

        if (queries.size() > 0) {
            return timeseriesDao.findAllAsync(tenantId, entityView.getEntityId(), queries);
        } else {
            return Futures.immediateFuture(new ArrayList<>());
        }
    }
    keys.forEach(key -> futures.add(timeseriesDao.findLatest(tenantId, entityId, key)));
    return Futures.allAsList(futures);
}

From source file:org.opendaylight.distributed.tx.it.provider.datawriter.DtxDataStoreAsyncWriter.java

/**
 * Asynchronously write to datastore with distributed-tx API
 *//*  w  ww .j  a v a  2 s  . c  o m*/
@Override
public void writeData() {
    int putsPerTx = input.getPutsPerTx();
    int counter = 0;

    InstanceIdentifier<DatastoreTestData> nodeId = InstanceIdentifier.create(DatastoreTestData.class);
    List<ListenableFuture<Void>> putFutures = new ArrayList<ListenableFuture<Void>>((int) putsPerTx);
    List<OuterList> outerLists = dataStoreListBuilder.buildOuterList();

    if (input.getOperation() == OperationType.DELETE) {
        dataStoreListBuilder.buildTestInnerList();
    }

    dtx = dTxProvider.newTx(nodesMap);
    startTime = System.nanoTime();
    for (OuterList outerList : outerLists) {
        for (InnerList innerList : outerList.getInnerList()) {
            InstanceIdentifier<InnerList> innerIid = InstanceIdentifier.create(DatastoreTestData.class)
                    .child(OuterList.class, outerList.getKey()).child(InnerList.class, innerList.getKey());

            CheckedFuture<Void, DTxException> writeFuture;
            if (input.getOperation() == OperationType.PUT) {
                writeFuture = dtx.putAndRollbackOnFailure(DTXLogicalTXProviderType.DATASTORE_TX_PROVIDER,
                        LogicalDatastoreType.CONFIGURATION, innerIid, innerList, nodeId);
            } else if (input.getOperation() == OperationType.MERGE) {
                writeFuture = dtx.mergeAndRollbackOnFailure(DTXLogicalTXProviderType.DATASTORE_TX_PROVIDER,
                        LogicalDatastoreType.CONFIGURATION, innerIid, innerList, nodeId);
            } else {
                writeFuture = dtx.deleteAndRollbackOnFailure(DTXLogicalTXProviderType.DATASTORE_TX_PROVIDER,
                        LogicalDatastoreType.CONFIGURATION, innerIid, nodeId);
            }
            putFutures.add(writeFuture);
            counter++;

            if (counter == putsPerTx) {
                //Aggregate all the put futures into a listenable future which can ensure all asynchronous writes has been finished
                ListenableFuture<Void> aggregatePutFuture = Futures.transform(Futures.allAsList(putFutures),
                        new Function<List<Void>, Void>() {
                            @Nullable
                            @Override
                            public Void apply(@Nullable List<Void> voids) {
                                return null;
                            }
                        });

                try {
                    aggregatePutFuture.get();
                    CheckedFuture<Void, TransactionCommitFailedException> submitFuture = dtx.submit();
                    try {
                        submitFuture.checkedGet();
                        txSucceed++;
                    } catch (TransactionCommitFailedException e) {
                        txError++;
                    }
                } catch (Exception e) {
                    txError++;
                    dtx.cancel();
                }

                counter = 0;
                dtx = dTxProvider.newTx(nodesMap);
                putFutures = new ArrayList<ListenableFuture<Void>>(putsPerTx);
            }
        }
    }

    ListenableFuture<Void> aggregatePutFuture = Futures.transform(Futures.allAsList(putFutures),
            new Function<List<Void>, Void>() {
                @Nullable
                @Override
                public Void apply(@Nullable List<Void> voids) {
                    return null;
                }
            });

    try {
        aggregatePutFuture.get();
        CheckedFuture<Void, TransactionCommitFailedException> restSubmitFuture = dtx.submit();
        try {
            restSubmitFuture.checkedGet();
            txSucceed++;
        } catch (Exception e) {
            txError++;
        }
    } catch (Exception e) {
        txError++;
    }
    endTime = System.nanoTime();
}

From source file:org.thingsboard.server.dao.sql.timeseries.JpaTimeseriesDao.java

@Override
public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId,
        List<ReadTsKvQuery> queries) {
    List<ListenableFuture<List<TsKvEntry>>> futures = queries.stream()
            .map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList());
    return Futures.transform(Futures.allAsList(futures),
            new Function<List<List<TsKvEntry>>, List<TsKvEntry>>() {
                @Nullable//from ww  w.  ja va 2  s  .  c  o  m
                @Override
                public List<TsKvEntry> apply(@Nullable List<List<TsKvEntry>> results) {
                    if (results == null || results.isEmpty()) {
                        return null;
                    }
                    return results.stream().flatMap(List::stream).collect(Collectors.toList());
                }
            }, service);
}

From source file:com.google.caliper.runner.target.ProxyConnectionService.java

private void waitForAllVmsToExit(long timeout, TimeUnit unit) {
    if (vms.isEmpty()) {
        return;//from  w w  w .j a  va 2 s.  c om
    }

    List<ListenableFuture<?>> exitFutures = new ArrayList<>();
    for (VmProxy vm : vms.values()) {
        exitFutures.add(vm.exitCode);
    }

    ListenableFuture<?> allExited = Futures.allAsList(exitFutures);
    try {
        allExited.get(timeout, unit);
    } catch (Exception ignore) {
        // oh well
        return;
    }

    // ensure the shutdown hooks are removed
    for (VmProxy vm : vms.values()) {
        try {
            vm.awaitExit();
        } catch (InterruptedException e) {
            throw new AssertionError(e);
        }
    }
}

From source file:com.facebook.buck.slb.ClientSideSlb.java

private void backgroundThreadCallForHealthCheck() {

    LOG.verbose("Starting pings. %s", toString());

    List<ListenableFuture<PerServerPingData>> futures = new ArrayList<>();
    for (URI serverUri : serverPool) {
        ServerPing serverPing = new ServerPing(serverUri);
        futures.add(serverPing.getFuture());
    }/*from ww w . j  a v a 2s . c om*/

    // Wait for all executions to complete or fail.
    try {
        List<PerServerPingData> allServerData = Futures.allAsList(futures).get();
        LoadBalancerPingEventData.Builder eventData = LoadBalancerPingEventData.builder();
        eventData.addAllPerServerData(allServerData);
        eventBus.post(new LoadBalancerPingEvent(eventData.build()));
        LOG.verbose("all pings complete %s", toString());
    } catch (InterruptedException ex) {
        LOG.verbose("pings interrupted");
    } catch (ExecutionException ex) {
        LOG.verbose(ex, "some pings failed");
    }
}

From source file:com.facebook.presto.execution.TaskExecutorSimulator.java

public void run() throws Exception {
    Multimap<Integer, SimulationTask> tasks = Multimaps
            .synchronizedListMultimap(ArrayListMultimap.<Integer, SimulationTask>create());
    Set<ListenableFuture<?>> finishFutures = newConcurrentHashSet();
    AtomicBoolean done = new AtomicBoolean();

    long start = System.nanoTime();

    // large tasks
    for (int userId = 0; userId < 2; userId++) {
        ListenableFuture<?> future = createUser("large_" + userId, 100, taskExecutor, done, tasks);
        finishFutures.add(future);/*  www  . j a  v  a  2  s  .  co  m*/
    }

    // small tasks
    for (int userId = 0; userId < 4; userId++) {
        ListenableFuture<?> future = createUser("small_" + userId, 5, taskExecutor, done, tasks);
        finishFutures.add(future);
    }

    // tiny tasks
    for (int userId = 0; userId < 1; userId++) {
        ListenableFuture<?> future = createUser("tiny_" + userId, 1, taskExecutor, done, tasks);
        finishFutures.add(future);
    }

    // warm up
    for (int i = 0; i < 30; i++) {
        TimeUnit.MILLISECONDS.sleep(1000);
        System.out.println(taskExecutor);
    }
    tasks.clear();

    // run
    for (int i = 0; i < 60; i++) {
        TimeUnit.MILLISECONDS.sleep(1000);
        System.out.println(taskExecutor);
    }

    // capture finished tasks
    Map<Integer, Collection<SimulationTask>> middleTasks;
    synchronized (tasks) {
        middleTasks = new TreeMap<>(tasks.asMap());
    }

    // wait for finish
    done.set(true);
    Futures.allAsList(finishFutures).get(1, TimeUnit.MINUTES);

    Duration runtime = Duration.nanosSince(start).convertToMostSuccinctTimeUnit();
    synchronized (this) {
        System.out.println();
        System.out.println("Simulation finished in  " + runtime);
        System.out.println();

        for (Entry<Integer, Collection<SimulationTask>> entry : middleTasks.entrySet()) {
            Distribution durationDistribution = new Distribution();
            Distribution taskParallelismDistribution = new Distribution();

            for (SimulationTask task : entry.getValue()) {
                long taskStart = Long.MAX_VALUE;
                long taskEnd = 0;
                long totalCpuTime = 0;

                for (SimulationSplit split : task.getSplits()) {
                    taskStart = Math.min(taskStart, split.getStartNanos());
                    taskEnd = Math.max(taskEnd, split.getDoneNanos());
                    totalCpuTime += TimeUnit.MILLISECONDS.toNanos(split.getRequiredProcessMillis());
                }

                Duration taskDuration = new Duration(taskEnd - taskStart, NANOSECONDS)
                        .convertTo(TimeUnit.MILLISECONDS);
                durationDistribution.add(taskDuration.toMillis());

                double taskParallelism = 1.0 * totalCpuTime / (taskEnd - taskStart);
                taskParallelismDistribution.add((long) (taskParallelism * 100));
            }

            System.out.println("Splits " + entry.getKey() + ": Completed " + entry.getValue().size());

            Map<Double, Long> durationPercentiles = durationDistribution.getPercentiles();
            System.out.printf(
                    "   wall time ms :: p01 %4s :: p05 %4s :: p10 %4s :: p97 %4s :: p50 %4s :: p75 %4s :: p90 %4s :: p95 %4s :: p99 %4s\n",
                    durationPercentiles.get(0.01), durationPercentiles.get(0.05), durationPercentiles.get(0.10),
                    durationPercentiles.get(0.25), durationPercentiles.get(0.50), durationPercentiles.get(0.75),
                    durationPercentiles.get(0.90), durationPercentiles.get(0.95),
                    durationPercentiles.get(0.99));

            Map<Double, Long> parallelismPercentiles = taskParallelismDistribution.getPercentiles();
            System.out.printf(
                    "    parallelism :: p99 %4.2f :: p95 %4.2f :: p90 %4.2f :: p75 %4.2f :: p50 %4.2f :: p25 %4.2f :: p10 %4.2f :: p05 %4.2f :: p01 %4.2f\n",
                    parallelismPercentiles.get(0.99) / 100.0, parallelismPercentiles.get(0.95) / 100.0,
                    parallelismPercentiles.get(0.90) / 100.0, parallelismPercentiles.get(0.75) / 100.0,
                    parallelismPercentiles.get(0.50) / 100.0, parallelismPercentiles.get(0.25) / 100.0,
                    parallelismPercentiles.get(0.10) / 100.0, parallelismPercentiles.get(0.05) / 100.0,
                    parallelismPercentiles.get(0.01) / 100.0);
        }
    }
    Thread.sleep(10);
}

From source file:com.google.idea.blaze.cpp.BlazeConfigurationResolver.java

private static ImmutableMap<File, VirtualFile> doCollectHeaderRoots(BlazeContext context,
        BlazeProjectData projectData, Set<ExecutionRootPath> rootPaths) {
    ConcurrentMap<File, VirtualFile> rootsMap = Maps.newConcurrentMap();
    List<ListenableFuture<Void>> futures = Lists.newArrayListWithCapacity(rootPaths.size());
    for (ExecutionRootPath path : rootPaths) {
        futures.add(submit(() -> {// ww w . ja v  a 2  s  .  co m
            ImmutableList<File> possibleDirectories = projectData.workspacePathResolver
                    .resolveToIncludeDirectories(path);
            for (File file : possibleDirectories) {
                VirtualFile vf = getVirtualFile(file);
                if (vf != null) {
                    rootsMap.put(file, vf);
                } else if (!projectData.blazeRoots.isOutputArtifact(path)
                        && FileAttributeProvider.getInstance().exists(file)) {
                    // If it's not a blaze output file, we expect it to always resolve.
                    LOG.info(String.format("Unresolved header root %s", file.getAbsolutePath()));
                }
            }
            return null;
        }));
    }
    try {
        Futures.allAsList(futures).get();
        return ImmutableMap.copyOf(rootsMap);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        context.setCancelled();
    } catch (ExecutionException e) {
        IssueOutput.error("Error resolving header include roots: " + e).submit(context);
        LOG.error("Error resolving header include roots", e);
    }
    return ImmutableMap.of();
}

From source file:com.facebook.buck.autodeps.AutodepsWriter.java

/**
 * Writes the {@code .autodeps} files in parallel using the {@code commandThreadManager}.
 * @param depsForBuildFiles Abstraction that contains the data that needs to be written to the
 *     {@code .autodeps} files./*from  w  w w . j  a  v a2 s . c  o m*/
 * @param buildFileName In practice, this should be derived from
 *     {@link com.facebook.buck.rules.Cell#getBuildFileName()}
 * @param includeSignature Whether to insert a signature for the contents of the file.
 * @param mapper To aid in JSON serialization.
 * @return the number of files that were written.
 */
public static int write(DepsForBuildFiles depsForBuildFiles, String buildFileName, boolean includeSignature,
        ObjectMapper mapper, ListeningExecutorService executorService, int numThreads)
        throws ExecutionException {
    Preconditions.checkArgument(numThreads > 0, "Must be at least one thread available");

    // We are going to divide the work into N groups, where N is the size of the thread pool.
    ImmutableList<DepsForBuildFiles.BuildFileWithDeps> buildFilesWithDeps = ImmutableList
            .copyOf(depsForBuildFiles);
    int numBuildFiles = buildFilesWithDeps.size();
    if (numBuildFiles == 0) {
        return 0;
    }
    int chunkSize = numBuildFiles / numThreads;
    int extraItems = numBuildFiles % numThreads;

    // Add the work to the executor. Note that instead of creating one future per build file, we
    // create one future per thread. This should reduce object allocation and context switching.
    List<ListenableFuture<Integer>> futures = new ArrayList<>();
    String autodepsFileName = buildFileName + GENERATED_BUILD_FILE_SUFFIX;
    for (int i = 0, endIndex = 0; i < numThreads; i++) {
        // Calculate how many items the thread should process.
        int numItemsToProcess = chunkSize;
        if (extraItems > 0) {
            numItemsToProcess++;
            extraItems--;
        }

        // Note that if buildFilesWithDeps.size() < numThreads, then this will be true for some
        // iterations of this loop.
        if (numItemsToProcess == 0) {
            break;
        }

        // Determine the subset of buildFilesWithDeps for the thread to process.
        int startIndex = endIndex;
        endIndex = startIndex + numItemsToProcess;
        ImmutableList<DepsForBuildFiles.BuildFileWithDeps> work = buildFilesWithDeps.subList(startIndex,
                endIndex);

        // Submit a job to the executor that will write .autodeps files, as appropriate. It will
        // return the number of .autodeps files it needed to write.
        ListenableFuture<Integer> future = executorService
                .submit(new AutodepsCallable(work, autodepsFileName, includeSignature, mapper));
        futures.add(future);
    }

    // Sum up the total number of files written from each worker.
    int totalWritten = 0;
    ListenableFuture<List<Integer>> futuresList = Futures.allAsList(futures);
    for (int numWritten : Uninterruptibles.getUninterruptibly(futuresList)) {
        totalWritten += numWritten;
    }
    return totalWritten;
}