List of usage examples for com.google.common.util.concurrent Futures transformAsync
public static <I, O> ListenableFuture<O> transformAsync(ListenableFuture<I> input, AsyncFunction<? super I, ? extends O> function, Executor executor)
From source file:com.facebook.buck.distributed.DistBuildService.java
public ListenableFuture<Void> uploadBuckDotFiles(final BuildId id, final ProjectFilesystem filesystem, FileHashCache fileHashCache, ListeningExecutorService executorService) throws IOException { ListenableFuture<Pair<List<FileInfo>, List<PathInfo>>> filesFuture = executorService .submit(new Callable<Pair<List<FileInfo>, List<PathInfo>>>() { @Override/*from w w w. j av a2s. c o m*/ public Pair<List<FileInfo>, List<PathInfo>> call() throws IOException { Path[] buckDotFilesExceptConfig = Arrays.stream(filesystem.listFiles(Paths.get("."))) .filter(f -> !f.isDirectory()).filter(f -> !Files.isSymbolicLink(f.toPath())) .filter(f -> f.getName().startsWith(".")).filter(f -> f.getName().contains("buck")) .filter(f -> !f.getName().startsWith(".buckconfig")).map(f -> f.toPath()) .toArray(Path[]::new); List<FileInfo> fileEntriesToUpload = new LinkedList<>(); List<PathInfo> pathEntriesToUpload = new LinkedList<>(); for (Path path : buckDotFilesExceptConfig) { FileInfo fileInfoObject = new FileInfo(); fileInfoObject.setContent(filesystem.readFileIfItExists(path).get().getBytes()); fileInfoObject.setContentHash(fileHashCache.get(path.toAbsolutePath()).toString()); fileEntriesToUpload.add(fileInfoObject); PathInfo pathInfoObject = new PathInfo(); pathInfoObject.setPath(path.toString()); pathInfoObject.setContentHash(fileHashCache.get(path.toAbsolutePath()).toString()); pathEntriesToUpload.add(pathInfoObject); } return new Pair<List<FileInfo>, List<PathInfo>>(fileEntriesToUpload, pathEntriesToUpload); } }); ListenableFuture<Void> setFilesFuture = Futures.transformAsync(filesFuture, new AsyncFunction<Pair<List<FileInfo>, List<PathInfo>>, Void>() { @Override public ListenableFuture<Void> apply( @Nullable Pair<List<FileInfo>, List<PathInfo>> filesAndPaths) throws IOException { setBuckDotFiles(id, filesAndPaths.getSecond()); return Futures.immediateFuture(null); } }, executorService); ListenableFuture<Void> uploadFilesFuture = Futures.transformAsync(filesFuture, new AsyncFunction<Pair<List<FileInfo>, List<PathInfo>>, Void>() { @Override public ListenableFuture<Void> apply( @Nullable Pair<List<FileInfo>, List<PathInfo>> filesAndPaths) throws Exception { uploadMissingFilesFromList(filesAndPaths.getFirst(), executorService); return Futures.immediateFuture(null); } }, executorService); return Futures.transform(Futures.allAsList(ImmutableList.of(setFilesFuture, uploadFilesFuture)), new Function<List<Void>, Void>() { @Nullable @Override public Void apply(@Nullable List<Void> input) { return null; } }); }
From source file:com.facebook.buck.core.build.engine.impl.CachingBuildEngine.java
private ListenableFuture<BuildResult> getBuildRuleResultWithRuntimeDepsUnlocked(BuildRule rule, BuildEngineBuildContext buildContext, ExecutionContext executionContext) { // If the rule is already executing, return its result future from the cache. ListenableFuture<BuildResult> existingResult = results.get(rule.getBuildTarget()); if (existingResult != null) { return existingResult; }/*from w w w.ja v a 2 s . c o m*/ // Get the future holding the result for this rule and, if we have no additional runtime deps // to attach, return it. ListenableFuture<RuleKey> ruleKey = calculateRuleKey(rule, buildContext); ListenableFuture<BuildResult> result = Futures.transformAsync(ruleKey, input -> processBuildRule(rule, buildContext, executionContext), serviceByAdjustingDefaultWeightsTo(SCHEDULING_MORE_WORK_RESOURCE_AMOUNTS)); if (!(rule instanceof HasRuntimeDeps)) { results.put(rule.getBuildTarget(), result); return result; } // Collect any runtime deps we have into a list of futures. Stream<BuildTarget> runtimeDepPaths = ((HasRuntimeDeps) rule).getRuntimeDeps(ruleFinder); List<ListenableFuture<BuildResult>> runtimeDepResults = new ArrayList<>(); ImmutableSet<BuildRule> runtimeDeps = resolver .getAllRules(runtimeDepPaths.collect(ImmutableSet.toImmutableSet())); for (BuildRule dep : runtimeDeps) { runtimeDepResults.add(getBuildRuleResultWithRuntimeDepsUnlocked(dep, buildContext, executionContext)); } // Create a new combined future, which runs the original rule and all the runtime deps in // parallel, but which propagates an error if any one of them fails. // It also checks that all runtime deps succeeded. ListenableFuture<BuildResult> chainedResult = Futures.transformAsync(Futures.allAsList(runtimeDepResults), results -> { if (!isKeepGoingEnabled(buildContext)) { for (BuildResult buildResult : results) { if (!buildResult.isSuccess()) { return Futures .immediateFuture(BuildResult.canceled(rule, buildResult.getFailure())); } } } return result; }, MoreExecutors.directExecutor()); results.put(rule.getBuildTarget(), chainedResult); return chainedResult; }
From source file:com.facebook.buck.core.build.engine.impl.CachingBuildRuleBuilder.java
ListenableFuture<BuildResult> build() {
AtomicReference<Long> outputSize = Atomics.newReference();
ListenableFuture<List<BuildResult>> depResults = Futures.immediateFuture(Collections.emptyList());
// If we're performing a deep build, guarantee that all dependencies will *always* get
// materialized locally
if (buildMode == BuildType.DEEP || buildMode == BuildType.POPULATE_FROM_REMOTE_CACHE) {
depResults = buildRuleBuilderDelegate.getDepResults(rule, executionContext);
}/* w w w . j a v a2 s . c o m*/
ListenableFuture<BuildResult> buildResult = Futures.transformAsync(depResults,
input -> buildOrFetchFromCache(),
serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.SCHEDULING_MORE_WORK_RESOURCE_AMOUNTS));
// Check immediately (without posting a new task) for a failure so that we can short-circuit
// pending work. Use .catchingAsync() instead of .catching() so that we can propagate unchecked
// exceptions.
buildResult = Futures.catchingAsync(buildResult, Throwable.class, throwable -> {
Objects.requireNonNull(throwable);
BuildRuleFailedException failedException = getFailedException(throwable);
buildRuleBuilderDelegate.setFirstFailure(failedException);
throw failedException;
});
buildResult = Futures.transform(buildResult, (result) -> {
buildRuleBuilderDelegate.markRuleAsUsed(rule, eventBus);
return result;
}, MoreExecutors.directExecutor());
buildResult = Futures.transformAsync(buildResult,
ruleAsyncFunction(result -> finalizeBuildRule(result, outputSize)),
serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.RULE_KEY_COMPUTATION_RESOURCE_AMOUNTS));
buildResult = Futures.catchingAsync(buildResult, Throwable.class, thrown -> {
String message = String.format("Building rule [%s] failed.", rule.getBuildTarget());
BuildRuleFailedException failedException = getFailedException(thrown);
LOG.debug(failedException, message);
if (consoleLogBuildFailuresInline) {
// TODO(cjhopman): This probably shouldn't be a thing. Why can't we just rely on the
// propagated failure being printed?
eventBus.post(ConsoleEvent.severe(message));
}
recordFailureAndCleanUp(failedException);
return Futures.immediateFuture(failure(thrown));
});
// Do things that need to happen after either success or failure, but don't block the dependents
// while doing so:
buildRuleBuilderDelegate
.addAsyncCallback(MoreFutures.addListenableCallback(buildResult, new FutureCallback<BuildResult>() {
@Override
public void onSuccess(BuildResult input) {
handleResult(input);
// Reset interrupted flag once failure has been recorded.
if (!input.isSuccess() && input.getFailure() instanceof InterruptedException) {
Threads.interruptCurrentThread();
}
}
@Override
public void onFailure(@Nonnull Throwable thrown) {
throw new AssertionError("Dead code", thrown);
}
}, serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.RULE_KEY_COMPUTATION_RESOURCE_AMOUNTS)));
return buildResult;
}
From source file:com.facebook.buck.core.build.engine.impl.CachingBuildEngine.java
public ListenableFuture<?> walkRule(BuildRule rule, Set<BuildRule> seen) { return Futures.transformAsync(Futures.immediateFuture(ruleDeps.get(rule)), deps -> { List<ListenableFuture<?>> results1 = new ArrayList<>(SortedSets.sizeEstimate(deps)); for (BuildRule dep : deps) { if (seen.add(dep)) { results1.add(walkRule(dep, seen)); }/*ww w .ja v a 2 s . c o m*/ } return Futures.allAsList(results1); }, serviceByAdjustingDefaultWeightsTo(SCHEDULING_MORE_WORK_RESOURCE_AMOUNTS)); }
From source file:com.facebook.presto.server.protocol.Query.java
private ListenableFuture<?> queryDoneFuture(QueryState currentState) { if (currentState.isDone()) { return immediateFuture(null); }/*from w w w.ja v a 2 s .c o m*/ return Futures.transformAsync(queryManager.getStateChange(queryId, currentState), this::queryDoneFuture, directExecutor()); }
From source file:org.thingsboard.server.dao.timeseries.CassandraBaseTimeseriesDao.java
@Override public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { ListenableFuture<TsKvEntry> latestEntryFuture = findLatest(tenantId, entityId, query.getKey()); ListenableFuture<Boolean> booleanFuture = Futures.transform(latestEntryFuture, latestEntry -> { long ts = latestEntry.getTs(); if (ts > query.getStartTs() && ts <= query.getEndTs()) { return true; } else {//from w w w. jav a2 s . com log.trace("Won't be deleted latest value for [{}], key - {}", entityId, query.getKey()); } return false; }, readResultsProcessingExecutor); ListenableFuture<Void> removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { if (isRemove) { return deleteLatest(tenantId, entityId, query.getKey()); } return Futures.immediateFuture(null); }, readResultsProcessingExecutor); final SimpleListenableFuture<Void> resultFuture = new SimpleListenableFuture<>(); Futures.addCallback(removedLatestFuture, new FutureCallback<Void>() { @Override public void onSuccess(@Nullable Void result) { if (query.getRewriteLatestIfDeleted()) { ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { if (isRemove) { return getNewLatestEntryFuture(tenantId, entityId, query); } return Futures.immediateFuture(null); }, readResultsProcessingExecutor); try { resultFuture.set(savedLatestFuture.get()); } catch (InterruptedException | ExecutionException e) { log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); } } else { resultFuture.set(null); } } @Override public void onFailure(Throwable t) { log.warn("[{}] Failed to process remove of the latest value", entityId, t); } }); return resultFuture; }
From source file:org.thingsboard.server.dao.timeseries.CassandraBaseTimeseriesDao.java
private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {/*from w w w .ja va 2s .c om*/ long startTs = 0; long endTs = query.getStartTs() - 1; ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, Aggregation.NONE, DESC_ORDER); ListenableFuture<List<TsKvEntry>> future = findAllAsync(tenantId, entityId, findNewLatestQuery); return Futures.transformAsync(future, entryList -> { if (entryList.size() == 1) { return saveLatest(tenantId, entityId, entryList.get(0)); } else { log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); } return Futures.immediateFuture(null); }, readResultsProcessingExecutor); }
From source file:com.facebook.buck.rules.CachingBuildRuleBuilder.java
private ListenableFuture<BuildResult> buildOrFetchFromCache() { // If we've already seen a failure, exit early. if (!buildContext.isKeepGoing() && buildRuleBuilderDelegate.getFirstFailure() != null) { return Futures.immediateFuture(BuildResult.canceled(rule, buildRuleBuilderDelegate.getFirstFailure())); }/*from ww w . j a v a 2 s. c om*/ // 1. Check if it's already built. try (Scope scope = BuildRuleEvent.resumeSuspendScope(buildContext.getEventBus(), rule, buildRuleDurationTracker, ruleKeyFactories.getDefaultRuleKeyFactory())) { Optional<BuildResult> buildResult = checkMatchingLocalKey(); if (buildResult.isPresent()) { return Futures.immediateFuture(buildResult.get()); } } AtomicReference<CacheResult> rulekeyCacheResult = new AtomicReference<>(); ListenableFuture<Optional<BuildResult>> buildResultFuture; // 2. Rule key cache lookup. buildResultFuture = // TODO(cjhopman): This should follow the same, simple pattern as everything else. With a // large ui.thread_line_limit, SuperConsole tries to redraw more lines than are available. // These cache threads make it more likely to hit that problem when SuperConsole is aware // of them. cacheActivityService.withDefaultAmounts(CachingBuildEngine.CACHE_CHECK_RESOURCE_AMOUNTS) .submit(() -> { if (!buildRuleBuilderDelegate.shouldKeepGoing(buildContext)) { Preconditions.checkNotNull(buildRuleBuilderDelegate.getFirstFailure()); return Optional .of(BuildResult.canceled(rule, buildRuleBuilderDelegate.getFirstFailure())); } CacheResult cacheResult = performRuleKeyCacheCheck(); rulekeyCacheResult.set(cacheResult); return getBuildResultForRuleKeyCacheResult(cacheResult); }); // 3. Build deps. buildResultFuture = transformBuildResultAsyncIfNotPresent(buildResultFuture, () -> { if (SupportsPipelining.isSupported(rule)) { addToPipelinesRunner((SupportsPipelining<?>) rule, Preconditions.checkNotNull(rulekeyCacheResult.get())); } return Futures.transformAsync( buildRuleBuilderDelegate.getDepResults(rule, buildContext, executionContext), (depResults) -> handleDepsResults(depResults), serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.SCHEDULING_MORE_WORK_RESOURCE_AMOUNTS)); }); // 4. Return to the current rule and check if it was (or is being) built in a pipeline with // one of its dependencies if (SupportsPipelining.isSupported(rule)) { buildResultFuture = transformBuildResultAsyncIfNotPresent(buildResultFuture, () -> { SupportsPipelining<?> pipelinedRule = (SupportsPipelining<?>) rule; return pipelinesRunner.runningPipelinesContainRule(pipelinedRule) ? pipelinesRunner.getFuture(pipelinedRule) : Futures.immediateFuture(Optional.empty()); }); } // 5. Return to the current rule and check caches to see if we can avoid building if (SupportsInputBasedRuleKey.isSupported(rule)) { buildResultFuture = transformBuildResultIfNotPresent(buildResultFuture, this::checkInputBasedCaches, serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.CACHE_CHECK_RESOURCE_AMOUNTS)); } // 6. Then check if the depfile matches. if (useDependencyFileRuleKey()) { buildResultFuture = transformBuildResultIfNotPresent(buildResultFuture, this::checkMatchingDepfile, serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.CACHE_CHECK_RESOURCE_AMOUNTS)); } // 7. Check for a manifest-based cache hit. if (useManifestCaching()) { buildResultFuture = transformBuildResultIfNotPresent(buildResultFuture, this::checkManifestBasedCaches, serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.CACHE_CHECK_RESOURCE_AMOUNTS)); } // 8. Fail if populating the cache and cache lookups failed. if (buildMode == CachingBuildEngine.BuildMode.POPULATE_FROM_REMOTE_CACHE) { buildResultFuture = transformBuildResultIfNotPresent(buildResultFuture, () -> { LOG.info("Cannot populate cache for " + rule.getBuildTarget().getFullyQualifiedName()); return Optional.of(BuildResult.canceled(rule, new HumanReadableException( "Skipping %s: in cache population mode local builds are disabled", rule))); }, MoreExecutors.newDirectExecutorService()); } // 9. Build the current rule locally, if we have to. buildResultFuture = transformBuildResultAsyncIfNotPresent(buildResultFuture, () -> buildLocally(Preconditions.checkNotNull(rulekeyCacheResult.get()), service // This needs to adjust the default amounts even in the non-resource-aware scheduling // case so that RuleScheduleInfo works correctly. .withDefaultAmounts(getRuleResourceAmounts()))); if (SupportsPipelining.isSupported(rule)) { buildResultFuture.addListener(() -> pipelinesRunner.removeRule((SupportsPipelining<?>) rule), MoreExecutors.directExecutor()); } // Unwrap the result. return Futures.transform(buildResultFuture, Optional::get); }
From source file:com.facebook.buck.core.build.engine.impl.CachingBuildRuleBuilder.java
private ListenableFuture<BuildResult> buildOrFetchFromCache() { // If we've already seen a failure, exit early. if (!shouldKeepGoing()) { return Futures.immediateFuture(canceled(firstFailure)); }/*from w w w . jav a 2 s . co m*/ // 1. Check if it's already built. try (Scope ignored = buildRuleScope()) { Optional<BuildResult> buildResult = checkMatchingLocalKey(); if (buildResult.isPresent()) { return Futures.immediateFuture(buildResult.get()); } } AtomicReference<CacheResult> rulekeyCacheResult = new AtomicReference<>(); ListenableFuture<Optional<BuildResult>> buildResultFuture; // 2. Rule key cache lookup. buildResultFuture = // TODO(cjhopman): This should follow the same, simple pattern as everything else. With a // large ui.thread_line_limit, SuperConsole tries to redraw more lines than are available. // These cache threads make it more likely to hit that problem when SuperConsole is aware // of them. Futures.transform(performRuleKeyCacheCheck(/* cacheHitExpected */ false), cacheResult -> { Objects.requireNonNull(cacheResult); cacheResult.getType().verifyValidFinalType(); rulekeyCacheResult.set(cacheResult); return getBuildResultForRuleKeyCacheResult(cacheResult); }, MoreExecutors.directExecutor()); // 3. Before unlocking dependencies, ensure build rule hasn't started remotely. buildResultFuture = attemptDistributedBuildSynchronization(buildResultFuture, rulekeyCacheResult); // 4. Build deps. buildResultFuture = transformBuildResultAsyncIfNotPresent(buildResultFuture, () -> { if (SupportsPipelining.isSupported(rule)) { addToPipelinesRunner((SupportsPipelining<?>) rule, Objects.requireNonNull(rulekeyCacheResult.get())); } return Futures.transformAsync(buildRuleBuilderDelegate.getDepResults(rule, executionContext), (depResults) -> handleDepsResults(depResults), serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.SCHEDULING_MORE_WORK_RESOURCE_AMOUNTS)); }); // 5. Return to the current rule and check if it was (or is being) built in a pipeline with // one of its dependencies if (SupportsPipelining.isSupported(rule)) { buildResultFuture = transformBuildResultAsyncIfNotPresent(buildResultFuture, () -> { SupportsPipelining<?> pipelinedRule = (SupportsPipelining<?>) rule; return pipelinesRunner.runningPipelinesContainRule(pipelinedRule) ? pipelinesRunner.getFuture(pipelinedRule) : Futures.immediateFuture(Optional.empty()); }); } // 6. Return to the current rule and check caches to see if we can avoid building if (SupportsInputBasedRuleKey.isSupported(rule)) { buildResultFuture = transformBuildResultAsyncIfNotPresent(buildResultFuture, this::checkInputBasedCaches); } // 7. Then check if the depfile matches. if (dependencyFileRuleKeyManager.useDependencyFileRuleKey()) { buildResultFuture = transformBuildResultIfNotPresent(buildResultFuture, this::checkMatchingDepfile, serviceByAdjustingDefaultWeightsTo(CachingBuildEngine.CACHE_CHECK_RESOURCE_AMOUNTS)); } // 8. Check for a manifest-based cache hit. if (manifestRuleKeyManager.useManifestCaching()) { buildResultFuture = transformBuildResultAsyncIfNotPresent(buildResultFuture, this::checkManifestBasedCaches); } // 9. Fail if populating the cache and cache lookups failed. if (buildMode == BuildType.POPULATE_FROM_REMOTE_CACHE) { buildResultFuture = transformBuildResultIfNotPresent(buildResultFuture, () -> { LOG.info("Cannot populate cache for " + rule.getBuildTarget().getFullyQualifiedName()); return Optional.of(canceled(new HumanReadableException( "Skipping %s: in cache population mode local builds are disabled", rule))); }, MoreExecutors.newDirectExecutorService()); } // 10. Before building locally, do a final check that rule hasn't started building remotely. // (as time has passed due to building of dependencies) buildResultFuture = attemptDistributedBuildSynchronization(buildResultFuture, rulekeyCacheResult); // 11. Build the current rule locally, if we have to. buildResultFuture = transformBuildResultAsyncIfNotPresent(buildResultFuture, () -> buildLocally(Objects.requireNonNull(rulekeyCacheResult.get()), service // This needs to adjust the default amounts even in the non-resource-aware // scheduling case so that RuleScheduleInfo works correctly. .withDefaultAmounts(getRuleResourceAmounts()))); if (SupportsPipelining.isSupported(rule)) { buildResultFuture.addListener(() -> pipelinesRunner.removeRule((SupportsPipelining<?>) rule), MoreExecutors.directExecutor()); } // Unwrap the result. return Futures.transform(buildResultFuture, Optional::get, MoreExecutors.directExecutor()); }
From source file:com.facebook.buck.core.build.engine.impl.CachingBuildRuleBuilder.java
private ListenableFuture<Optional<BuildResult>> transformBuildResultAsyncIfNotPresent( ListenableFuture<Optional<BuildResult>> future, Callable<ListenableFuture<Optional<BuildResult>>> function) { // Immediately (i.e. without posting a task), returns the current result if it's already present // or a cancelled result if we've already seen a failure. return Futures.transformAsync(future, (result) -> { if (result.isPresent()) { return Futures.immediateFuture(result); }//from w w w .ja v a 2 s.c o m if (!shouldKeepGoing()) { return Futures.immediateFuture(Optional.of(canceled(firstFailure))); } return function.call(); }, MoreExecutors.directExecutor()); }