List of usage examples for com.google.common.util.concurrent Futures successfulAsList
@Beta @CheckReturnValue public static <V> ListenableFuture<List<V>> successfulAsList( Iterable<? extends ListenableFuture<? extends V>> futures)
From source file:com.google.monacoin.core.Peer.java
private ListenableFuture<Object> downloadDependenciesInternal(final Transaction tx, final Object marker, final List<Transaction> results) { checkNotNull(memoryPool, "Must have a configured MemoryPool object to download dependencies."); final SettableFuture<Object> resultFuture = SettableFuture.create(); final Sha256Hash rootTxHash = tx.getHash(); // We want to recursively grab its dependencies. This is so listeners can learn important information like // whether a transaction is dependent on a timelocked transaction or has an unexpectedly deep dependency tree // or depends on a no-fee transaction. ///*from www.j a v a2s . c om*/ // Firstly find any that are already in the memory pool so if they weren't garbage collected yet, they won't // be deleted. Use COW sets to make unit tests deterministic and because they are small. It's slower for // the case of transactions with tons of inputs. Set<Transaction> dependencies = new CopyOnWriteArraySet<Transaction>(); Set<Sha256Hash> needToRequest = new CopyOnWriteArraySet<Sha256Hash>(); for (TransactionInput input : tx.getInputs()) { // There may be multiple inputs that connect to the same transaction. Sha256Hash hash = input.getOutpoint().getHash(); Transaction dep = memoryPool.get(hash); if (dep == null) { needToRequest.add(hash); } else { dependencies.add(dep); } } results.addAll(dependencies); lock.lock(); try { // Build the request for the missing dependencies. List<ListenableFuture<Transaction>> futures = Lists.newArrayList(); GetDataMessage getdata = new GetDataMessage(params); final long nonce = (long) (Math.random() * Long.MAX_VALUE); if (needToRequest.size() > 1) log.info("{}: Requesting {} transactions for dep resolution", getAddress(), needToRequest.size()); for (Sha256Hash hash : needToRequest) { getdata.addTransaction(hash); GetDataRequest req = new GetDataRequest(); req.hash = hash; req.future = SettableFuture.create(); if (!isNotFoundMessageSupported()) { req.nonce = nonce; } futures.add(req.future); getDataFutures.add(req); } // The transactions we already grabbed out of the mempool must still be considered by the code below. for (Transaction dep : dependencies) { futures.add(Futures.immediateFuture(dep)); } ListenableFuture<List<Transaction>> successful = Futures.successfulAsList(futures); Futures.addCallback(successful, new FutureCallback<List<Transaction>>() { public void onSuccess(List<Transaction> transactions) { // Once all transactions either were received, or we know there are no more to come ... // Note that transactions will contain "null" for any positions that weren't successful. List<ListenableFuture<Object>> childFutures = Lists.newLinkedList(); for (Transaction tx : transactions) { if (tx == null) continue; log.info("{}: Downloaded dependency of {}: {}", getAddress(), rootTxHash, tx.getHashAsString()); results.add(tx); // Now recurse into the dependencies of this transaction too. childFutures.add(downloadDependenciesInternal(tx, marker, results)); } if (childFutures.size() == 0) { // Short-circuit: we're at the bottom of this part of the tree. resultFuture.set(marker); } else { // There are some children to download. Wait until it's done (and their children and their // children...) to inform the caller that we're finished. Futures.addCallback(Futures.successfulAsList(childFutures), new FutureCallback<List<Object>>() { public void onSuccess(List<Object> objects) { resultFuture.set(marker); } public void onFailure(Throwable throwable) { resultFuture.setException(throwable); } }); } } public void onFailure(Throwable throwable) { resultFuture.setException(throwable); } }); // Start the operation. sendMessage(getdata); if (!isNotFoundMessageSupported()) { // If the peer isn't new enough to support the notfound message, we use a nasty hack instead and // assume if we send a ping message after the getdata message, it'll be processed after all answers // from getdata are done, so we can watch for the pong message as a substitute. log.info("{}: Dep resolution waiting for a pong with nonce {}", this, nonce); ping(nonce).addListener(new Runnable() { public void run() { // The pong came back so clear out any transactions we requested but didn't get. for (GetDataRequest req : getDataFutures) { if (req.nonce == nonce) { log.info("{}: Bottomed out dep tree at {}", this, req.hash); req.future.cancel(true); getDataFutures.remove(req); } } } }, Threading.SAME_THREAD); } } catch (Exception e) { log.error("{}: Couldn't send getdata in downloadDependencies({})", this, tx.getHash()); resultFuture.setException(e); return resultFuture; } finally { lock.unlock(); } return resultFuture; }
From source file:io.druid.indexing.jdbc.supervisor.JDBCSupervisor.java
private void updateTaskStatus() throws ExecutionException, InterruptedException, TimeoutException { final List<ListenableFuture<Boolean>> futures = Lists.newArrayList(); final List<String> futureTaskIds = Lists.newArrayList(); // update status (and startTime if unknown) of current tasks in taskGroups for (TaskGroup group : taskGroups.values()) { for (Map.Entry<String, TaskData> entry : group.tasks.entrySet()) { final String taskId = entry.getKey(); final TaskData taskData = entry.getValue(); if (taskData.startTime == null) { futureTaskIds.add(taskId); futures.add(Futures.transform(taskClient.getStartTimeAsync(taskId), new Function<DateTime, Boolean>() { @Nullable// w w w .j a va 2 s .co m @Override public Boolean apply(@Nullable DateTime startTime) { if (startTime == null) { return false; } taskData.startTime = startTime; long millisRemaining = ioConfig.getTaskDuration().getMillis() - (System.currentTimeMillis() - taskData.startTime.getMillis()); if (millisRemaining > 0) { log.info("buildRunTask scheduled......"); scheduledExec.schedule(buildRunTask(), millisRemaining + MAX_RUN_FREQUENCY_MILLIS, TimeUnit.MILLISECONDS); } return true; } }, workerExec)); } taskData.status = taskStorage.getStatus(taskId).get(); } } // update status of pending completion tasks in pendingCompletionTaskGroups for (List<TaskGroup> taskGroups : pendingCompletionTaskGroups.values()) { for (TaskGroup group : taskGroups) { for (Map.Entry<String, TaskData> entry : group.tasks.entrySet()) { entry.getValue().status = taskStorage.getStatus(entry.getKey()).get(); } } } List<Boolean> results = Futures.successfulAsList(futures).get(futureTimeoutInSeconds, TimeUnit.SECONDS); for (int i = 0; i < results.size(); i++) { // false means the task hasn't started running yet and that's okay; null means it should be running but the HTTP // request threw an exception so kill the task if (results.get(i) == null) { String taskId = futureTaskIds.get(i); log.warn("Task [%s] failed to return start time, killing task", taskId); killTask(taskId); } } }
From source file:io.druid.indexing.kafka.supervisor.KafkaSupervisor.java
private void checkTaskDuration() throws InterruptedException, ExecutionException { final List<ListenableFuture<Map<Integer, Long>>> futures = Lists.newArrayList(); final List<Integer> futureGroupIds = Lists.newArrayList(); for (Map.Entry<Integer, TaskGroup> entry : taskGroups.entrySet()) { Integer groupId = entry.getKey(); TaskGroup group = entry.getValue(); // find the longest running task from this group DateTime earliestTaskStart = DateTime.now(); for (TaskData taskData : group.tasks.values()) { if (earliestTaskStart.isAfter(taskData.startTime)) { earliestTaskStart = taskData.startTime; }/*from ww w. j a v a 2 s . c o m*/ } // if this task has run longer than the configured duration, signal all tasks in the group to persist if (earliestTaskStart.plus(ioConfig.getTaskDuration()).isBeforeNow()) { log.info("Task group [%d] has run for [%s]", groupId, ioConfig.getTaskDuration()); futureGroupIds.add(groupId); futures.add(signalTasksToFinish(groupId)); } } List<Map<Integer, Long>> results = Futures.successfulAsList(futures).get(); for (int j = 0; j < results.size(); j++) { Integer groupId = futureGroupIds.get(j); TaskGroup group = taskGroups.get(groupId); Map<Integer, Long> endOffsets = results.get(j); if (endOffsets != null) { // set a timeout and put this group in pendingCompletionTaskGroups so that it can be monitored for completion group.completionTimeout = DateTime.now().plus(ioConfig.getCompletionTimeout()); pendingCompletionTaskGroups.putIfAbsent(groupId, Lists.<TaskGroup>newCopyOnWriteArrayList()); pendingCompletionTaskGroups.get(groupId).add(group); // set endOffsets as the next startOffsets for (Map.Entry<Integer, Long> entry : endOffsets.entrySet()) { partitionGroups.get(groupId).put(entry.getKey(), entry.getValue()); } } else { log.warn("All tasks in group [%s] failed to transition to publishing state, killing tasks [%s]", groupId, group.tasks.keySet()); for (String id : group.tasks.keySet()) { killTask(id); } } // remove this task group from the list of current task groups now that it has been handled taskGroups.remove(groupId); } }
From source file:io.druid.indexing.jdbc.supervisor.JDBCSupervisor.java
private void checkTaskDuration() throws InterruptedException, ExecutionException, TimeoutException { final List<ListenableFuture<Map<Integer, Long>>> futures = Lists.newArrayList(); final List<Integer> futureGroupIds = Lists.newArrayList(); for (Map.Entry<Integer, TaskGroup> entry : taskGroups.entrySet()) { Integer groupId = entry.getKey(); TaskGroup group = entry.getValue(); // find the longest running task from this group DateTime earliestTaskStart = DateTime.now(); for (TaskData taskData : group.tasks.values()) { if (earliestTaskStart.isAfter(taskData.startTime)) { earliestTaskStart = taskData.startTime; }/*from w ww .j a v a2 s .c om*/ } // if this task has run longer than the configured duration, signal all tasks in the group to persist if (earliestTaskStart.plus(ioConfig.getTaskDuration()).isBeforeNow()) { log.info("Task group [%d] has run for [%s]", groupId, ioConfig.getTaskDuration()); futureGroupIds.add(groupId); futures.add(signalTasksToFinish(groupId)); } } List<Map<Integer, Long>> results = Futures.successfulAsList(futures).get(futureTimeoutInSeconds, TimeUnit.SECONDS); log.info("checkTaskDuration results size is [%s]", results.size()); log.info("checkTaskDuration results info is [%s]", results.toString()); for (int j = 0; j < results.size(); j++) { Integer groupId = futureGroupIds.get(j); TaskGroup group = taskGroups.get(groupId); Map<Integer, Long> endOffsets = results.get(j); log.info("checkTaskDuration endOffsets is [%s]", endOffsets); if (endOffsets != null) { // set a timeout and put this group in pendingCompletionTaskGroups so that it can be monitored for completion group.completionTimeout = DateTime.now().plus(ioConfig.getCompletionTimeout()); pendingCompletionTaskGroups.putIfAbsent(groupId, Lists.<TaskGroup>newCopyOnWriteArrayList()); pendingCompletionTaskGroups.get(groupId).add(group); // set endOffsets as the next startOffsets for (Map.Entry<Integer, Long> entry : endOffsets.entrySet()) { groups.get(groupId).put(entry.getKey(), entry.getValue()); log.info("checkTaskDuration groups info [%d], [%d]", entry.getKey(), entry.getValue()); } } else { log.warn("All tasks in group [%s] failed to transition to publishing state, killing tasks [%s]", groupId, group.taskIds()); for (String id : group.taskIds()) { killTask(id); } } // remove this task group from the list of current task groups now that it has been handled log.info("TaskGroups removed by [%s]", groupId); taskGroups.remove(groupId); } }
From source file:io.mappum.altcoinj.core.Peer.java
private ListenableFuture<Object> downloadDependenciesInternal(final Transaction tx, final Object marker, final List<Transaction> results) { checkNotNull(memoryPool, "Must have a configured MemoryPool object to download dependencies."); final SettableFuture<Object> resultFuture = SettableFuture.create(); final Sha256Hash rootTxHash = tx.getHash(); // We want to recursively grab its dependencies. This is so listeners can learn important information like // whether a transaction is dependent on a timelocked transaction or has an unexpectedly deep dependency tree // or depends on a no-fee transaction. ///*from w ww . j a v a 2s . c o m*/ // Firstly find any that are already in the memory pool so if they weren't garbage collected yet, they won't // be deleted. Use COW sets to make unit tests deterministic and because they are small. It's slower for // the case of transactions with tons of inputs. Set<Transaction> dependencies = new CopyOnWriteArraySet<Transaction>(); Set<Sha256Hash> needToRequest = new CopyOnWriteArraySet<Sha256Hash>(); for (TransactionInput input : tx.getInputs()) { // There may be multiple inputs that connect to the same transaction. Sha256Hash hash = input.getOutpoint().getHash(); Transaction dep = memoryPool.get(hash); if (dep == null) { needToRequest.add(hash); } else { dependencies.add(dep); } } results.addAll(dependencies); lock.lock(); try { // Build the request for the missing dependencies. List<ListenableFuture<Transaction>> futures = Lists.newArrayList(); GetDataMessage getdata = new GetDataMessage(params); final long nonce = (long) (Math.random() * Long.MAX_VALUE); if (needToRequest.size() > 1) log.info("{}: Requesting {} transactions for dep resolution", getAddress(), needToRequest.size()); for (Sha256Hash hash : needToRequest) { getdata.addTransaction(hash); GetDataRequest req = new GetDataRequest(); req.hash = hash; req.future = SettableFuture.create(); if (!isNotFoundMessageSupported()) { req.nonce = nonce; } futures.add(req.future); getDataFutures.add(req); } // The transactions we already grabbed out of the mempool must still be considered by the code below. for (Transaction dep : dependencies) { futures.add(Futures.immediateFuture(dep)); } ListenableFuture<List<Transaction>> successful = Futures.successfulAsList(futures); Futures.addCallback(successful, new FutureCallback<List<Transaction>>() { @Override public void onSuccess(List<Transaction> transactions) { // Once all transactions either were received, or we know there are no more to come ... // Note that transactions will contain "null" for any positions that weren't successful. List<ListenableFuture<Object>> childFutures = Lists.newLinkedList(); for (Transaction tx : transactions) { if (tx == null) continue; log.info("{}: Downloaded dependency of {}: {}", getAddress(), rootTxHash, tx.getHashAsString()); results.add(tx); // Now recurse into the dependencies of this transaction too. childFutures.add(downloadDependenciesInternal(tx, marker, results)); } if (childFutures.size() == 0) { // Short-circuit: we're at the bottom of this part of the tree. resultFuture.set(marker); } else { // There are some children to download. Wait until it's done (and their children and their // children...) to inform the caller that we're finished. Futures.addCallback(Futures.successfulAsList(childFutures), new FutureCallback<List<Object>>() { @Override public void onSuccess(List<Object> objects) { resultFuture.set(marker); } @Override public void onFailure(Throwable throwable) { resultFuture.setException(throwable); } }); } } @Override public void onFailure(Throwable throwable) { resultFuture.setException(throwable); } }); // Start the operation. sendMessage(getdata); if (!isNotFoundMessageSupported()) { // If the peer isn't new enough to support the notfound message, we use a nasty hack instead and // assume if we send a ping message after the getdata message, it'll be processed after all answers // from getdata are done, so we can watch for the pong message as a substitute. log.info("{}: Dep resolution waiting for a pong with nonce {}", this, nonce); ping(nonce).addListener(new Runnable() { @Override public void run() { // The pong came back so clear out any transactions we requested but didn't get. for (GetDataRequest req : getDataFutures) { if (req.nonce == nonce) { log.info("{}: Bottomed out dep tree at {}", this, req.hash); req.future.cancel(true); getDataFutures.remove(req); } } } }, Threading.SAME_THREAD); } } catch (Exception e) { log.error("{}: Couldn't send getdata in downloadDependencies({})", this, tx.getHash()); resultFuture.setException(e); return resultFuture; } finally { lock.unlock(); } return resultFuture; }
From source file:com.github.kryptohash.kryptohashj.core.Peer.java
private ListenableFuture<Object> downloadDependenciesInternal(final Transaction tx, final Object marker, final List<Transaction> results) { checkNotNull(memoryPool, "Must have a configured MemoryPool object to download dependencies."); final SettableFuture<Object> resultFuture = SettableFuture.create(); final Shake320Hash rootTxHash = tx.getHash(); // We want to recursively grab its dependencies. This is so listeners can learn important information like // whether a transaction is dependent on a timelocked transaction or has an unexpectedly deep dependency tree // or depends on a no-fee transaction. ////from w ww .j a v a 2s. c o m // Firstly find any that are already in the memory pool so if they weren't garbage collected yet, they won't // be deleted. Use COW sets to make unit tests deterministic and because they are small. It's slower for // the case of transactions with tons of inputs. Set<Transaction> dependencies = new CopyOnWriteArraySet<Transaction>(); Set<Shake320Hash> needToRequest = new CopyOnWriteArraySet<Shake320Hash>(); for (TransactionInput input : tx.getInputs()) { // There may be multiple inputs that connect to the same transaction. Shake320Hash hash = input.getOutpoint().getHash(); Transaction dep = memoryPool.get(hash); if (dep == null) { needToRequest.add(hash); } else { dependencies.add(dep); } } results.addAll(dependencies); lock.lock(); try { // Build the request for the missing dependencies. List<ListenableFuture<Transaction>> futures = Lists.newArrayList(); GetDataMessage getdata = new GetDataMessage(params); final long nonce = (long) (Math.random() * Long.MAX_VALUE); if (needToRequest.size() > 1) log.info("{}: Requesting {} transactions for dep resolution", getAddress(), needToRequest.size()); for (Shake320Hash hash : needToRequest) { getdata.addTransaction(hash); GetDataRequest req = new GetDataRequest(); req.hash = hash; req.future = SettableFuture.create(); if (!isNotFoundMessageSupported()) { req.nonce = nonce; } futures.add(req.future); getDataFutures.add(req); } // The transactions we already grabbed out of the mempool must still be considered by the code below. for (Transaction dep : dependencies) { futures.add(Futures.immediateFuture(dep)); } ListenableFuture<List<Transaction>> successful = Futures.successfulAsList(futures); Futures.addCallback(successful, new FutureCallback<List<Transaction>>() { @Override public void onSuccess(List<Transaction> transactions) { // Once all transactions either were received, or we know there are no more to come ... // Note that transactions will contain "null" for any positions that weren't successful. List<ListenableFuture<Object>> childFutures = Lists.newLinkedList(); for (Transaction tx : transactions) { if (tx == null) continue; log.info("{}: Downloaded dependency of {}: {}", getAddress(), rootTxHash, tx.getHashAsString()); results.add(tx); // Now recurse into the dependencies of this transaction too. childFutures.add(downloadDependenciesInternal(tx, marker, results)); } if (childFutures.size() == 0) { // Short-circuit: we're at the bottom of this part of the tree. resultFuture.set(marker); } else { // There are some children to download. Wait until it's done (and their children and their // children...) to inform the caller that we're finished. Futures.addCallback(Futures.successfulAsList(childFutures), new FutureCallback<List<Object>>() { @Override public void onSuccess(List<Object> objects) { resultFuture.set(marker); } @Override public void onFailure(Throwable throwable) { resultFuture.setException(throwable); } }); } } @Override public void onFailure(Throwable throwable) { resultFuture.setException(throwable); } }); // Start the operation. sendMessage(getdata); if (!isNotFoundMessageSupported()) { // If the peer isn't new enough to support the notfound message, we use a nasty hack instead and // assume if we send a ping message after the getdata message, it'll be processed after all answers // from getdata are done, so we can watch for the pong message as a substitute. log.info("{}: Dep resolution waiting for a pong with nonce {}", this, nonce); ping(nonce).addListener(new Runnable() { @Override public void run() { // The pong came back so clear out any transactions we requested but didn't get. for (GetDataRequest req : getDataFutures) { if (req.nonce == nonce) { log.info("{}: Bottomed out dep tree at {}", this, req.hash); req.future.cancel(true); getDataFutures.remove(req); } } } }, Threading.SAME_THREAD); } } catch (Exception e) { log.error("{}: Couldn't send getdata in downloadDependencies({})", this, tx.getHash()); resultFuture.setException(e); return resultFuture; } finally { lock.unlock(); } return resultFuture; }
From source file:io.druid.indexing.kafka.supervisor.KafkaSupervisor.java
private ListenableFuture<Map<Integer, Long>> signalTasksToFinish(final int groupId) { final TaskGroup taskGroup = taskGroups.get(groupId); // 1) Check if any task completed (in which case we're done) and kill unassigned tasks Iterator<Map.Entry<String, TaskData>> i = taskGroup.tasks.entrySet().iterator(); while (i.hasNext()) { Map.Entry<String, TaskData> taskEntry = i.next(); String taskId = taskEntry.getKey(); TaskData task = taskEntry.getValue(); if (task.status.isSuccess()) { // If any task in this group has already completed, stop the rest of the tasks in the group and return. // This will cause us to create a new set of tasks next cycle that will start from the offsets in // metadata store (which will have advanced if we succeeded in publishing and will remain the same if publishing // failed and we need to re-ingest) return Futures.transform(stopTasksInGroup(taskGroup), new Function<Void, Map<Integer, Long>>() { @Nullable//from w w w . j a va 2s . c o m @Override public Map<Integer, Long> apply(@Nullable Void input) { return null; } }, workerExec); } if (task.status.isRunnable()) { if (taskInfoProvider.getTaskLocation(taskId).equals(TaskLocation.unknown())) { log.info("Killing task [%s] which hasn't been assigned to a worker", taskId); killTask(taskId); i.remove(); } } } // 2) Pause running tasks final List<ListenableFuture<Map<Integer, Long>>> pauseFutures = Lists.newArrayList(); final List<String> pauseTaskIds = ImmutableList.copyOf(taskGroup.tasks.keySet()); for (final String taskId : pauseTaskIds) { pauseFutures.add(taskClient.pauseAsync(taskId)); } return Futures.transform(Futures.successfulAsList(pauseFutures), new Function<List<Map<Integer, Long>>, Map<Integer, Long>>() { @Nullable @Override public Map<Integer, Long> apply(List<Map<Integer, Long>> input) { // 3) Build a map of the highest offset read by any task in the group for each partition final Map<Integer, Long> endOffsets = new HashMap<>(); for (int i = 0; i < input.size(); i++) { Map<Integer, Long> result = input.get(i); if (result == null || result.isEmpty()) { // kill tasks that didn't return a value String taskId = pauseTaskIds.get(i); log.warn("Task [%s] failed to respond to [pause] in a timely manner, killing task", taskId); killTask(taskId); taskGroup.tasks.remove(taskId); } else { // otherwise build a map of the highest offsets seen for (Map.Entry<Integer, Long> offset : result.entrySet()) { if (!endOffsets.containsKey(offset.getKey()) || endOffsets.get(offset.getKey()).compareTo(offset.getValue()) < 0) { endOffsets.put(offset.getKey(), offset.getValue()); } } } } // 4) Set the end offsets for each task to the values from step 3 and resume the tasks. All the tasks should // finish reading and start publishing within a short period, depending on how in sync the tasks were. final List<ListenableFuture<Boolean>> setEndOffsetFutures = Lists.newArrayList(); final List<String> setEndOffsetTaskIds = ImmutableList.copyOf(taskGroup.tasks.keySet()); if (setEndOffsetTaskIds.isEmpty()) { log.info("All tasks in taskGroup [%d] have failed, tasks will be re-created", groupId); return null; } log.info("Setting endOffsets for tasks in taskGroup [%d] to %s and resuming", groupId, endOffsets); for (final String taskId : setEndOffsetTaskIds) { setEndOffsetFutures.add(taskClient.setEndOffsetsAsync(taskId, endOffsets, true)); } try { List<Boolean> results = Futures.successfulAsList(setEndOffsetFutures).get(); for (int i = 0; i < results.size(); i++) { if (results.get(i) == null || !results.get(i)) { String taskId = setEndOffsetTaskIds.get(i); log.warn( "Task [%s] failed to respond to [set end offsets] in a timely manner, killing task", taskId); killTask(taskId); taskGroup.tasks.remove(taskId); } } } catch (Exception e) { Throwables.propagate(e); } if (taskGroup.tasks.isEmpty()) { log.info("All tasks in taskGroup [%d] have failed, tasks will be re-created", groupId); return null; } return endOffsets; } }, workerExec); }
From source file:com.dogecoin.dogecoinj.core.Peer.java
private ListenableFuture<Object> downloadDependenciesInternal(final Transaction tx, final Object marker, final List<Transaction> results) { checkNotNull(confidenceTable, "Must have a configured TxConfidenceTable object to download dependencies."); final SettableFuture<Object> resultFuture = SettableFuture.create(); final Sha256Hash rootTxHash = tx.getHash(); // We want to recursively grab its dependencies. This is so listeners can learn important information like // whether a transaction is dependent on a timelocked transaction or has an unexpectedly deep dependency tree // or depends on a no-fee transaction. ///*w w w .ja v a 2s .co m*/ // Firstly find any that are already in the memory pool so if they weren't garbage collected yet, they won't // be deleted. Use COW sets to make unit tests deterministic and because they are small. It's slower for // the case of transactions with tons of inputs. Set<Transaction> dependencies = new CopyOnWriteArraySet<Transaction>(); Set<Sha256Hash> needToRequest = new CopyOnWriteArraySet<Sha256Hash>(); for (TransactionInput input : tx.getInputs()) { // There may be multiple inputs that connect to the same transaction. Sha256Hash hash = input.getOutpoint().getHash(); Transaction dep = confidenceTable.get(hash); if (dep == null) { needToRequest.add(hash); } else { dependencies.add(dep); } } results.addAll(dependencies); lock.lock(); try { // Build the request for the missing dependencies. List<ListenableFuture<Transaction>> futures = Lists.newArrayList(); GetDataMessage getdata = new GetDataMessage(params); if (needToRequest.size() > 1) log.info("{}: Requesting {} transactions for dep resolution", getAddress(), needToRequest.size()); for (Sha256Hash hash : needToRequest) { getdata.addTransaction(hash); GetDataRequest req = new GetDataRequest(); req.hash = hash; req.future = SettableFuture.create(); futures.add(req.future); getDataFutures.add(req); } // The transactions we already grabbed out of the mempool must still be considered by the code below. for (Transaction dep : dependencies) { futures.add(Futures.immediateFuture(dep)); } ListenableFuture<List<Transaction>> successful = Futures.successfulAsList(futures); Futures.addCallback(successful, new FutureCallback<List<Transaction>>() { @Override public void onSuccess(List<Transaction> transactions) { // Once all transactions either were received, or we know there are no more to come ... // Note that transactions will contain "null" for any positions that weren't successful. List<ListenableFuture<Object>> childFutures = Lists.newLinkedList(); for (Transaction tx : transactions) { if (tx == null) continue; log.info("{}: Downloaded dependency of {}: {}", getAddress(), rootTxHash, tx.getHashAsString()); results.add(tx); // Now recurse into the dependencies of this transaction too. childFutures.add(downloadDependenciesInternal(tx, marker, results)); } if (childFutures.size() == 0) { // Short-circuit: we're at the bottom of this part of the tree. resultFuture.set(marker); } else { // There are some children to download. Wait until it's done (and their children and their // children...) to inform the caller that we're finished. Futures.addCallback(Futures.successfulAsList(childFutures), new FutureCallback<List<Object>>() { @Override public void onSuccess(List<Object> objects) { resultFuture.set(marker); } @Override public void onFailure(Throwable throwable) { resultFuture.setException(throwable); } }); } } @Override public void onFailure(Throwable throwable) { resultFuture.setException(throwable); } }); // Start the operation. sendMessage(getdata); } catch (Exception e) { log.error("{}: Couldn't send getdata in downloadDependencies({})", this, tx.getHash()); resultFuture.setException(e); return resultFuture; } finally { lock.unlock(); } return resultFuture; }
From source file:io.druid.indexing.jdbc.supervisor.JDBCSupervisor.java
private ListenableFuture<Map<Integer, Long>> signalTasksToFinish(final int groupId) { final TaskGroup taskGroup = taskGroups.get(groupId); // 1) Check if any task completed (in which case we're done) and kill unassigned tasks Iterator<Map.Entry<String, TaskData>> i = taskGroup.tasks.entrySet().iterator(); while (i.hasNext()) { Map.Entry<String, TaskData> taskEntry = i.next(); String taskId = taskEntry.getKey(); TaskData task = taskEntry.getValue(); if (task.status.isSuccess()) { // If any task in this group has already completed, stop the rest of the tasks in the group and return. // This will cause us to create a new set of tasks next cycle that will start from the offsets in // metadata store (which will have advanced if we succeeded in publishing and will remain the same if publishing // failed and we need to re-ingest) return Futures.transform(stopTasksInGroup(taskGroup), new Function<Object, Map<Integer, Long>>() { @Nullable//w w w . ja va2s. c om @Override public Map<Integer, Long> apply(@Nullable Object input) { return null; } }); } if (task.status.isRunnable()) { if (taskInfoProvider.getTaskLocation(taskId).equals(TaskLocation.unknown())) { log.info("Killing task [%s] which hasn't been assigned to a worker", taskId); killTask(taskId); i.remove(); } } } // 2) Pause running tasks final List<ListenableFuture<Map<Integer, Long>>> pauseFutures = Lists.newArrayList(); final List<String> pauseTaskIds = ImmutableList.copyOf(taskGroup.taskIds()); for (final String taskId : pauseTaskIds) { log.info("taskClient pauseAsync called by [%s]", taskId); pauseFutures.add(taskClient.pauseAsync(taskId)); } return Futures.transform(Futures.successfulAsList(pauseFutures), new Function<List<Map<Integer, Long>>, Map<Integer, Long>>() { @Nullable @Override public Map<Integer, Long> apply(List<Map<Integer, Long>> input) { // 3) Build a map of the highest offset read by any task in the group for each partition final Map<Integer, Long> endOffsets = new HashMap<>(); for (int i = 0; i < input.size(); i++) { Map<Integer, Long> result = input.get(i); if (result == null) { // kill tasks that didn't return a value String taskId = pauseTaskIds.get(i); log.warn("Task [%s] failed to respond to [pause] in a timely manner, killing task", taskId); killTask(taskId); taskGroup.tasks.remove(taskId); } else { // otherwise build a map of the highest offsets seen for (Map.Entry<Integer, Long> offset : result.entrySet()) { if (!endOffsets.containsKey(offset.getKey()) || endOffsets.get(offset.getKey()).compareTo(offset.getValue()) < 0) { endOffsets.put(offset.getKey(), offset.getValue()); } } } } // 4) Set the end offsets for each task to the values from step 3 and resume the tasks. All the tasks should // finish reading and start publishing within a short period, depending on how in sync the tasks were. final List<ListenableFuture<Boolean>> setEndOffsetFutures = Lists.newArrayList(); final List<String> setEndOffsetTaskIds = ImmutableList.copyOf(taskGroup.taskIds()); if (setEndOffsetTaskIds.isEmpty()) { log.info("All tasks in taskGroup [%d] have failed, tasks will be re-created", groupId); return null; } log.info("Setting endOffsets for tasks in taskGroup [%d] to %s and resuming", groupId, endOffsets); for (final String taskId : setEndOffsetTaskIds) { setEndOffsetFutures.add(taskClient.setEndOffsetsAsync(taskId, endOffsets, true)); } try { List<Boolean> results = Futures.successfulAsList(setEndOffsetFutures) .get(futureTimeoutInSeconds, TimeUnit.SECONDS); for (int i = 0; i < results.size(); i++) { if (results.get(i) == null || !results.get(i)) { String taskId = setEndOffsetTaskIds.get(i); log.warn( "Task [%s] failed to respond to [set end offsets] in a timely manner, killing task", taskId); killTask(taskId); taskGroup.tasks.remove(taskId); } } } catch (Exception e) { Throwables.propagate(e); } if (taskGroup.tasks.isEmpty()) { log.info("All tasks in taskGroup [%d] have failed, tasks will be re-created", groupId); return null; } return endOffsets; } }, workerExec); }
From source file:org.litecoinj.core.Peer.java
protected ListenableFuture<Object> downloadDependenciesInternal(final int maxDepth, final int depth, final Transaction tx, final Object marker, final List<Transaction> results) { final SettableFuture<Object> resultFuture = SettableFuture.create(); final Sha256Hash rootTxHash = tx.getHash(); // We want to recursively grab its dependencies. This is so listeners can learn important information like // whether a transaction is dependent on a timelocked transaction or has an unexpectedly deep dependency tree // or depends on a no-fee transaction. // We may end up requesting transactions that we've already downloaded and thrown away here. Set<Sha256Hash> needToRequest = new CopyOnWriteArraySet<>(); for (TransactionInput input : tx.getInputs()) { // There may be multiple inputs that connect to the same transaction. needToRequest.add(input.getOutpoint().getHash()); }//ww w . j a va2 s . com lock.lock(); try { // Build the request for the missing dependencies. List<ListenableFuture<Transaction>> futures = Lists.newArrayList(); GetDataMessage getdata = new GetDataMessage(params); if (needToRequest.size() > 1) log.info("{}: Requesting {} transactions for depth {} dep resolution", getAddress(), needToRequest.size(), depth + 1); for (Sha256Hash hash : needToRequest) { getdata.addTransaction(hash); GetDataRequest req = new GetDataRequest(hash, SettableFuture.create()); futures.add(req.future); getDataFutures.add(req); } ListenableFuture<List<Transaction>> successful = Futures.successfulAsList(futures); Futures.addCallback(successful, new FutureCallback<List<Transaction>>() { @Override public void onSuccess(List<Transaction> transactions) { // Once all transactions either were received, or we know there are no more to come ... // Note that transactions will contain "null" for any positions that weren't successful. List<ListenableFuture<Object>> childFutures = Lists.newLinkedList(); for (Transaction tx : transactions) { if (tx == null) continue; log.info("{}: Downloaded dependency of {}: {}", getAddress(), rootTxHash, tx.getHashAsString()); results.add(tx); // Now recurse into the dependencies of this transaction too. if (depth + 1 < maxDepth) childFutures .add(downloadDependenciesInternal(maxDepth, depth + 1, tx, marker, results)); } if (childFutures.size() == 0) { // Short-circuit: we're at the bottom of this part of the tree. resultFuture.set(marker); } else { // There are some children to download. Wait until it's done (and their children and their // children...) to inform the caller that we're finished. Futures.addCallback(Futures.successfulAsList(childFutures), new FutureCallback<List<Object>>() { @Override public void onSuccess(List<Object> objects) { resultFuture.set(marker); } @Override public void onFailure(Throwable throwable) { resultFuture.setException(throwable); } }); } } @Override public void onFailure(Throwable throwable) { resultFuture.setException(throwable); } }); // Start the operation. sendMessage(getdata); } catch (Exception e) { log.error("{}: Couldn't send getdata in downloadDependencies({})", this, tx.getHash(), e); resultFuture.setException(e); return resultFuture; } finally { lock.unlock(); } return resultFuture; }