List of usage examples for java.util.concurrent CopyOnWriteArrayList removeAll
public boolean removeAll(Collection<?> c)
From source file:io.druid.indexing.kafka.supervisor.KafkaSupervisor.java
/** * Monitors [pendingCompletionTaskGroups] for tasks that have completed. If any task in a task group has completed, we * can safely stop the rest of the tasks in that group. If a task group has exceeded its publishing timeout, then * we need to stop all tasks in not only that task group but also 1) any subsequent task group that is also pending * completion and 2) the current task group that is running, because the assumption that we have handled up to the * starting offset for subsequent task groups is no longer valid, and subsequent tasks would fail as soon as they * attempted to publish because of the contiguous range consistency check. *//*from www . j a va 2s . c o m*/ private void checkPendingCompletionTasks() throws ExecutionException, InterruptedException { List<ListenableFuture<Void>> futures = Lists.newArrayList(); for (Map.Entry<Integer, CopyOnWriteArrayList<TaskGroup>> pendingGroupList : pendingCompletionTaskGroups .entrySet()) { boolean stopTasksInTaskGroup = false; Integer groupId = pendingGroupList.getKey(); CopyOnWriteArrayList<TaskGroup> taskGroupList = pendingGroupList.getValue(); List<TaskGroup> toRemove = Lists.newArrayList(); for (TaskGroup group : taskGroupList) { boolean foundSuccess = false, entireTaskGroupFailed = false; if (stopTasksInTaskGroup) { // One of the earlier groups that was handling the same partition set timed out before the segments were // published so stop any additional groups handling the same partition set that are pending completion. futures.add(stopTasksInGroup(group)); toRemove.add(group); continue; } Iterator<Map.Entry<String, TaskData>> iTask = group.tasks.entrySet().iterator(); while (iTask.hasNext()) { Map.Entry<String, TaskData> task = iTask.next(); if (task.getValue().status.isFailure()) { iTask.remove(); // remove failed task if (group.tasks.isEmpty()) { // if all tasks in the group have failed, just nuke all task groups with this partition set and restart entireTaskGroupFailed = true; break; } } if (task.getValue().status.isSuccess()) { // If one of the pending completion tasks was successful, stop the rest of the tasks in the group as // we no longer need them to publish their segment. log.info("Task [%s] completed successfully, stopping tasks %s", task.getKey(), group.tasks.keySet()); futures.add(stopTasksInGroup(group)); foundSuccess = true; toRemove.add(group); // remove the TaskGroup from the list of pending completion task groups break; // skip iterating the rest of the tasks in this group as they've all been stopped now } } if ((!foundSuccess && group.completionTimeout.isBeforeNow()) || entireTaskGroupFailed) { if (entireTaskGroupFailed) { log.warn( "All tasks in group [%d] failed to publish, killing all tasks for these partitions", groupId); } else { log.makeAlert("No task in [%s] succeeded before the completion timeout elapsed [%s]!", group.tasks.keySet(), ioConfig.getCompletionTimeout()).emit(); } // reset partitions offsets for this task group so that they will be re-read from metadata storage partitionGroups.remove(groupId); // stop all the tasks in this pending completion group futures.add(stopTasksInGroup(group)); // set a flag so the other pending completion groups for this set of partitions will also stop stopTasksInTaskGroup = true; // stop all the tasks in the currently reading task group and remove the bad task group futures.add(stopTasksInGroup(taskGroups.remove(groupId))); toRemove.add(group); } } taskGroupList.removeAll(toRemove); } Futures.successfulAsList(futures).get(); // wait for all task shutdowns to complete before returning }
From source file:io.druid.indexing.jdbc.supervisor.JDBCSupervisor.java
/** * Monitors [pendingCompletionTaskGroups] for tasks that have completed. If any task in a task group has completed, we * can safely stop the rest of the tasks in that group. If a task group has exceeded its publishing timeout, then * we need to stop all tasks in not only that task group but also 1) any subsequent task group that is also pending * completion and 2) the current task group that is running, because the assumption that we have handled up to the * starting offset for subsequent task groups is no longer valid, and subsequent tasks would fail as soon as they * attempted to publish because of the contiguous range consistency check. *//*from w w w .j a va 2 s . c o m*/ private void checkPendingCompletionTasks() throws ExecutionException, InterruptedException, TimeoutException { List<ListenableFuture<?>> futures = Lists.newArrayList(); for (Map.Entry<Integer, CopyOnWriteArrayList<TaskGroup>> pendingGroupList : pendingCompletionTaskGroups .entrySet()) { boolean stopTasksInTaskGroup = false; Integer groupId = pendingGroupList.getKey(); CopyOnWriteArrayList<TaskGroup> taskGroupList = pendingGroupList.getValue(); List<TaskGroup> toRemove = Lists.newArrayList(); for (TaskGroup group : taskGroupList) { boolean foundSuccess = false, entireTaskGroupFailed = false; if (stopTasksInTaskGroup) { // One of the earlier groups that was handling the same partition set timed out before the segments were // published so stop any additional groups handling the same partition set that are pending completion. futures.add(stopTasksInGroup(group)); toRemove.add(group); continue; } Iterator<Map.Entry<String, TaskData>> iTask = group.tasks.entrySet().iterator(); while (iTask.hasNext()) { Map.Entry<String, TaskData> task = iTask.next(); if (task.getValue().status.isFailure()) { iTask.remove(); // remove failed task if (group.tasks.isEmpty()) { // if all tasks in the group have failed, just nuke all task groups with this partition set and restart entireTaskGroupFailed = true; break; } } if (task.getValue().status.isSuccess()) { // If one of the pending completion tasks was successful, stop the rest of the tasks in the group as // we no longer need them to publish their segment. log.info("Task [%s] completed successfully, stopping tasks %s", task.getKey(), group.taskIds()); futures.add(stopTasksInGroup(group)); foundSuccess = true; toRemove.add(group); // remove the TaskGroup from the list of pending completion task groups break; // skip iterating the rest of the tasks in this group as they've all been stopped now } } if ((!foundSuccess && group.completionTimeout.isBeforeNow()) || entireTaskGroupFailed) { if (entireTaskGroupFailed) { log.warn( "All tasks in group [%d] failed to publish, killing all tasks for these partitions", groupId); } else { log.makeAlert("No task in [%s] succeeded before the completion timeout elapsed [%s]!", group.taskIds(), ioConfig.getCompletionTimeout()).emit(); } // reset partitions offsets for this task group so that they will be re-read from metadata storage groups.remove(groupId); // stop all the tasks in this pending completion group futures.add(stopTasksInGroup(group)); // set a flag so the other pending completion groups for this set of partitions will also stop stopTasksInTaskGroup = true; // stop all the tasks in the currently reading task group and remove the bad task group futures.add(stopTasksInGroup(taskGroups.remove(groupId))); toRemove.add(group); } } taskGroupList.removeAll(toRemove); } // wait for all task shutdowns to complete before returning Futures.successfulAsList(futures).get(futureTimeoutInSeconds, TimeUnit.SECONDS); }
From source file:org.apache.druid.indexing.kafka.supervisor.KafkaSupervisor.java
/** * Monitors [pendingCompletionTaskGroups] for tasks that have completed. If any task in a task group has completed, we * can safely stop the rest of the tasks in that group. If a task group has exceeded its publishing timeout, then * we need to stop all tasks in not only that task group but also 1) any subsequent task group that is also pending * completion and 2) the current task group that is running, because the assumption that we have handled up to the * starting offset for subsequent task groups is no longer valid, and subsequent tasks would fail as soon as they * attempted to publish because of the contiguous range consistency check. */// www.j a va2 s. co m private void checkPendingCompletionTasks() throws ExecutionException, InterruptedException, TimeoutException { List<ListenableFuture<?>> futures = Lists.newArrayList(); for (Entry<Integer, CopyOnWriteArrayList<TaskGroup>> pendingGroupList : pendingCompletionTaskGroups .entrySet()) { boolean stopTasksInTaskGroup = false; Integer groupId = pendingGroupList.getKey(); CopyOnWriteArrayList<TaskGroup> taskGroupList = pendingGroupList.getValue(); List<TaskGroup> toRemove = Lists.newArrayList(); for (TaskGroup group : taskGroupList) { boolean foundSuccess = false, entireTaskGroupFailed = false; if (stopTasksInTaskGroup) { // One of the earlier groups that was handling the same partition set timed out before the segments were // published so stop any additional groups handling the same partition set that are pending completion. futures.add(stopTasksInGroup(group)); toRemove.add(group); continue; } Iterator<Entry<String, TaskData>> iTask = group.tasks.entrySet().iterator(); while (iTask.hasNext()) { final Entry<String, TaskData> entry = iTask.next(); final String taskId = entry.getKey(); final TaskData taskData = entry.getValue(); Preconditions.checkNotNull(taskData.status, "WTH? task[%s] has a null status", taskId); if (taskData.status.isFailure()) { iTask.remove(); // remove failed task if (group.tasks.isEmpty()) { // if all tasks in the group have failed, just nuke all task groups with this partition set and restart entireTaskGroupFailed = true; break; } } if (taskData.status.isSuccess()) { // If one of the pending completion tasks was successful, stop the rest of the tasks in the group as // we no longer need them to publish their segment. log.info("Task [%s] completed successfully, stopping tasks %s", taskId, group.taskIds()); futures.add(stopTasksInGroup(group)); foundSuccess = true; toRemove.add(group); // remove the TaskGroup from the list of pending completion task groups break; // skip iterating the rest of the tasks in this group as they've all been stopped now } } if ((!foundSuccess && group.completionTimeout.isBeforeNow()) || entireTaskGroupFailed) { if (entireTaskGroupFailed) { log.warn( "All tasks in group [%d] failed to publish, killing all tasks for these partitions", groupId); } else { log.makeAlert( "No task in [%s] for taskGroup [%d] succeeded before the completion timeout elapsed [%s]!", group.taskIds(), groupId, ioConfig.getCompletionTimeout()).emit(); } // reset partitions offsets for this task group so that they will be re-read from metadata storage partitionGroups.get(groupId).replaceAll((partition, offset) -> NOT_SET); // kill all the tasks in this pending completion group killTasksInGroup(group); // set a flag so the other pending completion groups for this set of partitions will also stop stopTasksInTaskGroup = true; // kill all the tasks in the currently reading task group and remove the bad task group killTasksInGroup(taskGroups.remove(groupId)); toRemove.add(group); } } taskGroupList.removeAll(toRemove); } // wait for all task shutdowns to complete before returning Futures.successfulAsList(futures).get(futureTimeoutInSeconds, TimeUnit.SECONDS); }