List of usage examples for com.google.common.collect Iterables partition
public static <T> Iterable<List<T>> partition(final Iterable<T> iterable, final int size)
From source file:google.registry.dns.ReadDnsQueueAction.java
/** Leases all tasks from the pull queue and creates per-tld update actions for them. */ @Override/* w ww.jav a 2 s .com*/ public void run() { Set<String> tldsOfInterest = getTlds(); List<TaskHandle> tasks = dnsQueue.leaseTasks(writeLockTimeout); if (tasks.isEmpty()) { return; } logger.infofmt("leased %d tasks", tasks.size()); // Normally, all tasks will be deleted from the pull queue. But some might have to remain if // we are not interested in the associated TLD, or if the TLD is paused. Remember which these // are. Set<TaskHandle> tasksToKeep = new HashSet<>(); // The paused TLDs for which we found at least one refresh request. Set<String> pausedTlds = new HashSet<>(); // Create a sorted multimap into which we will insert the refresh items, so that the items for // each TLD will be grouped together, and domains and hosts will be grouped within a TLD. The // grouping and ordering of domains and hosts is not technically necessary, but a predictable // ordering makes it possible to write detailed tests. SortedSetMultimap<String, RefreshItem> refreshItemMultimap = TreeMultimap.create(); // Read all tasks on the DNS pull queue and load them into the refresh item multimap. for (TaskHandle task : tasks) { try { Map<String, String> params = ImmutableMap.copyOf(task.extractParams()); String tld = params.get(RequestParameters.PARAM_TLD); if (tld == null) { logger.severe("discarding invalid DNS refresh request; no TLD specified"); } else if (!tldsOfInterest.contains(tld)) { tasksToKeep.add(task); } else if (Registry.get(tld).getDnsPaused()) { tasksToKeep.add(task); pausedTlds.add(tld); } else { String typeString = params.get(DNS_TARGET_TYPE_PARAM); String name = params.get(DNS_TARGET_NAME_PARAM); TargetType type = TargetType.valueOf(typeString); switch (type) { case DOMAIN: case HOST: refreshItemMultimap.put(tld, RefreshItem.create(type, name)); break; default: logger.severefmt("discarding DNS refresh request of type %s", typeString); break; } } } catch (RuntimeException | UnsupportedEncodingException e) { logger.severefmt(e, "discarding invalid DNS refresh request (task %s)", task); } } if (!pausedTlds.isEmpty()) { logger.infofmt("the dns-pull queue is paused for tlds: %s", pausedTlds); } // Loop through the multimap by TLD and generate refresh tasks for the hosts and domains. for (Map.Entry<String, Collection<RefreshItem>> tldRefreshItemsEntry : refreshItemMultimap.asMap() .entrySet()) { for (List<RefreshItem> chunk : Iterables.partition(tldRefreshItemsEntry.getValue(), tldUpdateBatchSize)) { TaskOptions options = withUrl(PublishDnsUpdatesAction.PATH).countdownMillis( jitterSeconds.isPresent() ? random.nextInt((int) SECONDS.toMillis(jitterSeconds.get())) : 0) .param(RequestParameters.PARAM_TLD, tldRefreshItemsEntry.getKey()); for (RefreshItem refreshItem : chunk) { options.param((refreshItem.type() == TargetType.HOST) ? PublishDnsUpdatesAction.HOSTS_PARAM : PublishDnsUpdatesAction.DOMAINS_PARAM, refreshItem.name()); } taskEnqueuer.enqueue(dnsPublishPushQueue, options); } } Set<TaskHandle> tasksToDelete = difference(ImmutableSet.copyOf(tasks), tasksToKeep); // In keepTasks mode, never delete any tasks. if (keepTasks) { logger.infofmt("would have deleted %d tasks", tasksToDelete.size()); for (TaskHandle task : tasks) { dnsQueue.dropTaskLease(task); } // Otherwise, either delete or drop the lease of each task. } else { logger.infofmt("deleting %d tasks", tasksToDelete.size()); dnsQueue.deleteTasks(ImmutableList.copyOf(tasksToDelete)); logger.infofmt("dropping %d tasks", tasksToKeep.size()); for (TaskHandle task : tasksToKeep) { dnsQueue.dropTaskLease(task); } logger.infofmt("done"); } }
From source file:com.google.devtools.build.lib.query2.ParallelVisitor.java
/** Gets tasks to visit pending keys. */ protected Iterable<Task> getVisitTasks(Collection<T> pendingKeysToVisit) { ImmutableList.Builder<Task> builder = ImmutableList.builder(); for (Iterable<T> keysToVisitBatch : Iterables.partition(pendingKeysToVisit, visitBatchSize)) { builder.add(new VisitTask(keysToVisitBatch)); }/* w w w. j a va 2 s . co m*/ return builder.build(); }
From source file:org.candlepin.model.ConsumerCurator.java
/** * Lookup all consumers matching the given guest IDs. * * Maps guest ID to the most recent registered consumer that matches it. * Any guest ID not found will not return null. * * If multiple registered consumers report this guest ID (re-registraiton), only the * most recently updated will be returned. * * @param guestId/*from www .ja va 2 s. c o m*/ * * @return VirtConsumerMap of guest ID to it's registered guest consumer, or null if * none exists. */ @Transactional public VirtConsumerMap getGuestConsumersMap(Owner owner, Set<String> guestIds) { VirtConsumerMap guestConsumersMap = new VirtConsumerMap(); if (guestIds.size() == 0) { return guestConsumersMap; } List<String> possibleGuestIds = Util.getPossibleUuids(guestIds.toArray(new String[guestIds.size()])); String sql = "select cp_consumer.uuid from cp_consumer " + "inner join cp_consumer_facts_lower " + "on cp_consumer.id = cp_consumer_facts_lower.cp_consumer_id " + "where cp_consumer_facts_lower.mapkey = 'virt.uuid' and " + "cp_consumer_facts_lower.element in (:guestids) " + "and cp_consumer.owner_id = :ownerid " + "order by cp_consumer.updated desc"; // We need to filter down to only the most recently registered consumer with // each guest ID. List<String> consumerUuids = new LinkedList<String>(); Iterable<List<String>> blocks = Iterables.partition(possibleGuestIds, AbstractHibernateCurator.IN_OPERATOR_BLOCK_SIZE); Query query = this.currentSession().createSQLQuery(sql).setParameter("ownerid", owner.getId()); for (List<String> block : blocks) { query.setParameterList("guestids", block); consumerUuids.addAll(query.list()); } if (consumerUuids.isEmpty()) { return guestConsumersMap; } List<Consumer> guestConsumersWithDupes = findByUuidsAndOwner(consumerUuids, owner); // At this point we might have duplicates for re-registered consumers: for (Consumer c : guestConsumersWithDupes) { String virtUuid = c.getFact("virt.uuid").toLowerCase(); if (guestConsumersMap.get(virtUuid) == null) { // Store both big and little endian forms in the result: guestConsumersMap.add(virtUuid, c); } // Can safely ignore if already in the map, this would be another consumer // reporting the same guest ID (re-registration), but we already sorted by // last update time. } return guestConsumersMap; }
From source file:org.sonar.db.DatabaseUtils.java
/** * Ensure values {@code inputs} are unique (which avoids useless arguments) and sorted before creating the partition. *//*from ww w.ja v a 2s .c o m*/ public static <INPUT extends Comparable<INPUT>> Iterable<List<INPUT>> toUniqueAndSortedPartitions( Collection<INPUT> inputs, IntFunction<Integer> partitionSizeManipulations) { int partitionSize = partitionSizeManipulations.apply(PARTITION_SIZE_FOR_ORACLE); return Iterables.partition(toUniqueAndSortedList(inputs), partitionSize); }
From source file:ome.util.actions.PostgresSqlAction.java
@Override public void setFileRepo(Collection<Long> ids, String repoId) { for (final List<Long> idsBatch : Iterables.partition(ids, 256)) { final Map<String, Object> parameters = new HashMap<String, Object>(); parameters.put("ids", idsBatch); parameters.put("repo", repoId); _jdbc().update(_lookup("set_file_repo"), //$NON-NLS-1$ parameters);/*from w w w .j av a 2 s. c om*/ } }
From source file:ru.tehkode.permissions.bukkit.commands.UtilityCommands.java
@Command(name = "pex", syntax = "convert uuid [force]", permission = "permissions.convert", description = "Bulk convert user data to UUID-based storage") public void convertUUID(final PermissionsEx plugin, final CommandSender sender, Map<String, String> args) { final PermissionBackend backend = plugin.getPermissionsManager().getBackend(); if (!plugin.getServer().getOnlineMode() && !"force".equals(args.get("force"))) { sender.sendMessage(ChatColor.RED + "This server is running in offline mode and UUIDs may not be stable. Please run '/pex convert uuid force' to perform conversion anyway, or switch to online mode."); return;/*w w w .jav a2 s. c o m*/ } final ProfileRepository repo = new HttpProfileRepository("minecraft"); final Collection<String> userIdentifiers = new HashSet<>(backend.getUserIdentifiers()); for (Iterator<String> it = userIdentifiers.iterator(); it.hasNext();) { try { UUID.fromString(it.next()); it.remove(); } catch (IllegalArgumentException ex) { } } if (userIdentifiers.isEmpty()) { sender.sendMessage(ChatColor.RED + "No users to convert!"); return; } sender.sendMessage("Beginning conversion to UUID in " + (int) Math.ceil(userIdentifiers.size() / 50000.0) + " batches of max 50k (1 batch is executed every 10 minutes)"); backend.setPersistent(false); final Iterator<List<String>> splitIdentifiers = Iterables.partition(userIdentifiers, 50 * 1000).iterator(); // 50k users per 10 minutes final AtomicInteger batchNum = new AtomicInteger(1); plugin.getPermissionsManager().getExecutor().execute(new Runnable() { @Override public void run() { List<String> names = splitIdentifiers.next(); try { for (Profile profile : repo.findProfilesByNames(names.toArray(new String[names.size()]))) { PermissionsUserData data = backend.getUserData(profile.getName()); data.setIdentifier(profile.getId().replaceFirst("(\\w{8})(\\w{4})(\\w{4})(\\w{4})(\\w{12})", "$1-$2-$3-$4-$5")); data.setOption("name", profile.getName(), null); } } catch (Exception e) { ErrorReport.handleError("While converting batch " + batchNum.get() + " to UUID", e); backend.setPersistent(true); return; } if (splitIdentifiers.hasNext()) { plugin.getPermissionsManager().getExecutor().schedule(this, 10, TimeUnit.MINUTES); plugin.getLogger().info("Completed conversion batch " + batchNum.getAndIncrement() + " of " + (int) Math.ceil(userIdentifiers.size() / 50000.0)); } else { plugin.getLogger().info("UUID conversion complete"); if (!(sender instanceof Player) || ((Player) sender).isOnline()) { sender.sendMessage("UUID conversion complete"); } backend.setPersistent(true); } } }); }
From source file:org.codice.ddf.commands.catalog.DuplicateCommands.java
/** * In batches, loops through a query of the queryFacade and an ingest to the ingestFacade of the * metacards from the response until there are no more metacards from the queryFacade or the * maxMetacards has been reached./*ww w . ja va 2 s .c om*/ * * @param queryFacade - the CatalogFacade to duplicate from * @param ingestFacade - the CatalogFacade to duplicate to * @param filter - the filter to query with */ protected void duplicateInBatches(CatalogFacade queryFacade, CatalogFacade ingestFacade, Filter filter, String sourceId) throws InterruptedException { AtomicInteger queryIndex = new AtomicInteger(1); final long originalQuerySize; if (maxMetacards > 0 && maxMetacards < batchSize) { originalQuerySize = maxMetacards; } else { originalQuerySize = batchSize; } Function<Integer, QueryRequest> queryTemplate = (index) -> new QueryRequestImpl(new QueryImpl(filter, index, (int) originalQuerySize, new SortByImpl(Metacard.EFFECTIVE, SortOrder.DESCENDING), true, TimeUnit.MINUTES.toMillis(5)), Collections.singletonList(sourceId)); List<Metacard> initialMetacards = ResultIterable.resultIterable((queryRequest -> { SourceResponse response = queryFacade.query(queryRequest); if (response.getHits() != -1) { maxMetacards = (int) response.getHits(); } return response; }), queryTemplate.apply(queryIndex.get()), (int) originalQuerySize).stream().map(Result::getMetacard) .collect(Collectors.toList()); if (initialMetacards.isEmpty()) { LOGGER.debug("Query returned 0 results."); console.println(String.format("No results were returned by the source [%s]", sourceId)); return; } ingestMetacards(ingestFacade, initialMetacards); if (initialMetacards.size() < originalQuerySize) { // all done if results exhausted in the first batch printProgressAndFlush(start, maxMetacards < 1 ? initialMetacards.size() : maxMetacards, ingestedCount.get()); return; } final long totalWanted = maxMetacards; final AtomicBoolean done = new AtomicBoolean(false); if (multithreaded > 1) { BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<>(multithreaded); RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy(); final ExecutorService executorService = new ThreadPoolExecutor(multithreaded, multithreaded, 0L, TimeUnit.MILLISECONDS, blockingQueue, StandardThreadFactoryBuilder.newThreadFactory("duplicateCommandsThread"), rejectedExecutionHandler); console.printf("Running a maximum of %d threads during replication.%n", multithreaded); printProgressAndFlush(start, Math.max(totalWanted, initialMetacards.size()), ingestedCount.get()); int index; while (!done.get()) { index = queryIndex.addAndGet(batchSize); final int taskIndex = index; executorService.submit(() -> { int querySize = (int) getQuerySizeFromIndex(totalWanted, taskIndex); if (querySize < 1) { // If we don't need any more metacards, we're finished done.set(true); return; } List<Metacard> metacards = ResultIterable .resultIterable(queryFacade::query, queryTemplate.apply(taskIndex), querySize).stream() .map(Result::getMetacard).collect(Collectors.toList()); if (metacards.size() < querySize) { done.set(true); } if (!metacards.isEmpty()) { ingestMetacards(ingestFacade, metacards); } printProgressAndFlush(start, Math.max(totalWanted, ingestedCount.get()), ingestedCount.get()); }); } executorService.shutdown(); try { executorService.awaitTermination(1, TimeUnit.SECONDS); } catch (InterruptedException e) { executorService.shutdownNow(); throw e; } printProgressAndFlush(start, Math.max(totalWanted, ingestedCount.get()), ingestedCount.get()); } else { // Single threaded ResultIterable iter; if (maxMetacards > 0) { iter = ResultIterable.resultIterable(queryFacade::query, queryTemplate.apply(1 + batchSize), maxMetacards); } else { iter = ResultIterable.resultIterable(queryFacade::query, queryTemplate.apply(1 + batchSize)); } Iterables.partition(iter, batchSize).forEach((batch) -> { printProgressAndFlush(start, totalWanted, ingestedCount.get()); if (batch.isEmpty()) { return; } ingestMetacards(ingestFacade, batch.stream().map(Result::getMetacard).collect(Collectors.toList())); }); } printProgressAndFlush(start, totalWanted, ingestedCount.get()); if (failedCount.get() > 0) { LOGGER.info("Not all records were ingested. [{}] failed", failedCount.get()); if (StringUtils.isNotBlank(failedDir)) { try { writeFailedMetacards(failedMetacards); } catch (IOException e) { console.println("Error occurred while writing failed metacards to failedDir."); } } } }
From source file:google.registry.tools.GenerateEscrowDepositCommand.java
private <T extends EppResource> Iterable<T> load(final Iterable<T> resources) { return FluentIterable.from(Iterables.partition(Iterables.transform(resources, new Function<T, Result<T>>() { @Override/* w w w. j a v a2 s . c om*/ public Result<T> apply(T resource) { return EppResourceUtils.loadAtPointInTime(resource, watermark); } }), 1000)).transformAndConcat(new Function<Iterable<Result<T>>, Iterable<T>>() { @Override public Iterable<T> apply(Iterable<Result<T>> results) { return Iterables.transform(results, new Function<Result<T>, T>() { @Override public T apply(Result<T> result) { return result.now(); } }); } }).filter(Predicates.notNull()); }
From source file:com.b2international.snowowl.snomed.datastore.id.cis.CisSnomedIdentifierService.java
@Override public void register(final Set<String> componentIds) { LOGGER.debug(String.format("Registering {} component IDs.", componentIds.size())); final Map<String, SctId> sctIds = getSctIds(componentIds); final Map<String, SctId> problemSctIds = ImmutableMap.copyOf(Maps.filterValues(sctIds, Predicates.<SctId>not(Predicates.or(SctId::isAvailable, SctId::isReserved, SctId::isAssigned)))); if (!problemSctIds.isEmpty()) { throw new SctIdStatusException( "Cannot register %s component IDs because they are not available, reserved, or already assigned.", problemSctIds);/*from ww w. j a v a2s . c o m*/ } final Map<String, SctId> availableOrReservedSctIds = ImmutableMap .copyOf(Maps.filterValues(sctIds, Predicates.or(SctId::isAvailable, SctId::isReserved))); if (availableOrReservedSctIds.isEmpty()) { return; } HttpPost registerRequest = null; String currentNamespace = null; try { if (availableOrReservedSctIds.size() > 1) { final Multimap<String, String> componentIdsByNamespace = toNamespaceMultimap( availableOrReservedSctIds.keySet()); for (final Entry<String, Collection<String>> entry : componentIdsByNamespace.asMap().entrySet()) { currentNamespace = entry.getKey(); for (final Collection<String> bulkIds : Iterables.partition(entry.getValue(), BULK_LIMIT)) { LOGGER.debug( String.format("Sending bulk registration request for namespace %s with size %d.", currentNamespace, bulkIds.size())); registerRequest = httpPost(String.format("sct/bulk/register?token=%s", getToken()), createBulkRegistrationData(bulkIds)); execute(registerRequest); } } } else { final String componentId = Iterables.getOnlyElement(availableOrReservedSctIds.keySet()); currentNamespace = SnomedIdentifiers.getNamespace(componentId); registerRequest = httpPost(String.format("sct/register?token=%s", getToken()), createRegistrationData(componentId)); execute(registerRequest); } } catch (IOException e) { throw new SnowowlRuntimeException( String.format("Exception while reserving IDs for namespace %s.", currentNamespace), e); } finally { release(registerRequest); } }
From source file:org.dcache.alarms.dao.impl.DataNucleusLogEntryStore.java
@Override public long remove(Collection<LogEntry> selected) { if (selected.isEmpty()) { return 0; }//from w w w . j a va 2 s .c o m PersistenceManager deleteManager = pmf.getPersistenceManager(); if (deleteManager == null) { return 0; } /** * Too many deletes in a single transaction will * cause errors, even StackOverflow exceptions. * Break up into smaller batches. */ long[] total = new long[] { 0L }; try { Iterables.partition(selected, 100).forEach((partition) -> { AlarmDAOFilter filter = AlarmJDOUtils.getIdFilter(partition); total[0] += doRemove(filter, deleteManager); }); } finally { deleteManager.close(); } return total[0]; }