List of usage examples for com.google.common.collect Iterables partition
public static <T> Iterable<List<T>> partition(final Iterable<T> iterable, final int size)
From source file:com.palantir.atlasdb.transaction.impl.SerializableTransaction.java
private void verifyCells(Transaction ro) { for (String table : cellsRead.keySet()) { final ConcurrentNavigableMap<Cell, byte[]> readsForTable = getReadsForTable(table); for (Iterable<Cell> batch : Iterables.partition(cellsRead.get(table), 1000)) { if (writesByTable.get(table) != null) { // We don't want to verify any reads that we wrote to cause we will just read our own values. // NB: If the value has changed between read and write, our normal SI checking handles this case batch = Iterables.filter(batch, Predicates.not(Predicates.in(writesByTable.get(table).keySet()))); }/*from ww w . ja va 2s . co m*/ ImmutableSet<Cell> batchSet = ImmutableSet.copyOf(batch); Map<Cell, byte[]> currentBatch = ro.get(table, batchSet); ImmutableMap<Cell, byte[]> originalReads = Maps.toMap( Sets.intersection(batchSet, readsForTable.keySet()), Functions.forMap(readsForTable)); if (!areMapsEqual(currentBatch, originalReads)) { throw TransactionSerializableConflictException.create(table, getTimestamp(), System.currentTimeMillis() - timeCreated); } } } }
From source file:com.freiheit.fuava.simplebatch.BatchJob.java
protected void processWithStreams(final DelegatingProcessingResultListener<OriginalInput, Output> listeners, final boolean useParallelStream, final Iterable<Result<FetchedItem<OriginalInput>, OriginalInput>> sourceIterable) { final Iterable<List<Result<FetchedItem<OriginalInput>, OriginalInput>>> partitions = Iterables .partition(sourceIterable, processingBatchSize); StreamSupport.stream(partitions.spliterator(), parallel) .forEach(new CallProcessor(listeners, panicCallback)); }
From source file:org.jclouds.openstack.swift.v1.blobstore.RegionScopedSwiftBlobStore.java
/** * Delete multiple single-part objects. Note that this does not remove the * subobjects of a multi-part upload./*from w ww .j a v a2 s. c o m*/ */ @Override public void removeBlobs(String container, Iterable<String> names) { BulkApi bulkApi = api.getBulkApi(regionId); for (List<String> partition : Iterables.partition(names, 1000)) { ImmutableList.Builder<String> builder = ImmutableList.builder(); for (String name : partition) { builder.add(container + "/" + name); } bulkApi.bulkDelete(builder.build()); } }
From source file:org.locationtech.geogig.storage.postgresql.v9.PGConflictsDatabase.java
@Override public void removeConflicts(final @Nullable String ns, final Iterable<String> paths) { checkNotNull(paths, "paths is null"); final String namespace = namespace(ns); final String sql = format("DELETE FROM %s WHERE repository = ? AND namespace = ? AND path = ANY(?)", conflictsTable);/*from w ww .jav a 2 s . com*/ try (Connection cx = PGStorage.newConnection(dataSource)) { cx.setAutoCommit(false); try (PreparedStatement ps = cx.prepareStatement(sql)) { final int partitionSize = 1000; Iterable<List<String>> partitions = Iterables.partition(paths, partitionSize); for (List<String> partition : partitions) { String[] pathsArg = partition.toArray(new String[partition.size()]); Array array = cx.createArrayOf("varchar", pathsArg); ps.clearParameters(); ps.setInt(1, repositoryId); ps.setString(2, namespace); ps.setArray(3, array); ps.executeUpdate(); } cx.commit(); } catch (SQLException e) { cx.rollback(); throw e; } finally { cx.setAutoCommit(true); } } catch (SQLException e) { throw new RuntimeException(e); } }
From source file:com.b2international.snowowl.snomed.datastore.id.cis.CisSnomedIdentifierService.java
@Override public void publish(final Set<String> componentIds) { LOGGER.debug("Publishing {} component IDs.", componentIds.size()); final Map<String, SctId> sctIds = getSctIds(componentIds); final Map<String, SctId> problemSctIds = ImmutableMap.copyOf(Maps.filterValues(sctIds, Predicates.<SctId>not(Predicates.or(SctId::isAssigned, SctId::isPublished)))); HttpPut deprecateRequest = null;/*w w w. ja v a 2s.c om*/ String currentNamespace = null; try { final Map<String, SctId> assignedSctIds = ImmutableMap .copyOf(Maps.filterValues(sctIds, SctId::isAssigned)); if (!assignedSctIds.isEmpty()) { if (assignedSctIds.size() > 1) { final Multimap<String, String> componentIdsByNamespace = toNamespaceMultimap( assignedSctIds.keySet()); for (final Entry<String, Collection<String>> entry : componentIdsByNamespace.asMap() .entrySet()) { currentNamespace = entry.getKey(); for (final Collection<String> bulkIds : Iterables.partition(entry.getValue(), BULK_LIMIT)) { LOGGER.debug( String.format("Sending bulk publication request for namespace %s with size %d.", currentNamespace, bulkIds.size())); deprecateRequest = httpPut(String.format("sct/bulk/publish?token=%s", getToken()), createBulkPublishData(currentNamespace, bulkIds)); execute(deprecateRequest); } } } else { final String componentId = Iterables.getOnlyElement(assignedSctIds.keySet()); currentNamespace = SnomedIdentifiers.getNamespace(componentId); deprecateRequest = httpPut(String.format("sct/publish?token=%s", getToken()), createPublishData(componentId)); execute(deprecateRequest); } } if (!problemSctIds.isEmpty()) { throw new SctIdStatusException( "Cannot publish %s component IDs because they are not assigned or already published.", problemSctIds); } } catch (IOException e) { throw new SnowowlRuntimeException( String.format("Exception while publishing IDs for namespace %s.", currentNamespace), e); } finally { release(deprecateRequest); } }
From source file:org.locationtech.geogig.storage.postgresql.PGConflictsDatabase.java
@Override public Set<String> findConflicts(@Nullable String namespace, Set<String> paths) { checkNotNull(paths, "paths is null"); Set<String> matches = new HashSet<>(); namespace = namespace(namespace);// w w w . j av a2s.c om final int partitionSize = 1000; final String sql = format("SELECT path FROM %s WHERE repository = ? AND namespace = ? AND path = ANY(?)", conflictsTable); try (Connection cx = PGStorage.newConnection(dataSource)) { cx.setAutoCommit(true); try (PreparedStatement ps = cx.prepareStatement(sql)) { Iterable<List<String>> partitions = Iterables.partition(paths, partitionSize); for (List<String> partition : partitions) { String[] pathsArg = partition.toArray(new String[partition.size()]); Array array = cx.createArrayOf("varchar", pathsArg); ps.clearParameters(); ps.setInt(1, repositoryId); ps.setString(2, namespace); ps.setArray(3, array); try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { matches.add(rs.getString(1)); } } } } catch (SQLException e) { throw e; } } catch (SQLException e) { throw propagate(e); } return matches; }
From source file:com.google.googlejavaformat.java.JavaInputAstVisitor.java
public boolean visitArrayInitializer(List<? extends ExpressionTree> expressions) { int cols;/*w ww . jav a 2 s . c om*/ if (expressions.isEmpty()) { tokenBreakTrailingComment("{", plusTwo); if (builder.peekToken().equals(Optional.of(","))) { token(","); } token("}", plusTwo); } else if ((cols = argumentsAreTabular(expressions)) != -1) { builder.open(plusTwo); token("{"); builder.forcedBreak(); boolean first = true; for (Iterable<? extends ExpressionTree> row : Iterables.partition(expressions, cols)) { if (!first) { builder.forcedBreak(); } builder.open(row.iterator().next().getKind() == NEW_ARRAY || cols == 1 ? ZERO : plusFour); boolean firstInRow = true; for (ExpressionTree item : row) { if (!firstInRow) { token(","); builder.breakToFill(" "); } scan(item, null); firstInRow = false; } builder.guessToken(","); builder.close(); first = false; } builder.breakOp(minusTwo); builder.close(); token("}", plusTwo); } else { // Special-case the formatting of array initializers inside annotations // to more eagerly use a one-per-line layout. boolean inMemberValuePair = false; // walk up past the enclosing NewArrayTree (and maybe an enclosing AssignmentTree) TreePath path = getCurrentPath(); for (int i = 0; i < 2; i++) { if (path == null) { break; } if (path.getLeaf().getKind() == ANNOTATION) { inMemberValuePair = true; break; } path = path.getParentPath(); } boolean shortItems = hasOnlyShortItems(expressions); boolean allowFilledElementsOnOwnLine = shortItems || !inMemberValuePair; builder.open(plusTwo); tokenBreakTrailingComment("{", plusTwo); boolean hasTrailingComma = hasTrailingToken(builder.getInput(), expressions, ","); builder.breakOp(hasTrailingComma ? FillMode.FORCED : FillMode.UNIFIED, "", ZERO); if (allowFilledElementsOnOwnLine) { builder.open(ZERO); } boolean first = true; FillMode fillMode = shortItems ? FillMode.INDEPENDENT : FillMode.UNIFIED; for (ExpressionTree expression : expressions) { if (!first) { token(","); builder.breakOp(fillMode, " ", ZERO); } scan(expression, null); first = false; } builder.guessToken(","); if (allowFilledElementsOnOwnLine) { builder.close(); } builder.breakOp(minusTwo); builder.close(); token("}", plusTwo); } return false; }
From source file:omero.cmd.graphs.DuplicateI.java
/** * Duplicate model object properties, linking them as appropriate with each other and with other model objects. * @throws GraphException if duplication failed *//* ww w . j av a 2 s .c o m*/ private void setDuplicatePropertyValues() throws GraphException { /* organize duplicate index by class name and ID */ final Map<Entry<String, Long>, IObject> duplicatesByOriginalClassAndId = new HashMap<Entry<String, Long>, IObject>(); for (final Entry<IObject, IObject> originalAndDuplicate : originalsToDuplicates.entrySet()) { final IObject original = originalAndDuplicate.getKey(); final String originalClass = Hibernate.getClass(original).getName(); final Long originalId = original.getId(); final IObject duplicate = originalAndDuplicate.getValue(); duplicatesByOriginalClassAndId.put(Maps.immutableEntry(originalClass, originalId), duplicate); } /* allow lookup regardless of if original is actually a Hibernate proxy object */ final Function<Object, Object> duplicateLookup = new Function<Object, Object>() { @Override public Object apply(Object original) { if (original instanceof IObject) { final String originalClass; if (original instanceof HibernateProxy) { originalClass = Hibernate.getClass(original).getName(); } else { originalClass = original.getClass().getName(); } final Long originalId = ((IObject) original).getId(); return duplicatesByOriginalClassAndId.get(Maps.immutableEntry(originalClass, originalId)); } else { return null; } } }; /* copy property values into duplicates and link with other model objects */ final Session session = helper.getSession(); for (final Entry<IObject, IObject> originalAndDuplicate : originalsToDuplicates.entrySet()) { final IObject original = originalAndDuplicate.getKey(); final IObject duplicate = originalAndDuplicate.getValue(); final String originalClass = Hibernate.getClass(original).getName(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("copying properties from " + originalClass + ":" + original.getId()); } try { /* process property values for a given object that is duplicated */ for (final String superclassName : graphPathBean.getSuperclassesOfReflexive(originalClass)) { /* process property values that link from the duplicate to other model objects */ for (final Entry<String, String> forwardLink : graphPathBean.getLinkedTo(superclassName)) { /* next forward link */ final String linkedClassName = forwardLink.getKey(); final String property = forwardLink.getValue(); /* ignore details for now, duplicates never preserve original ownership */ if (property.startsWith("details.")) { continue; } /* note which of the objects to which the original links should be ignored */ final Set<Long> linkedToIdsToIgnore = new HashSet<Long>(); for (final Entry<String, Collection<Long>> linkedToClassIds : graphTraversal .getLinkeds(superclassName, property, original.getId()).asMap().entrySet()) { final String linkedToClass = linkedToClassIds.getKey(); final Collection<Long> linkedToIds = linkedToClassIds.getValue(); if (classifier.getClass( Class.forName(linkedToClass).asSubclass(IObject.class)) == Inclusion.IGNORE) { linkedToIdsToIgnore.addAll(linkedToIds); } } /* check for another accessor for inaccessible properties */ if (graphPathBean.isPropertyAccessible(superclassName, property)) { /* copy the linking from the original's property over to the duplicate's */ Object value; try { value = PropertyUtils.getNestedProperty(original, property); } catch (NestedNullException e) { continue; } if (value instanceof Collection) { /* if a collection property, include only the objects that aren't to be ignored */ final Collection<IObject> valueCollection = (Collection<IObject>) value; final Collection<IObject> valueToCopy; if (value instanceof List) { valueToCopy = new ArrayList<IObject>(); } else if (value instanceof Set) { valueToCopy = new HashSet<IObject>(); } else { throw new GraphException("unexpected collection type: " + value.getClass()); } for (final IObject linkedTo : valueCollection) { if (!linkedToIdsToIgnore.contains(linkedTo.getId())) { valueToCopy.add(linkedTo); } } value = valueToCopy; } else if (value instanceof IObject) { /* if the property value is to be ignored then null it */ if (linkedToIdsToIgnore.contains(((IObject) value).getId())) { value = null; } } /* copy the property value, replacing originals with corresponding duplicates */ final Object duplicateValue = GraphUtil.copyComplexValue(duplicateLookup, value); try { PropertyUtils.setNestedProperty(duplicate, property, duplicateValue); } catch (NestedNullException e) { throw new GraphException( "cannot set property " + superclassName + '.' + property + " on duplicate"); } } else { /* this could be a one-to-many property with direct accessors protected */ final Class<? extends IObject> linkerClass = Class.forName(superclassName) .asSubclass(IObject.class); final Class<? extends IObject> linkedClass = Class.forName(linkedClassName) .asSubclass(IObject.class); final Method reader, writer; try { reader = linkerClass.getMethod("iterate" + StringUtils.capitalize(property)); writer = linkerClass.getMethod("add" + linkedClass.getSimpleName(), linkedClass); } catch (NoSuchMethodException | SecurityException e) { /* no luck, so ignore this property */ continue; } /* copy the linking from the original's property over to the duplicate's */ final Iterator<IObject> linkedTos = (Iterator<IObject>) reader.invoke(original); while (linkedTos.hasNext()) { final IObject linkedTo = linkedTos.next(); /* copy only links to other duplicates, as otherwise we may steal objects from the original */ final IObject duplicateOfLinkedTo = (IObject) duplicateLookup.apply(linkedTo); if (duplicateOfLinkedTo != null) { writer.invoke(duplicate, duplicateOfLinkedTo); } } } } /* process property values that link to the duplicate from other model objects */ for (final Entry<String, String> backwardLink : graphPathBean.getLinkedBy(superclassName)) { /* next backward link */ final String linkingClass = backwardLink.getKey(); final String property = backwardLink.getValue(); /* ignore inaccessible properties */ if (!graphPathBean.isPropertyAccessible(linkingClass, property)) { continue; } for (final Entry<String, Collection<Long>> linkedFromClassIds : graphTraversal .getLinkers(linkingClass, property, original.getId()).asMap().entrySet()) { final String linkedFromClass = linkedFromClassIds.getKey(); final Collection<Long> linkedFromIds = linkedFromClassIds.getValue(); if (classifier.getClass( Class.forName(linkedFromClass).asSubclass(IObject.class)) == Inclusion.IGNORE) { /* these linkers are to be ignored */ continue; } /* load the instances that link to the original */ final String rootQuery = "FROM " + linkedFromClass + " WHERE id IN (:ids)"; for (final List<Long> idsBatch : Iterables.partition(linkedFromIds, BATCH_SIZE)) { final List<IObject> linkers = session.createQuery(rootQuery) .setParameterList("ids", idsBatch).list(); for (final IObject linker : linkers) { if (originalsToDuplicates.containsKey(linker)) { /* ignore linkers that are to be duplicated, those are handled as forward links */ continue; } /* copy the linking from the original's property over to the duplicate's */ Object value; try { value = PropertyUtils.getNestedProperty(linker, property); } catch (NestedNullException e) { continue; } /* for linkers only adjust collection properties */ if (value instanceof Collection) { final Collection<IObject> valueCollection = (Collection<IObject>) value; final Collection<IObject> newDuplicates = new ArrayList<IObject>(); for (final IObject originalLinker : valueCollection) { final IObject duplicateOfValue = originalsToDuplicates .get(originalLinker); if (duplicateOfValue != null) { /* previous had just original, now include duplicate too */ newDuplicates.add(duplicateOfValue); } } valueCollection.addAll(newDuplicates); } } } } } /* process property values that do not relate to edges in the model object graph */ for (final String property : graphPathBean.getSimpleProperties(superclassName)) { /* ignore inaccessible properties */ if (!graphPathBean.isPropertyAccessible(superclassName, property)) { continue; } /* copy original property value to duplicate */ final Object value = PropertyUtils.getProperty(original, property); PropertyUtils.setProperty(duplicate, property, GraphUtil.copyComplexValue(duplicateLookup, value)); } } } catch (ClassNotFoundException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { throw new GraphException("failed to duplicate " + originalClass + ':' + original.getId()); } } }
From source file:org.locationtech.geogig.storage.postgresql.v9.PGConflictsDatabase.java
@Override public Set<String> findConflicts(@Nullable String namespace, Set<String> paths) { checkNotNull(paths, "paths is null"); Set<String> matches = new HashSet<>(); namespace = namespace(namespace);//from w w w . jav a 2s .c o m final int partitionSize = 1000; final String sql = format("SELECT path FROM %s WHERE repository = ? AND namespace = ? AND path = ANY(?)", conflictsTable); try (Connection cx = PGStorage.newConnection(dataSource)) { cx.setAutoCommit(true); try (PreparedStatement ps = cx.prepareStatement(sql)) { Iterable<List<String>> partitions = Iterables.partition(paths, partitionSize); for (List<String> partition : partitions) { String[] pathsArg = partition.toArray(new String[partition.size()]); Array array = cx.createArrayOf("varchar", pathsArg); ps.clearParameters(); ps.setInt(1, repositoryId); ps.setString(2, namespace); ps.setArray(3, array); try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { matches.add(rs.getString(1)); } } } } catch (SQLException e) { throw e; } } catch (SQLException e) { throw new RuntimeException(e); } return matches; }
From source file:org.candlepin.model.AbstractHibernateCurator.java
/** * Performs a direct SQL update or delete operation with a collection by breaking the collection * into chunks and repeatedly performing the update. * <p></p>/* w w w. j av a2 s .c om*/ * The parameter receiving the collection chunks must be the last parameter in the query and the * provided collection must support the subList operation. * * @param sql * The SQL statement to execute; must be an UPDATE or DELETE operation * * @param collection * The collection to be broken up into chunks * * @return * the number of rows updated as a result of this query */ protected int safeSQLUpdateWithCollection(String sql, Collection<?> collection, Object... params) { int count = 0; Session session = this.currentSession(); SQLQuery query = session.createSQLQuery(sql); for (List<?> block : Iterables.partition(collection, IN_OPERATOR_BLOCK_SIZE)) { int index = 1; if (params != null) { for (; index <= params.length; ++index) { query.setParameter(String.valueOf(index), params[index - 1]); } } query.setParameterList(String.valueOf(index), block); count += query.executeUpdate(); } return count; }