Example usage for com.google.common.collect Iterators partition

Introduction

In this page you can find the example usage for com.google.common.collect Iterators partition.

Prototype

public static <T> UnmodifiableIterator<List<T>> partition(Iterator<T> iterator, int size) 

Source Link

Document

Divides an iterator into unmodifiable sublists of the given size (the final list may be smaller).

Usage

From source file:org.eclipse.scada.ae.server.common.event.AbstractEventQueryImpl.java

@Override
public synchronized void addListener(final EventListener eventListener) {
    this.listeners.add(eventListener);

    final UnmodifiableIterator<List<Event>> it = Iterators
            .partition(AbstractEventQueryImpl.this.events.iterator(), chunkSize);
    while (it.hasNext()) {
        final List<org.eclipse.scada.ae.Event> chunk = it.next();
        this.executor.execute(new Runnable() {

            @Override/*from  w  ww. jav  a2  s.c om*/
            public void run() {
                eventListener.handleEvent(chunk);
            }
        });
    }

}

---

From source file:net.tridentsdk.server.threads.TaskGroup.java

/**
 * Partitions the items and processes them
 *
 * @param consumer the action to process each item with
 *//*from  w ww. j  a  v  a  2  s .  c om*/
public void using(Consumer<T> consumer) {
    Iterator<T> iterator = items.iterator();
    Iterator<List<T>> parted = Iterators.partition(iterator, per);

    while (parted.hasNext()) {
        List<T> list = parted.next();
        factory.execute(() -> list.forEach(consumer::accept));
    }
}

---

From source file:org.openscada.ae.server.common.event.AbstractEventQueryImpl.java

@Override
public synchronized void addListener(final EventListener eventListener) {
    this.listeners.add(eventListener);

    final UnmodifiableIterator<List<Event>> it = Iterators
            .partition(AbstractEventQueryImpl.this.events.iterator(), chunkSize);
    while (it.hasNext()) {
        final List<org.openscada.ae.Event> chunk = it.next();
        this.executor.execute(new Runnable() {

            @Override//w  w w  . j a va 2  s  .  c  o  m
            public void run() {
                eventListener.handleEvent(chunk);
            }
        });
    }

}

---

From source file:com.torodb.torod.pipeline.impl.SameThreadInsertPipeline.java

@Override
public void insert(Stream<KvDocument> docs) throws RollbackException, UserException {
    D2RTranslationBatchFunction d2rFun = new D2RTranslationBatchFunction(translatorFactory, metaDb,
            mutableMetaCollection);/* ww  w . j av a  2  s . c o m*/
    DefaultToBackendFunction r2BackendFun = new DefaultToBackendFunction(jobFactory, metaDb,
            mutableMetaCollection);

    try {
        Iterators.partition(docs.iterator(), docBatchSize).forEachRemaining(list -> {
            CollectionData collData = d2rFun.apply(list);
            Iterable<BackendTransactionJob> jobs = r2BackendFun.apply(collData);
            jobs.forEach(Unchecked.consumer(job -> job.execute(backendConnection)));
        });
    } catch (UncheckedException ex) {
        Throwable cause = ex.getCause();
        if (cause != null && cause instanceof UserException) {
            throw (UserException) cause;
        }
        throw ex;
    }
}

---

From source file:org.locationtech.geogig.storage.internal.ObjectStoreDiffObjectIterator.java

private @Nullable DiffObjectInfo<T> computeNext() {
    if (nextBatch != null && nextBatch.hasNext()) {
        return nextBatch.next();
    }/* w  ww .ja v a 2 s  .  com*/
    if (!nodes.hasNext()) {
        return null;
    }

    final int queryBatchSize = this.getAllBatchSize;

    List<DiffEntry> nextEntries = Iterators.partition(this.nodes, queryBatchSize).next();
    Set<ObjectId> leftEntriesIds = new HashSet<>();
    Set<ObjectId> rightEntriesIds = this.leftStore == this.rightStore ? leftEntriesIds : new HashSet<>();

    nextEntries.forEach((e) -> {
        ObjectId oldId = e.oldObjectId();
        ObjectId newId = e.newObjectId();
        if (!oldId.isNull()) {
            leftEntriesIds.add(oldId);
        }
        if (!newId.isNull()) {
            rightEntriesIds.add(newId);
        }
    });

    Iterator<T> objects = leftStore.getAll(leftEntriesIds, NOOP_LISTENER, this.type);
    if (rightEntriesIds != leftEntriesIds && !rightEntriesIds.isEmpty()) {
        objects = Iterators.concat(objects, rightStore.getAll(rightEntriesIds, NOOP_LISTENER, this.type));
    }
    Map<ObjectId, T> objectsById = new HashMap<>();
    objects.forEachRemaining((o) -> objectsById.putIfAbsent(o.getId(), o));
    nextBatch = createBatch(nextEntries, objectsById);
    return computeNext();
}

---

From source file:org.locationtech.geogig.osm.internal.history.HistoryDownloader.java

/**
 * @return the next available changeset, or absent if reached the last one
 * @throws IOException//from   w  w  w .  j a v a2s  .co m
 * @throws InterruptedException
 */
public Iterator<Changeset> fetchChangesets() {

    Range<Long> range = Range.closed(initialChangeset, finalChangeset);
    ContiguousSet<Long> changesetIds = ContiguousSet.create(range, DiscreteDomain.longs());
    final int fetchSize = 100;
    Iterator<List<Long>> partitions = Iterators.partition(changesetIds.iterator(), fetchSize);

    Function<List<Long>, Iterator<Changeset>> asChangesets = new Function<List<Long>, Iterator<Changeset>>() {
        @Override
        public Iterator<Changeset> apply(List<Long> batchIds) {

            Iterable<Changeset> changesets = downloader.fetchChangesets(batchIds);

            for (Changeset changeset : changesets) {
                if (filter.apply(changeset)) {
                    Supplier<Optional<File>> changesFile;
                    changesFile = downloader.fetchChanges(changeset.getId());
                    Supplier<Optional<Iterator<Change>>> changes = new ChangesSupplier(changesFile);
                    changeset.setChanges(changes);
                }
            }

            return changesets.iterator();
        }
    };

    Iterator<Iterator<Changeset>> changesets = Iterators.transform(partitions, asChangesets);
    Iterator<Changeset> concat = Iterators.concat(changesets);
    return concat;
}

---

From source file:org.calrissian.mango.collect.CloseableIterators.java

/**
 * Divides a closeableiterator into unmodifiable sublists of the given size (the final
 * list may be smaller). For example, partitioning a closeableiterator containing
 * {@code [a, b, c, d, e]} with a partition size of 3 yields {@code
 * [[a, b, c], [d, e]]} -- an outer iterator containing two inner lists of
 * three and two elements, all in the original order.
 *
 * <p>The returned lists implement {@link java.util.RandomAccess}.
 *///from w w w.  jav  a 2s. c o m
public static <T> CloseableIterator<List<T>> partition(final CloseableIterator<T> iterator, final int size) {
    return wrap(Iterators.partition(iterator, size), iterator);
}

---

From source file:org.eclipse.scada.ae.server.common.event.pool.internal.EventPoolImpl.java

private void loadFromStorage() {
    // load initial set from storage, but restrict it to *daysToRetrieve* days
    try {// ww  w .  jav a2 s . c  o  m
        final long t = System.currentTimeMillis();
        // retrieve data per day, to restrict database load
        for (int daysBack = 1; daysBack <= daysToRetrieve; daysBack++) {
            final Calendar calStart = new GregorianCalendar();
            final Calendar calEnd = new GregorianCalendar();
            calStart.setTimeInMillis(t);
            calStart.add(Calendar.DAY_OF_YEAR, -daysBack);
            calEnd.setTimeInMillis(t);
            calEnd.add(Calendar.DAY_OF_YEAR, -daysBack + 1);
            final StringBuilder filter = new StringBuilder();
            filter.append("(&");
            if (this.filter != null) {
                filter.append(this.filter);
            }
            filter.append("(sourceTimestamp>=" + isoDateFormat.format(calStart.getTime()) + ")");
            if (daysBack > 1) {
                filter.append("(sourceTimestamp<" + isoDateFormat.format(calEnd.getTime()) + ")");
            }
            filter.append(")");
            logger.debug("load events from filter: " + filter);
            final Query query = this.storage.query(filter.toString());
            try {
                int count;
                synchronized (this) {
                    count = this.events.getCapacity();
                }

                final Collection<Event> result = query.getNext(count);

                logger.debug("Loaded {} entries from storage", result.size());
                synchronized (this) {
                    this.events.addAll(result);

                    final UnmodifiableIterator<List<Event>> it = Iterators.partition(this.events.iterator(),
                            chunkSize);
                    while (it.hasNext()) {
                        final List<org.eclipse.scada.ae.Event> chunk = it.next();
                        notifyEvent(chunk);
                    }
                }
            } finally {
                query.dispose();
            }
            if (this.events.size() >= this.events.getCapacity()) {
                return;
            }
        }
        logger.debug("load of events complete");
    } catch (final Exception e) {
        logger.error("loadFromStorage failed", e);
    }
}

---

From source file:org.openscada.ae.server.common.event.pool.internal.EventPoolImpl.java

private void loadFromStorage() {
    // load initial set from storage, but restrict it to *daysToRetrieve* days
    try {/*from  w ww .ja  v  a 2 s  .co m*/
        final long t = System.currentTimeMillis();
        // retrieve data per day, to restrict database load
        for (int daysBack = 1; daysBack <= daysToRetrieve; daysBack++) {
            final Calendar calStart = new GregorianCalendar();
            final Calendar calEnd = new GregorianCalendar();
            calStart.setTimeInMillis(t);
            calStart.add(Calendar.DAY_OF_YEAR, -daysBack);
            calEnd.setTimeInMillis(t);
            calEnd.add(Calendar.DAY_OF_YEAR, -daysBack + 1);
            final StringBuilder filter = new StringBuilder();
            filter.append("(&");
            filter.append(this.filter);
            filter.append("(sourceTimestamp>=" + isoDateFormat.format(calStart.getTime()) + ")");
            if (daysBack > 1) {
                filter.append("(sourceTimestamp<" + isoDateFormat.format(calEnd.getTime()) + ")");
            }
            filter.append(")");
            logger.debug("load events from filter: " + filter);
            final Query query = this.storage.query(filter.toString());
            try {
                int count;
                synchronized (this) {
                    count = this.events.getCapacity();
                }

                final Collection<Event> result = query.getNext(count);

                logger.debug("Loaded {} entries from storage", result.size());
                synchronized (this) {
                    this.events.addAll(result);

                    final UnmodifiableIterator<List<Event>> it = Iterators.partition(this.events.iterator(),
                            chunkSize);
                    while (it.hasNext()) {
                        final List<org.openscada.ae.Event> chunk = it.next();
                        notifyEvent(chunk);
                    }
                }
            } finally {
                query.dispose();
            }
            if (this.events.size() >= this.events.getCapacity()) {
                return;
            }
        }
        logger.debug("load of events complete");
    } catch (final Exception e) {
        logger.error("loadFromStorage failed", e);
    }
}

---

From source file:org.geogit.storage.sqlite.XerialObjectDatabase.java

/**
 * Override to optimize batch insert./*from w ww .  j a  va 2s  . c  om*/
 */
@Override
public void putAll(final Iterator<? extends RevObject> objects, final BulkOpListener listener) {
    Preconditions.checkState(isOpen(), "No open database connection");
    new DbOp<Void>() {
        @Override
        protected boolean isAutoCommit() {
            return false;
        }

        @Override
        protected Void doRun(Connection cx) throws SQLException, IOException {
            // use INSERT OR IGNORE to deal with duplicates cleanly
            String sql = format("INSERT OR IGNORE INTO %s (object,id) VALUES (?,?)", OBJECTS);
            PreparedStatement stmt = open(cx.prepareStatement(log(sql, LOG)));

            // partition the objects into chunks for batch processing
            Iterator<List<? extends RevObject>> it = (Iterator) Iterators.partition(objects, partitionSize);

            while (it.hasNext()) {
                List<? extends RevObject> objs = it.next();
                for (RevObject obj : objs) {
                    stmt.setBytes(1, ByteStreams.toByteArray(writeObject(obj)));
                    stmt.setString(2, obj.getId().toString());
                    stmt.addBatch();
                }

                notifyInserted(stmt.executeBatch(), objs, listener);
                stmt.clearParameters();
            }
            cx.commit();

            return null;
        }
    }.run(cx);
}

---

• «
• 1
• 2
• 3
• »