Example usage for com.google.common.collect Lists partition

Introduction

In this page you can find the example usage for com.google.common.collect Lists partition.

Prototype

public static <T> List<List<T>> partition(List<T> list, int size) 

Source Link

Document

Returns consecutive List#subList(int,int) sublists of a list, each of the same size (the final list may be smaller).

Usage

From source file:com.spotify.folsom.client.ascii.DefaultAsciiMemcacheClient.java

private ListenableFuture<List<GetResult<V>>> multiget(List<String> keys, boolean withCas) {
    final int size = keys.size();
    if (size == 0) {
        return Futures.immediateFuture(Collections.<GetResult<V>>emptyList());
    }//from   ww w.  j  a  v  a 2  s  .co  m

    final List<List<String>> keyPartition = Lists.partition(keys, MemcacheEncoder.MAX_MULTIGET_SIZE);
    final List<ListenableFuture<List<GetResult<byte[]>>>> futureList = new ArrayList<>(keyPartition.size());

    for (final List<String> part : keyPartition) {
        MultigetRequest request = MultigetRequest.create(part, charset, withCas);
        futureList.add(rawMemcacheClient.send(request));
    }

    final ListenableFuture<List<GetResult<byte[]>>> future = Utils.transform(Futures.allAsList(futureList),
            Utils.<GetResult<byte[]>>flatten());

    metrics.measureMultigetFuture(future);
    return transformerUtil.decodeList(future);
}

---

From source file:com.netflix.spinnaker.clouddriver.elasticsearch.model.ElasticSearchEntityTagsProvider.java

@Override
public void bulkIndex(Collection<EntityTags> multipleEntityTags) {
    Lists.partition(new ArrayList<>(multipleEntityTags), 1000).forEach(tags -> {
        Bulk.Builder builder = new Bulk.Builder().defaultIndex(activeElasticSearchIndex);

        for (EntityTags entityTags : tags) {
            builder = builder.addAction(new Index.Builder(
                    objectMapper.convertValue(prepareForWrite(objectMapper, entityTags), Map.class))
                            .index(activeElasticSearchIndex).type(entityTags.getEntityRef().getEntityType())
                            .id(entityTags.getId()).build());
        }/*from  w w  w  .  java  2  s  .c  o  m*/

        Bulk bulk = builder.build();
        try {
            JestResult jestResult = jestClient.execute(bulk);
            if (!jestResult.isSucceeded()) {
                throw new ElasticSearchException(
                        format("Failed to index bulk entity tags, reason: '%s'", jestResult.getErrorMessage()));
            }
        } catch (IOException e) {
            throw new ElasticSearchException(
                    format("Failed to index bulk entity tags, reason: '%s'", e.getMessage()));
        }
    });
}

---

From source file:com.carmanconsulting.cassidy.pojo.mapper.ColumnMapperPojoDao.java

private List<List<K>> breakIntoChunks(List<K> keys, int chunkSize) {
    return Lists.partition(keys, chunkSize);
}

---

From source file:com.stratio.cassandra.lucene.service.RowServiceWide.java

/**
 * {@inheritDoc}//from ww  w . j  a  v a  2 s.c o  m
 *
 * The {@link Row} is a logical one.
 */
@Override
protected List<Row> rows(List<SearchResult> searchResults, long timestamp, boolean relevance) {

    // Group key queries by partition keys
    Map<String, ScoreDoc> scoresByClusteringKey = new HashMap<>(searchResults.size());
    Map<DecoratedKey, List<CellName>> keys = new HashMap<>();
    for (SearchResult searchResult : searchResults) {
        DecoratedKey partitionKey = searchResult.getPartitionKey();
        CellName clusteringKey = searchResult.getClusteringKey();
        ScoreDoc scoreDoc = searchResult.getScoreDoc();
        String rowHash = rowMapper.hash(partitionKey, clusteringKey);
        scoresByClusteringKey.put(rowHash, scoreDoc);
        List<CellName> clusteringKeys = keys.get(partitionKey);
        if (clusteringKeys == null) {
            clusteringKeys = new ArrayList<>();
            keys.put(partitionKey, clusteringKeys);
        }
        clusteringKeys.add(clusteringKey);
    }

    List<Row> rows = new ArrayList<>(searchResults.size());
    for (Map.Entry<DecoratedKey, List<CellName>> entry : keys.entrySet()) {
        DecoratedKey partitionKey = entry.getKey();
        for (List<CellName> clusteringKeys : Lists.partition(entry.getValue(), ROWS_PER_SLICE_QUERY)) {
            Map<CellName, ColumnFamily> partitionRows = rows(partitionKey, clusteringKeys, timestamp);
            for (Map.Entry<CellName, ColumnFamily> entry1 : partitionRows.entrySet()) {
                CellName clusteringKey = entry1.getKey();
                ColumnFamily columnFamily = entry1.getValue();
                Row row = new Row(partitionKey, columnFamily);
                if (relevance) {
                    String rowHash = rowMapper.hash(partitionKey, clusteringKey);
                    ScoreDoc scoreDoc = scoresByClusteringKey.get(rowHash);
                    row = addScoreColumn(row, timestamp, scoreDoc);
                }
                rows.add(row);
            }
        }
    }
    return rows;
}

---

From source file:org.deeplearning4j.models.glove.Glove.java

public void doIteration(final int i, List<Pair<String, String>> pairList,
        final Counter<Integer> errorPerIteration, final AtomicInteger processed, final AtomicInteger countUp) {
    log.info("Iteration " + i);
    if (shuffle)//from   w  w  w  .  j av  a  2  s  . co m
        Collections.shuffle(pairList, new java.util.Random());
    List<List<Pair<String, String>>> miniBatches = Lists.partition(pairList, batchSize);
    ActorSystem actor = ActorSystem.create();
    Parallelization.iterateInParallel(miniBatches,
            new Parallelization.RunnableWithParams<List<Pair<String, String>>>() {
                @Override
                public void run(List<Pair<String, String>> currentItem, Object[] args) {
                    List<Pair<VocabWord, VocabWord>> send = new ArrayList<>();
                    for (Pair<String, String> next : currentItem) {
                        String w1 = next.getFirst();
                        String w2 = next.getSecond();
                        VocabWord vocabWord = vocab().wordFor(w1);
                        VocabWord vocabWord1 = vocab().wordFor(w2);
                        send.add(new Pair<>(vocabWord, vocabWord1));

                    }

                    jobQueue.add(new Pair<>(i, send));
                }
            }, actor);

    actor.shutdown();

    Parallelization.runInParallel(numWorkers, new Runnable() {
        @Override
        public void run() {
            while (processed.get() > 0 || !jobQueue.isEmpty()) {
                Pair<Integer, List<Pair<VocabWord, VocabWord>>> work = jobQueue.poll();
                if (work == null)
                    continue;
                List<Pair<VocabWord, VocabWord>> batch = work.getSecond();

                for (Pair<VocabWord, VocabWord> pair : batch) {
                    VocabWord w1 = pair.getFirst();
                    VocabWord w2 = pair.getSecond();
                    double weight = getCount(w1.getWord(), w2.getWord());
                    if (weight <= 0) {
                        countUp.incrementAndGet();
                        processed.decrementAndGet();
                        continue;

                    }
                    errorPerIteration.incrementCount(work.getFirst(),
                            lookupTable().iterateSample(w1, w2, weight));
                    countUp.incrementAndGet();
                    if (countUp.get() % 10000 == 0)
                        log.info("Processed " + countUp.get() + " co occurrences");
                    processed.decrementAndGet();
                }

            }
        }
    }, true);
}

---

From source file:com.google.cloud.pubsub.spi.v1.PollingSubscriberConnection.java

@Override
public void sendAckOperations(List<String> acksToSend, List<PendingModifyAckDeadline> ackDeadlineExtensions) {
    // Send the modify ack deadlines in bundles as not to exceed the max request
    // size.//w w  w . ja  v  a2 s  .co m
    for (PendingModifyAckDeadline modifyAckDeadline : ackDeadlineExtensions) {
        for (List<String> ackIdChunk : Lists.partition(modifyAckDeadline.ackIds, MAX_PER_REQUEST_CHANGES)) {
            stub.withDeadlineAfter(DEFAULT_TIMEOUT.getMillis(), TimeUnit.MILLISECONDS)
                    .modifyAckDeadline(ModifyAckDeadlineRequest.newBuilder().setSubscription(subscription)
                            .addAllAckIds(ackIdChunk)
                            .setAckDeadlineSeconds(modifyAckDeadline.deadlineExtensionSeconds).build());
        }
    }

    for (List<String> ackChunk : Lists.partition(acksToSend, MAX_PER_REQUEST_CHANGES)) {
        stub.withDeadlineAfter(DEFAULT_TIMEOUT.getMillis(), TimeUnit.MILLISECONDS).acknowledge(
                AcknowledgeRequest.newBuilder().setSubscription(subscription).addAllAckIds(ackChunk).build());
    }
}

---

From source file:com.google.cloud.pubsub.v1.PollingSubscriberConnection.java

@Override
public void sendAckOperations(List<String> acksToSend, List<PendingModifyAckDeadline> ackDeadlineExtensions) {
    // Send the modify ack deadlines in batches as not to exceed the max request
    // size.//  www . j  a v a 2s.c  o m
    for (PendingModifyAckDeadline modifyAckDeadline : ackDeadlineExtensions) {
        for (List<String> ackIdChunk : Lists.partition(modifyAckDeadline.ackIds, MAX_PER_REQUEST_CHANGES)) {
            stub.withDeadlineAfter(DEFAULT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)
                    .modifyAckDeadline(ModifyAckDeadlineRequest.newBuilder()
                            .setSubscription(subscription.getName()).addAllAckIds(ackIdChunk)
                            .setAckDeadlineSeconds(modifyAckDeadline.deadlineExtensionSeconds).build());
        }
    }

    for (List<String> ackChunk : Lists.partition(acksToSend, MAX_PER_REQUEST_CHANGES)) {
        stub.withDeadlineAfter(DEFAULT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).acknowledge(AcknowledgeRequest
                .newBuilder().setSubscription(subscription.getName()).addAllAckIds(ackChunk).build());
    }
}

---

From source file:com.romeikat.datamessie.core.sync.service.template.withIdAndVersion.EntityWithIdAndVersionSynchronizer.java

private List<Long> loadLhsIds(final List<Long> rhsIds) {
    final List<Long> lhsIds = Collections.synchronizedList(Lists.newArrayListWithExpectedSize(rhsIds.size()));
    final List<List<Long>> rhsIdsBatches = Lists.partition(rhsIds, batchSizeEntities);
    new ParallelProcessing<List<Long>>(sessionFactorySyncSource, rhsIdsBatches, parallelismFactor) {
        @Override/*from   w  ww  . ja va  2  s. c o  m*/
        public void doProcessing(final HibernateSessionProvider lhsSessionProvider,
                final List<Long> rhsIdsBatch) {
            final Collection<Long> lhsIdsBatch = dao.getIds(lhsSessionProvider.getStatelessSession(),
                    rhsIdsBatch);
            lhsIds.addAll(lhsIdsBatch);
        }
    };
    return lhsIds;
}

---

From source file:org.deeplearning4j.models.glove.LegacyGlove.java

public void doIteration(final int i, List<Pair<String, String>> pairList,
        final Counter<Integer> errorPerIteration, final AtomicInteger processed, final AtomicInteger countUp) {
    log.info("Iteration " + i);
    if (shuffle)/*  w w  w.j  a  va2 s. co m*/
        Collections.shuffle(pairList, new java.util.Random());
    List<List<Pair<String, String>>> miniBatches = Lists.partition(pairList, batchSize);
    ActorSystem actor = ActorSystem.create();
    Parallelization.iterateInParallel(miniBatches,
            new Parallelization.RunnableWithParams<List<Pair<String, String>>>() {
                @Override
                public void run(List<Pair<String, String>> currentItem, Object[] args) {
                    List<Pair<VocabWord, VocabWord>> send = new ArrayList<>();
                    for (Pair<String, String> next : currentItem) {
                        String w1 = next.getFirst();
                        String w2 = next.getSecond();
                        VocabWord vocabWord = vocab().wordFor(w1);
                        VocabWord vocabWord1 = vocab().wordFor(w2);
                        send.add(new Pair<>(vocabWord, vocabWord1));

                    }

                    jobQueue.add(new Pair<>(i, send));
                }
            }, actor);

    actor.shutdown();

    Parallelization.runInParallel(numWorkers, new Runnable() {
        @Override
        public void run() {
            while (processed.get() > 0 || !jobQueue.isEmpty()) {
                Pair<Integer, List<Pair<VocabWord, VocabWord>>> work = jobQueue.poll();
                if (work == null)
                    continue;
                List<Pair<VocabWord, VocabWord>> batch = work.getSecond();

                for (Pair<VocabWord, VocabWord> pair : batch) {
                    VocabWord w1 = pair.getFirst();
                    VocabWord w2 = pair.getSecond();
                    double weight = getCount(w1.getWord(), w2.getWord());
                    if (weight <= 0) {
                        countUp.incrementAndGet();
                        processed.decrementAndGet();
                        continue;

                    }
                    //errorPerIteration.incrementCount(work.getFirst(),lookupTable().iterateSample(w1, w2, weight));
                    countUp.incrementAndGet();
                    if (countUp.get() % 10000 == 0)
                        log.info("Processed " + countUp.get() + " co occurrences");
                    processed.decrementAndGet();
                }

            }
        }
    }, true);
}

---

From source file:edu.cmu.lti.oaqa.baseqa.providers.ml.classifiers.ClassifierProvider.java

default List<List<String>> crossTrainPredictMultiLabel(List<Map<String, Double>> X, List<Collection<String>> Y,
        ResampleType resampleType, int limit) throws AnalysisEngineProcessException {
    Set<Integer> indexes = IntStream.range(0, X.size()).boxed().collect(toSet());
    List<Integer> indexList = new ArrayList<>(indexes);
    Collections.shuffle(indexList);
    int nfolds = (int) Math.ceil(indexList.size() / 10.0);
    List<List<String>> ret = IntStream.range(0, X.size()).mapToObj(i -> new ArrayList<String>())
            .collect(toList());//from ww  w  .j  a  v  a2  s  .c o  m
    int fold = 1;
    for (List<Integer> cvTestIndexes : Lists.partition(indexList, nfolds)) {
        LOG.info("Train Predict Fold {}", fold++);
        List<Map<String, Double>> cvTrainX = new ArrayList<>();
        List<Collection<String>> cvTrainY = new ArrayList<>();
        Sets.difference(indexes, new HashSet<>(cvTestIndexes)).forEach(cvTrainIndex -> {
            cvTrainX.add(X.get(cvTrainIndex));
            cvTrainY.add(Y.get(cvTrainIndex));
        });
        trainMultiLabel(cvTrainX, cvTrainY, resampleType, false);
        for (int cvTestIndex : cvTestIndexes) {
            List<String> result = predict(X.get(cvTestIndex), limit).stream().collect(toList());
            ret.set(cvTestIndex, result);
        }
    }
    return ret;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »