Example usage for com.google.common.collect Lists partition

Introduction

In this page you can find the example usage for com.google.common.collect Lists partition.

Prototype

public static <T> List<List<T>> partition(List<T> list, int size) 

Source Link

Document

Returns consecutive List#subList(int,int) sublists of a list, each of the same size (the final list may be smaller).

Usage

From source file:org.apache.pulsar.testclient.ManagedLedgerWriter.java

public static void main(String[] args) throws Exception {

    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-producer");

    try {//from   ww w. j a va2 s  .  co m
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }

    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }

    arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime);

    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar managed-ledger perf writer with config: {}", w.writeValueAsString(arguments));

    byte[] payloadData = new byte[arguments.msgSize];
    ByteBuf payloadBuffer = PooledByteBufAllocator.DEFAULT.directBuffer(arguments.msgSize);
    payloadBuffer.writerIndex(arguments.msgSize);

    // Now processing command line arguments
    String managedLedgerPrefix = "test-" + DigestUtils.sha1Hex(UUID.randomUUID().toString()).substring(0, 5);

    ClientConfiguration bkConf = new ClientConfiguration();
    bkConf.setUseV2WireProtocol(true);
    bkConf.setAddEntryTimeout(30);
    bkConf.setReadEntryTimeout(30);
    bkConf.setThrottleValue(0);
    bkConf.setNumChannelsPerBookie(arguments.maxConnections);
    bkConf.setZkServers(arguments.zookeeperServers);

    ManagedLedgerFactoryConfig mlFactoryConf = new ManagedLedgerFactoryConfig();
    mlFactoryConf.setMaxCacheSize(0);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkConf, mlFactoryConf);

    ManagedLedgerConfig mlConf = new ManagedLedgerConfig();
    mlConf.setEnsembleSize(arguments.ensembleSize);
    mlConf.setWriteQuorumSize(arguments.writeQuorum);
    mlConf.setAckQuorumSize(arguments.ackQuorum);
    mlConf.setMinimumRolloverTime(10, TimeUnit.MINUTES);
    mlConf.setMetadataEnsembleSize(arguments.ensembleSize);
    mlConf.setMetadataWriteQuorumSize(arguments.writeQuorum);
    mlConf.setMetadataAckQuorumSize(arguments.ackQuorum);
    mlConf.setDigestType(arguments.digestType);
    mlConf.setMaxSizePerLedgerMb(2048);

    List<CompletableFuture<ManagedLedger>> futures = new ArrayList<>();

    for (int i = 0; i < arguments.numManagedLedgers; i++) {
        String name = String.format("%s-%03d", managedLedgerPrefix, i);
        CompletableFuture<ManagedLedger> future = new CompletableFuture<>();
        futures.add(future);
        factory.asyncOpen(name, mlConf, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                future.complete(ledger);
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                future.completeExceptionally(exception);
            }
        }, null);
    }

    List<ManagedLedger> managedLedgers = futures.stream().map(CompletableFuture::join)
            .collect(Collectors.toList());

    log.info("Created {} managed ledgers", managedLedgers.size());

    Runtime.getRuntime().addShutdownHook(new Thread() {
        public void run() {
            printAggregatedStats();
        }
    });

    Collections.shuffle(managedLedgers);
    AtomicBoolean isDone = new AtomicBoolean();

    List<List<ManagedLedger>> managedLedgersPerThread = Lists.partition(managedLedgers,
            Math.max(1, managedLedgers.size() / arguments.numThreads));

    for (int i = 0; i < arguments.numThreads; i++) {
        List<ManagedLedger> managedLedgersForThisThread = managedLedgersPerThread.get(i);
        int nunManagedLedgersForThisThread = managedLedgersForThisThread.size();
        long numMessagesForThisThread = arguments.numMessages / arguments.numThreads;
        int maxOutstandingForThisThread = arguments.maxOutstanding;

        executor.submit(() -> {
            try {
                final double msgRate = arguments.msgRate / (double) arguments.numThreads;
                final RateLimiter rateLimiter = RateLimiter.create(msgRate);

                // Acquire 1 sec worth of messages to have a slower ramp-up
                rateLimiter.acquire((int) msgRate);
                final long startTime = System.currentTimeMillis();

                final Semaphore semaphore = new Semaphore(maxOutstandingForThisThread);

                final AddEntryCallback addEntryCallback = new AddEntryCallback() {
                    @Override
                    public void addComplete(Position position, Object ctx) {
                        long sendTime = (Long) (ctx);
                        messagesSent.increment();
                        bytesSent.add(payloadData.length);

                        long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime);
                        recorder.recordValue(latencyMicros);
                        cumulativeRecorder.recordValue(latencyMicros);

                        semaphore.release();
                    }

                    @Override
                    public void addFailed(ManagedLedgerException exception, Object ctx) {
                        log.warn("Write error on message", exception);
                        System.exit(-1);
                    }
                };

                // Send messages on all topics/producers
                long totalSent = 0;
                while (true) {
                    for (int j = 0; j < nunManagedLedgersForThisThread; j++) {
                        if (arguments.testTime > 0) {
                            if (System.currentTimeMillis() - startTime > arguments.testTime) {
                                log.info("------------------- DONE -----------------------");
                                printAggregatedStats();
                                isDone.set(true);
                                Thread.sleep(5000);
                                System.exit(0);
                            }
                        }

                        if (numMessagesForThisThread > 0) {
                            if (totalSent++ >= numMessagesForThisThread) {
                                log.info("------------------- DONE -----------------------");
                                printAggregatedStats();
                                isDone.set(true);
                                Thread.sleep(5000);
                                System.exit(0);
                            }
                        }

                        semaphore.acquire();
                        rateLimiter.acquire();

                        final long sendTime = System.nanoTime();
                        managedLedgersForThisThread.get(j).asyncAddEntry(payloadBuffer, addEntryCallback,
                                sendTime);
                    }
                }
            } catch (Throwable t) {
                log.error("Got error", t);
            }
        });
    }

    // Print report stats
    long oldTime = System.nanoTime();

    Histogram reportHistogram = null;

    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }

        if (isDone.get()) {
            break;
        }

        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;

        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;

        reportHistogram = recorder.getIntervalHistogram(reportHistogram);

        log.info(
                "Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}",
                throughputFormat.format(rate), throughputFormat.format(throughput),
                dec.format(reportHistogram.getMean() / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0),
                dec.format(reportHistogram.getMaxValue() / 1000.0));

        reportHistogram.reset();

        oldTime = now;
    }

    factory.shutdown();
}

---

From source file:com.rackspacecloud.blueflood.types.MetricsCollection.java

public static List<List<Metric>> getMetricsAsBatches(MetricsCollection collection, int partitions) {
    if (partitions <= 0) {
        partitions = 1;/*from  w w w  .  ja v a2  s  . c o m*/
    }

    int sizePerBatch = collection.size() / partitions + 1;

    return Lists.partition(collection.getMetrics(), sizePerBatch);
}

---

From source file:sentiment.SentimentManager.java

public List<List<SentimentScore>> getScoresSubList(int size) {
    List<List<SentimentScore>> smallerLists = Lists.partition(scores, size);
    return smallerLists;
}

---

From source file:org.apache.beam.sdk.io.kinesis.KinesisUploader.java

public static void uploadAll(List<String> data, KinesisTestOptions options) {
    AmazonKinesisClient client = new AmazonKinesisClient(new StaticCredentialsProvider(
            new BasicAWSCredentials(options.getAwsAccessKey(), options.getAwsSecretKey())))
                    .withRegion(Regions.fromName(options.getAwsKinesisRegion()));

    List<List<String>> partitions = Lists.partition(data, MAX_NUMBER_OF_RECORDS_IN_BATCH);

    for (List<String> partition : partitions) {
        List<PutRecordsRequestEntry> allRecords = newArrayList();
        for (String row : partition) {
            allRecords.add(new PutRecordsRequestEntry().withData(ByteBuffer.wrap(row.getBytes(Charsets.UTF_8)))
                    .withPartitionKey(Integer.toString(row.hashCode()))

            );//from ww w  .  j  a  v a2 s  . com
        }

        PutRecordsResult result;
        do {
            result = client.putRecords(new PutRecordsRequest().withStreamName(options.getAwsKinesisStream())
                    .withRecords(allRecords));
            List<PutRecordsRequestEntry> failedRecords = newArrayList();
            int i = 0;
            for (PutRecordsResultEntry row : result.getRecords()) {
                if (row.getErrorCode() != null) {
                    failedRecords.add(allRecords.get(i));
                }
                ++i;
            }
            allRecords = failedRecords;
        }

        while (result.getFailedRecordCount() > 0);
    }
}

---

From source file:simulation.LoadBalancer.java

public void addLoad(List<SimVehicle> vehicles) {
    int groupPartition = 0;
    int vehiclesPerGroup = (int) Math.ceil((double) vehicles.size() / 15);

    if (!vehicles.isEmpty()) {
        for (List<SimVehicle> partition : Lists.partition(vehicles, vehiclesPerGroup)) {
            vehicleLists.get(groupPartition).addAll(partition);
            groupPartition++;/*from www . j a va  2 s  .com*/
        }
    } else {
        System.out.println("Vehiclelist Empty");
    }
}

---

From source file:com.ormanli.duplicatefinder.util.FileUtil.java

public FileUtil(String path) {
    List<File> fileList = getFileList(path);
    queue = Queues.newConcurrentLinkedQueue(Lists.partition(fileList,
            (int) Math.ceil(fileList.size() / (double) Runtime.getRuntime().availableProcessors())));
}

---

From source file:com.dianping.apistatic.Job.MovieShowDetailCrawlerJob.java

@Override
protected void execute() throws Exception {
    List<Integer> movieShowIds = readIntListFromFile(Constants.MOVIESHOWIDS_PATH);
    if (CollectionUtils.isEmpty(movieShowIds)) {
        log("movieShowIds?", new RuntimeException("movieShowIds "));
        return;//from w  w w  .  j a v  a 2s.  co  m
    }

    int partitionSize = (movieShowIds.size() > 5 ? movieShowIds.size() / 5 : movieShowIds.size());
    List<List<Integer>> movieShowIdsList = Lists.partition(movieShowIds, partitionSize);
    CountDownLatch countDownLatch = new CountDownLatch(movieShowIdsList.size());

    for (List<Integer> list : movieShowIdsList) {
        MovieShowCrawlerThread thread = new MovieShowCrawlerThread(list, countDownLatch);
        thread.start();
    }
    try {
        countDownLatch.await();
    } catch (InterruptedException e) {
        log("CountDownLatch InterruptedException", e);
    }
}

---

From source file:bio.gcat.operation.split.Partition.java

@SuppressWarnings("unchecked")
@Override//from   w  w  w .  ja v  a  2  s.  c  o m
public List<Collection<Tuple>> split(Collection<Tuple> tuples, Object... values) {
    return (List<Collection<Tuple>>) (List<?>) Lists.partition(
            (tuples instanceof List) ? (List<Tuple>) tuples : new ArrayList<>(tuples),
            (int) Math.ceil((double) tuples.size() / (Integer) values[0]));
}

---

From source file:com.dianping.apistatic.Job.SeatingplanCrawlerJob.java

@Override
protected void execute() throws Exception {
    List<Integer> movieShowIds = readIntListFromFile(Constants.MOVIESHOWIDS_PATH);
    if (CollectionUtils.isEmpty(movieShowIds)) {
        log("movieShowIds?", new RuntimeException("movieShowIds "));
        return;/*  ww  w .  ja va 2s .  com*/
    }

    int partitionSize = (movieShowIds.size() > 5 ? movieShowIds.size() / 5 : movieShowIds.size());
    List<List<Integer>> movieShowIdsList = Lists.partition(movieShowIds, partitionSize);
    CountDownLatch countDownLatch = new CountDownLatch(movieShowIdsList.size());
    for (List<Integer> list : movieShowIdsList) {
        SeatingPlanCrawlerThread thread = new SeatingPlanCrawlerThread(list, countDownLatch);
        thread.start();
    }
    try {
        countDownLatch.await();
    } catch (InterruptedException e) {
        log("CountDownLatch InterruptedException", e);
    }
}

---

From source file:com.netflix.spinnaker.clouddriver.aws.event.AfterResizeEventHandler.java

default void terminateInstancesInAutoScalingGroup(Task task, AmazonEC2 amazonEC2,
        AutoScalingGroup autoScalingGroup) {
    String serverGroupName = autoScalingGroup.getAutoScalingGroupName();

    List<String> instanceIds = autoScalingGroup.getInstances().stream().map(Instance::getInstanceId)
            .collect(Collectors.toList());

    int terminatedCount = 0;
    for (List<String> partition : Lists.partition(instanceIds, MAX_SIMULTANEOUS_TERMINATIONS)) {
        try {// w ww  . j ava 2  s. c  o m
            terminatedCount += partition.size();
            task.updateStatus(PHASE, String.format("Terminating %d of %d instances in %s", terminatedCount,
                    instanceIds.size(), serverGroupName));
            amazonEC2.terminateInstances(new TerminateInstancesRequest().withInstanceIds(partition));
        } catch (Exception e) {
            task.updateStatus(PHASE,
                    String.format("Unable to terminate instances, reason: '%s'", e.getMessage()));
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »