List of usage examples for com.google.common.util.concurrent RateLimiter acquire
public double acquire()
From source file:org.jclouds.smartos.SmartOSHostController.java
private Iterable<VM> toVMList(String string) { try {//from w w w .ja v a2 s. com BufferedReader r = new BufferedReader(new StringReader(string)); String line; ImmutableList.Builder<VM> resultBuilder = ImmutableList.builder(); while ((line = r.readLine()) != null) { VM vm = VM.builder().fromVmadmString(line).build(); Map<String, String> ipAddresses; RateLimiter limiter = RateLimiter.create(1.0); for (int i = 0; i < 30; i++) { ipAddresses = getVMIpAddresses(vm.getUuid()); if (!ipAddresses.isEmpty()) { // Got some String ip = ipAddresses.get("net0"); if (ip != null && !ip.equals("0.0.0.0")) { vm = vm.toBuilder().publicAddress(ip).build(); break; } } limiter.acquire(); } resultBuilder.add(vm); } return resultBuilder.build(); } catch (IOException e) { throw new HostException("Error parsing response when building VM list", e); } }
From source file:com.example.app.url.shortener.service.URLShortenerService.java
String tryURLShortenOperation(Function<Urlshortener, String> operation, double rate) throws IOException { final Stopwatch stopwatch = Stopwatch.createStarted(); final Urlshortener shortener = createURLShortener(); final RateLimiter rateLimiter = RateLimiter.create(rate); IOException lastException = null; int retries = 0; while (retries++ < 5) { try {/*from w ww . j a v a2s .c o m*/ return operation.apply(shortener); } catch (RuntimeException e) { if (e.getCause() instanceof IOException) lastException = (IOException) e.getCause(); if (e.getCause() instanceof InterruptedIOException) { if (_logger.isDebugEnabled()) { _logger.debug("Could not shorten URL due to connection error. Retry#" + retries + ". Time spent: " + stopwatch, e); } rateLimiter.acquire(); } else throw e; } } if (lastException != null) throw lastException; else throw new IOException("Unable to shorten URL. Retried " + retries + " times for " + stopwatch); }
From source file:com.yahoo.pulsar.proxy.socket.client.PerformanceClient.java
public void runPerformanceTest(long messages, long limit, int numOfTopic, int sizeOfMessage, String baseUrl, String destination) throws InterruptedException, FileNotFoundException { ExecutorService executor = Executors .newCachedThreadPool(new DefaultThreadFactory("pulsar-perf-producer-exec")); HashMap<String, Tuple> producersMap = new HashMap<>(); String produceBaseEndPoint = baseUrl + destination; for (int i = 0; i < numOfTopic; i++) { String topic = produceBaseEndPoint + "1" + "/"; URI produceUri = URI.create(topic); WebSocketClient produceClient = new WebSocketClient(new SslContextFactory(true)); ClientUpgradeRequest produceRequest = new ClientUpgradeRequest(); SimpleTestProducerSocket produceSocket = new SimpleTestProducerSocket(); try {//from w w w . ja v a 2 s . com produceClient.start(); produceClient.connect(produceSocket, produceUri, produceRequest); } catch (IOException e1) { log.error("Fail in connecting: [{}]", e1.getMessage()); return; } catch (Exception e1) { log.error("Fail in starting client[{}]", e1.getMessage()); return; } producersMap.put(produceUri.toString(), new Tuple(produceClient, produceRequest, produceSocket)); } // connection to be established TimeUnit.SECONDS.sleep(5); executor.submit(() -> { try { RateLimiter rateLimiter = RateLimiter.create(limit); // Send messages on all topics/producers long totalSent = 0; while (true) { for (String topic : producersMap.keySet()) { if (messages > 0) { if (totalSent++ >= messages) { log.trace("------------------- DONE -----------------------"); Thread.sleep(10000); System.exit(0); } } rateLimiter.acquire(); if (producersMap.get(topic).getSocket().getSession() == null) { Thread.sleep(10000); System.exit(0); } producersMap.get(topic).getSocket().sendMsg((String) String.valueOf(totalSent), sizeOfMessage); messagesSent.increment(); bytesSent.add(1000); } } } catch (Throwable t) { log.error(t.getMessage()); System.exit(0); } }); // Print report stats long oldTime = System.nanoTime(); Histogram reportHistogram = null; String statsFileName = "perf-websocket-producer-" + System.currentTimeMillis() + ".hgrm"; log.info("Dumping latency stats to %s \n", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); // Some log header bits histogramLogWriter.outputLogFormatVersion(); histogramLogWriter.outputLegend(); while (true) { try { Thread.sleep(5000); } catch (InterruptedException e) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesSent.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = SimpleTestProducerSocket.recorder.getIntervalHistogram(reportHistogram); log.info( "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} ms - 95pct: {} ms - 99pct: {} ms - 99.9pct: {} ms - 99.99pct: {} ms", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0)); histogramLogWriter.outputIntervalHistogram(reportHistogram); reportHistogram.reset(); oldTime = now; } TimeUnit.SECONDS.sleep(100); executor.shutdown(); }
From source file:org.apache.aurora.scheduler.async.TaskGroups.java
@VisibleForTesting TaskGroups(final ScheduledExecutorService executor, final Amount<Long, Time> firstScheduleDelay, final BackoffStrategy backoff, final RateLimiter rateLimiter, final TaskScheduler taskScheduler, final RescheduleCalculator rescheduleCalculator) { requireNonNull(firstScheduleDelay);// w ww . ja v a2 s .c o m Preconditions.checkArgument(firstScheduleDelay.getValue() > 0); this.executor = requireNonNull(executor); requireNonNull(rateLimiter); requireNonNull(taskScheduler); this.firstScheduleDelay = firstScheduleDelay.as(Time.MILLISECONDS); this.backoff = requireNonNull(backoff); this.rescheduleCalculator = requireNonNull(rescheduleCalculator); this.taskScheduler = new TaskScheduler() { @Override public boolean schedule(String taskId) { rateLimiter.acquire(); return taskScheduler.schedule(taskId); } }; }
From source file:org.apache.pulsar.proxy.socket.client.PerformanceClient.java
public void runPerformanceTest(long messages, long limit, int numOfTopic, int sizeOfMessage, String baseUrl, String destination, String authPluginClassName, String authParams) throws InterruptedException, FileNotFoundException { ExecutorService executor = Executors .newCachedThreadPool(new DefaultThreadFactory("pulsar-perf-producer-exec")); HashMap<String, Tuple> producersMap = new HashMap<>(); String produceBaseEndPoint = baseUrl + "ws/producer" + destination; for (int i = 0; i < numOfTopic; i++) { String topic = numOfTopic > 1 ? produceBaseEndPoint + String.valueOf(i) : produceBaseEndPoint; URI produceUri = URI.create(topic); WebSocketClient produceClient = new WebSocketClient(new SslContextFactory(true)); ClientUpgradeRequest produceRequest = new ClientUpgradeRequest(); if (StringUtils.isNotBlank(authPluginClassName) && StringUtils.isNotBlank(authParams)) { try { Authentication auth = AuthenticationFactory.create(authPluginClassName, authParams); auth.start();//from w w w . j a v a 2s. c o m AuthenticationDataProvider authData = auth.getAuthData(); if (authData.hasDataForHttp()) { for (Map.Entry<String, String> kv : authData.getHttpHeaders()) { produceRequest.setHeader(kv.getKey(), kv.getValue()); } } } catch (Exception e) { log.error("Authentication plugin error: " + e.getMessage()); } } SimpleTestProducerSocket produceSocket = new SimpleTestProducerSocket(); try { produceClient.start(); produceClient.connect(produceSocket, produceUri, produceRequest); } catch (IOException e1) { log.error("Fail in connecting: [{}]", e1.getMessage()); return; } catch (Exception e1) { log.error("Fail in starting client[{}]", e1.getMessage()); return; } producersMap.put(produceUri.toString(), new Tuple(produceClient, produceRequest, produceSocket)); } // connection to be established TimeUnit.SECONDS.sleep(5); executor.submit(() -> { try { RateLimiter rateLimiter = RateLimiter.create(limit); // Send messages on all topics/producers long totalSent = 0; while (true) { for (String topic : producersMap.keySet()) { if (messages > 0) { if (totalSent >= messages) { log.trace("------------------- DONE -----------------------"); Thread.sleep(10000); System.exit(0); } } rateLimiter.acquire(); if (producersMap.get(topic).getSocket().getSession() == null) { Thread.sleep(10000); System.exit(0); } producersMap.get(topic).getSocket().sendMsg(String.valueOf(totalSent++), sizeOfMessage); messagesSent.increment(); bytesSent.add(sizeOfMessage); } } } catch (Throwable t) { log.error(t.getMessage()); System.exit(0); } }); // Print report stats long oldTime = System.nanoTime(); Histogram reportHistogram = null; String statsFileName = "perf-websocket-producer-" + System.currentTimeMillis() + ".hgrm"; log.info("Dumping latency stats to {} \n", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); // Some log header bits histogramLogWriter.outputLogFormatVersion(); histogramLogWriter.outputLegend(); while (true) { try { Thread.sleep(5000); } catch (InterruptedException e) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesSent.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = SimpleTestProducerSocket.recorder.getIntervalHistogram(reportHistogram); log.info( "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} ms - 95pct: {} ms - 99pct: {} ms - 99.9pct: {} ms - 99.99pct: {} ms", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0)); histogramLogWriter.outputIntervalHistogram(reportHistogram); reportHistogram.reset(); oldTime = now; } TimeUnit.SECONDS.sleep(100); executor.shutdown(); }
From source file:com.twitter.aurora.scheduler.async.TaskGroups.java
TaskGroups(final ScheduledExecutorService executor, final Storage storage, final BackoffStrategy taskGroupBackoffStrategy, final RateLimiter rateLimiter, final SchedulingAction schedulingAction, final Clock clock, final RescheduleCalculator rescheduleCalculator, final Preemptor preemptor) { this.storage = checkNotNull(storage); checkNotNull(executor);/*from www . ja v a 2s. c om*/ checkNotNull(taskGroupBackoffStrategy); checkNotNull(rateLimiter); checkNotNull(schedulingAction); this.clock = checkNotNull(clock); this.rescheduleCalculator = checkNotNull(rescheduleCalculator); this.preemptor = checkNotNull(preemptor); final SchedulingAction rateLimitedAction = new SchedulingAction() { @Override public boolean schedule(String taskId) { rateLimiter.acquire(); return schedulingAction.schedule(taskId); } }; groups = CacheBuilder.newBuilder().build(new CacheLoader<GroupKey, TaskGroup>() { @Override public TaskGroup load(GroupKey key) { TaskGroup group = new TaskGroup(key, taskGroupBackoffStrategy); LOG.info("Evaluating group " + key + " in " + group.getPenaltyMs() + " ms"); startGroup(group, executor, rateLimitedAction); return group; } }); }