List of usage examples for com.google.common.util.concurrent RateLimiter create
public static RateLimiter create(double permitsPerSecond)
From source file:com.netflix.bdp.s3mper.metastore.impl.MetastoreJanitor.java
public void setScanLimit(int scanLimit) { this.scanLimit = scanLimit; scanLimiter = RateLimiter.create(scanLimit); }
From source file:org.apache.bookkeeper.tools.perf.table.PerfClient.java
private void runBenchmarkTasks() throws Exception { StorageClientSettings settings = StorageClientSettings.newBuilder() .serviceUri(serviceURI.getUri().toString()).build(); try (StorageClient client = StorageClientBuilder.newBuilder().withSettings(settings) .withNamespace(flags.namespace).build()) { try (Table<ByteBuf, ByteBuf> table = result(client.openTable(flags.tableName))) { long randSeed = System.currentTimeMillis(); KeyGenerator generator = new KeyGenerator(flags.numKeys, flags.keysPerPrefix, flags.prefixSize); RateLimiter limiter;/*w w w . ja v a 2 s. c o m*/ if (flags.rate <= 0) { limiter = null; } else { limiter = RateLimiter.create(flags.rate); } for (String benchmark : flags.benchmarks) { List<BenchmarkTask> tasks = new ArrayList<>(); int currentTaskId = 0; Semaphore semaphore; if (flags.maxOutstandingRequests <= 0) { semaphore = null; } else { semaphore = new Semaphore(flags.maxOutstandingRequests); } switch (benchmark) { case "fillseq": tasks.add(new WriteSequentialTask(table, currentTaskId++, randSeed, Math.max(flags.numOps, flags.numKeys), flags.numKeys, flags, generator, limiter, semaphore)); break; case "fillrandom": tasks.add(new WriteRandomTask(table, currentTaskId++, randSeed, Math.max(flags.numOps, flags.numKeys), flags.numKeys, flags, generator, limiter, semaphore)); break; case "incseq": tasks.add(new IncrementSequentialTask(table, currentTaskId++, randSeed, Math.max(flags.numOps, flags.numKeys), flags.numKeys, flags, generator, limiter, semaphore)); break; case "incrandom": tasks.add(new IncrementRandomTask(table, currentTaskId++, randSeed, Math.max(flags.numOps, flags.numKeys), flags.numKeys, flags, generator, limiter, semaphore)); break; default: System.err.println("Unknown benchmark: " + benchmark); break; } if (tasks.isEmpty()) { continue; } final CountDownLatch latch = new CountDownLatch(tasks.size()); @Cleanup("shutdown") ExecutorService executor = Executors.newCachedThreadPool(); for (BenchmarkTask task : tasks) { executor.submit(() -> { try { task.runTask(); } catch (Exception e) { log.error("Encountered issue at running benchmark task {}", task.tid, e); } finally { latch.countDown(); } }); } @Cleanup("shutdown") ExecutorService statsExecutor = Executors.newSingleThreadExecutor(); statsExecutor.submit(() -> reportStats(tasks)); latch.await(); log.info("------------------- DONE -----------------------"); tasks.forEach(task -> task.printAggregatedStats()); } } } }
From source file:com.netflix.bdp.s3mper.metastore.impl.MetastoreJanitor.java
public void setDeleteLimit(int deleteLimit) { this.deleteLimit = deleteLimit; deleteLimiter = RateLimiter.create(deleteLimit); }
From source file:org.apache.pulsar.proxy.socket.client.PerformanceClient.java
public void runPerformanceTest(long messages, long limit, int numOfTopic, int sizeOfMessage, String baseUrl, String destination, String authPluginClassName, String authParams) throws InterruptedException, FileNotFoundException { ExecutorService executor = Executors .newCachedThreadPool(new DefaultThreadFactory("pulsar-perf-producer-exec")); HashMap<String, Tuple> producersMap = new HashMap<>(); String produceBaseEndPoint = baseUrl + "ws/producer" + destination; for (int i = 0; i < numOfTopic; i++) { String topic = numOfTopic > 1 ? produceBaseEndPoint + String.valueOf(i) : produceBaseEndPoint; URI produceUri = URI.create(topic); WebSocketClient produceClient = new WebSocketClient(new SslContextFactory(true)); ClientUpgradeRequest produceRequest = new ClientUpgradeRequest(); if (StringUtils.isNotBlank(authPluginClassName) && StringUtils.isNotBlank(authParams)) { try { Authentication auth = AuthenticationFactory.create(authPluginClassName, authParams); auth.start();/*from w w w .j a v a 2s. c om*/ AuthenticationDataProvider authData = auth.getAuthData(); if (authData.hasDataForHttp()) { for (Map.Entry<String, String> kv : authData.getHttpHeaders()) { produceRequest.setHeader(kv.getKey(), kv.getValue()); } } } catch (Exception e) { log.error("Authentication plugin error: " + e.getMessage()); } } SimpleTestProducerSocket produceSocket = new SimpleTestProducerSocket(); try { produceClient.start(); produceClient.connect(produceSocket, produceUri, produceRequest); } catch (IOException e1) { log.error("Fail in connecting: [{}]", e1.getMessage()); return; } catch (Exception e1) { log.error("Fail in starting client[{}]", e1.getMessage()); return; } producersMap.put(produceUri.toString(), new Tuple(produceClient, produceRequest, produceSocket)); } // connection to be established TimeUnit.SECONDS.sleep(5); executor.submit(() -> { try { RateLimiter rateLimiter = RateLimiter.create(limit); // Send messages on all topics/producers long totalSent = 0; while (true) { for (String topic : producersMap.keySet()) { if (messages > 0) { if (totalSent >= messages) { log.trace("------------------- DONE -----------------------"); Thread.sleep(10000); System.exit(0); } } rateLimiter.acquire(); if (producersMap.get(topic).getSocket().getSession() == null) { Thread.sleep(10000); System.exit(0); } producersMap.get(topic).getSocket().sendMsg(String.valueOf(totalSent++), sizeOfMessage); messagesSent.increment(); bytesSent.add(sizeOfMessage); } } } catch (Throwable t) { log.error(t.getMessage()); System.exit(0); } }); // Print report stats long oldTime = System.nanoTime(); Histogram reportHistogram = null; String statsFileName = "perf-websocket-producer-" + System.currentTimeMillis() + ".hgrm"; log.info("Dumping latency stats to {} \n", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); // Some log header bits histogramLogWriter.outputLogFormatVersion(); histogramLogWriter.outputLegend(); while (true) { try { Thread.sleep(5000); } catch (InterruptedException e) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesSent.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = SimpleTestProducerSocket.recorder.getIntervalHistogram(reportHistogram); log.info( "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} ms - 95pct: {} ms - 99pct: {} ms - 99.9pct: {} ms - 99.99pct: {} ms", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0)); histogramLogWriter.outputIntervalHistogram(reportHistogram); reportHistogram.reset(); oldTime = now; } TimeUnit.SECONDS.sleep(100); executor.shutdown(); }
From source file:org.apache.bookkeeper.tools.perf.dlog.PerfWriter.java
void write(List<DistributedLogManager> logs, double writeRate, int maxOutstandingBytesForThisThread, long numRecordsForThisThread, long numBytesForThisThread) throws Exception { log.info(//from ww w. jav a 2 s . com "Write thread started with : logs = {}, rate = {}," + " num records = {}, num bytes = {}, max outstanding bytes = {}", logs.stream().map(l -> l.getStreamName()).collect(Collectors.toList()), writeRate, numRecordsForThisThread, numBytesForThisThread, maxOutstandingBytesForThisThread); List<CompletableFuture<AsyncLogWriter>> writerFutures = logs.stream() .map(manager -> manager.openAsyncLogWriter()).collect(Collectors.toList()); List<AsyncLogWriter> writers = result(FutureUtils.collect(writerFutures)); long txid = writers.stream().mapToLong(writer -> writer.getLastTxId()).max().orElse(0L); txid = Math.max(0L, txid); RateLimiter limiter; if (writeRate > 0) { limiter = RateLimiter.create(writeRate); } else { limiter = null; } final Semaphore semaphore; if (maxOutstandingBytesForThisThread > 0) { semaphore = new Semaphore(maxOutstandingBytesForThisThread); } else { semaphore = null; } // Acquire 1 second worth of records to have a slower ramp-up if (limiter != null) { limiter.acquire((int) writeRate); } long totalWritten = 0L; long totalBytesWritten = 0L; final int numLogs = logs.size(); while (true) { for (int i = 0; i < numLogs; i++) { if (numRecordsForThisThread > 0 && totalWritten >= numRecordsForThisThread) { markPerfDone(); } if (numBytesForThisThread > 0 && totalBytesWritten >= numBytesForThisThread) { markPerfDone(); } if (null != semaphore) { semaphore.acquire(payload.length); } totalWritten++; totalBytesWritten += payload.length; if (null != limiter) { limiter.acquire(payload.length); } final long sendTime = System.nanoTime(); writers.get(i).write(new LogRecord(++txid, Unpooled.wrappedBuffer(payload))).thenAccept(dlsn -> { if (null != semaphore) { semaphore.release(payload.length); } recordsWritten.increment(); bytesWritten.add(payload.length); long latencyMicros = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sendTime); recorder.recordValue(latencyMicros); cumulativeRecorder.recordValue(latencyMicros); }).exceptionally(cause -> { log.warn("Error at writing records", cause); System.exit(-1); return null; }); } } }
From source file:com.example.app.communication.service.NotificationService.java
/** * Send an SMS message delaying until an appropriate time if necessary. * * @param user the user.//from w w w . ja v a 2 s . com * @param phoneNumber the phone number to send too. * @param content the content of the message. * @param delayMillis the delay millis */ public void sendSMS(User user, PhoneNumber phoneNumber, String content, @Nullable Long delayMillis) { // FUTURE : make sure SMS messages are delayed if necessary based on user timezone, etc. final Optional<EmailAddress> emailAddress = ContactUtil.getEmailAddress(user.getPrincipal().getContact(), ContactDataCategory.values()); if (!emailAddress.isPresent()) { _logger.error( "Users are required to have an email address. Not sending SMS. User.id = " + user.getId()); return; } if (!_emailValidationService.checkForValidDomain(emailAddress.get().getEmail(), false)) { _logger.info( "User's email address is being filtered/sanitized. Not sending SMS. User.id = " + user.getId()); return; } if (content.length() < SMS_MESSAGE_LIMIT) _smsService.sendSms(phoneNumber, content); else { RateLimiter rateLimiter = RateLimiter.create(1.5); // Space out split message so it isn't interleaved final List<String> parts = splitContent(content); for (String part : parts) { rateLimiter.acquire(); _smsService.sendSms(phoneNumber, part); } } }
From source file:com.rapid7.diskstorage.dynamodb.Client.java
private void setupStore(com.thinkaurelius.titan.diskstorage.configuration.Configuration config, String prefix, final Map<String, RateLimiter> readRateLimit, final Map<String, RateLimiter> writeRateLimit, String store) {//w ww.j a v a 2 s . com final String dataModel = config.get(Constants.STORES_DATA_MODEL, store); final int scanLimit = config.get(Constants.STORES_SCAN_LIMIT, store); final long readCapacity = config.get(Constants.STORES_CAPACITY_READ, store); final long writeCapacity = config.get(Constants.STORES_CAPACITY_WRITE, store); final double readRate = config.get(Constants.STORES_READ_RATE_LIMIT, store); final double writeRate = config.get(Constants.STORES_WRITE_RATE_LIMIT, store); String actualTableName = prefix + "_" + store; this.dataModel.put(store, BackendDataModel.valueOf(dataModel)); this.capacityRead.put(actualTableName, readCapacity); this.capacityWrite.put(actualTableName, writeCapacity); readRateLimit.put(actualTableName, RateLimiter.create(readRate)); writeRateLimit.put(actualTableName, RateLimiter.create(writeRate)); this.scanLimit.put(actualTableName, scanLimit); }
From source file:com.example.app.service.NotificationService.java
/** * Send an SMS message delaying until an appropriate time if necessary. * * @param user the user./*from w ww . jav a 2 s . c om*/ * @param phoneNumber the phone number to send too. * @param content the content of the message. */ public void sendSMS(User user, PhoneNumber phoneNumber, String content) { // FIXME : make sure SMS messages are delayed if necessary based on user timezone, etc. final Optional<EmailAddress> emailAddress = ContactUtil.getEmailAddress(user.getPrincipal().getContact(), ContactDataCategory.values()); if (!emailAddress.isPresent()) { _logger.error( "Users are required to have an email address. Not sending SMS. User.id = " + user.getId()); return; } if (!_emailValidationService.checkForValidDomain(emailAddress.get().getEmail(), false)) { _logger.info( "User's email address is being filtered/sanitized. Not sending SMS. User.id = " + user.getId()); return; } if (content.length() < SMS_MESSAGE_LIMIT) _smsService.sendSms(phoneNumber, content); else { RateLimiter rateLimiter = RateLimiter.create(1.5); // Space out split message so it isn't interleaved final List<String> parts = splitContent(content); for (String part : parts) { rateLimiter.acquire(); _smsService.sendSms(phoneNumber, part); } } }
From source file:org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.java
ManagedCursorImpl(BookKeeper bookkeeper, ManagedLedgerConfig config, ManagedLedgerImpl ledger,
String cursorName) {/* ww w . jav a 2s . co m*/
this.bookkeeper = bookkeeper;
this.config = config;
this.ledger = ledger;
this.name = cursorName;
STATE_UPDATER.set(this, State.Uninitialized);
PENDING_MARK_DELETED_SUBMITTED_COUNT_UPDATER.set(this, 0);
PENDING_READ_OPS_UPDATER.set(this, 0);
RESET_CURSOR_IN_PROGRESS_UPDATER.set(this, FALSE);
WAITING_READ_OP_UPDATER.set(this, null);
this.lastLedgerSwitchTimestamp = System.currentTimeMillis();
this.protobufFormat = ledger.factory.getConfig().useProtobufBinaryFormatInZK() ? //
ZNodeProtobufFormat.Binary : //
ZNodeProtobufFormat.Text;
if (config.getThrottleMarkDelete() > 0.0) {
markDeleteLimiter = RateLimiter.create(config.getThrottleMarkDelete());
} else {
// Disable mark-delete rate limiter
markDeleteLimiter = null;
}
}
From source file:org.apache.bookkeeper.zookeeper.ZooKeeperClient.java
ZooKeeperClient(String connectString, int sessionTimeoutMs, ZooKeeperWatcherBase watcherManager, RetryPolicy connectRetryPolicy, RetryPolicy operationRetryPolicy, StatsLogger statsLogger, int retryExecThreadCount, double rate) throws IOException { super(connectString, sessionTimeoutMs, watcherManager); this.connectString = connectString; this.sessionTimeoutMs = sessionTimeoutMs; this.watcherManager = watcherManager; this.connectRetryPolicy = connectRetryPolicy; this.operationRetryPolicy = operationRetryPolicy; this.rateLimiter = rate > 0 ? RateLimiter.create(rate) : null; this.retryExecutor = Executors.newScheduledThreadPool(retryExecThreadCount, new ThreadFactoryBuilder().setNameFormat("ZKC-retry-executor-%d").build()); this.connectExecutor = Executors.newSingleThreadExecutor( new ThreadFactoryBuilder().setNameFormat("ZKC-connect-executor-%d").build()); // added itself to the watcher watcherManager.addChildWatcher(this); // Stats/* w ww .j a v a 2s . c o m*/ StatsLogger scopedStatsLogger = statsLogger.scope("zk"); createClientStats = scopedStatsLogger.getOpStatsLogger("create_client"); createStats = scopedStatsLogger.getOpStatsLogger("create"); getStats = scopedStatsLogger.getOpStatsLogger("get_data"); setStats = scopedStatsLogger.getOpStatsLogger("set_data"); deleteStats = scopedStatsLogger.getOpStatsLogger("delete"); getChildrenStats = scopedStatsLogger.getOpStatsLogger("get_children"); existsStats = scopedStatsLogger.getOpStatsLogger("exists"); multiStats = scopedStatsLogger.getOpStatsLogger("multi"); getACLStats = scopedStatsLogger.getOpStatsLogger("get_acl"); setACLStats = scopedStatsLogger.getOpStatsLogger("set_acl"); syncStats = scopedStatsLogger.getOpStatsLogger("sync"); }