List of usage examples for java.util.concurrent LinkedBlockingQueue poll
public E poll()
From source file:com.ibm.crail.tools.CrailBenchmark.java
void createFileAsync(String filename, int loop, int batch) throws Exception, InterruptedException { System.out.println("createFileAsync, filename " + filename + ", loop " + loop + ", batch " + batch); //warmup/*from w w w.jav a 2s.co m*/ ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>(); CrailBuffer buf = fs.allocateBuffer(); bufferQueue.add(buf); warmUp(filename, warmup, bufferQueue); fs.freeBuffer(buf); //benchmark System.out.println("starting benchmark..."); fs.getStatistics().reset(); LinkedBlockingQueue<Future<CrailNode>> futureQueue = new LinkedBlockingQueue<Future<CrailNode>>(); LinkedBlockingQueue<CrailFile> fileQueue = new LinkedBlockingQueue<CrailFile>(); LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>(); fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get() .syncDir(); for (int i = 0; i < loop; i++) { String name = "/" + i; String f = filename + name; pathQueue.add(f); } long start = System.currentTimeMillis(); for (int i = 0; i < loop; i += batch) { //single operation == loop for (int j = 0; j < batch; j++) { String path = pathQueue.poll(); Future<CrailNode> future = fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT); futureQueue.add(future); } for (int j = 0; j < batch; j++) { Future<CrailNode> future = futureQueue.poll(); CrailFile file = future.get().asFile(); fileQueue.add(file); } for (int j = 0; j < batch; j++) { CrailFile file = fileQueue.poll(); file.syncDir(); } } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)); double latency = executionTime * 1000.0 / ((double) loop); System.out.println("execution time [ms] " + executionTime); System.out.println("latency [us] " + latency); fs.delete(filename, true).get().syncDir(); fs.getStatistics().print("close"); }
From source file:com.ibm.crail.tools.CrailBenchmark.java
void readSequentialAsync(String filename, int size, int loop, int batch) throws Exception { System.out.println("readSequentialAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch " + batch); ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>(); for (int i = 0; i < batch; i++) { CrailBuffer buf = null;/*from w ww . java2 s . c o m*/ if (size == CrailConstants.BUFFER_SIZE) { buf = fs.allocateBuffer(); } else if (size < CrailConstants.BUFFER_SIZE) { CrailBuffer _buf = fs.allocateBuffer(); _buf.clear().limit(size); buf = _buf.slice(); } else { buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size)); } bufferQueue.add(buf); } //warmup warmUp(filename, warmup, bufferQueue); //benchmark System.out.println("starting benchmark..."); double sumbytes = 0; double ops = 0; fs.getStatistics().reset(); CrailFile file = fs.lookup(filename).get().asFile(); CrailInputStream directStream = file.getDirectInputStream(file.getCapacity()); HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>(); LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>(); long start = System.currentTimeMillis(); for (int i = 0; i < batch - 1 && ops < loop; i++) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.read(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); ops = ops + 1.0; } while (ops < loop) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.read(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); future = futureQueue.poll(); CrailResult result = future.get(); buf = futureMap.get(future.hashCode()); bufferQueue.add(buf); sumbytes = sumbytes + result.getLen(); ops = ops + 1.0; } while (!futureQueue.isEmpty()) { Future<CrailResult> future = futureQueue.poll(); CrailResult result = future.get(); futureMap.get(future.hashCode()); sumbytes = sumbytes + result.getLen(); ops = ops + 1.0; } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; double throughput = 0.0; double latency = 0.0; double sumbits = sumbytes * 8.0; if (executionTime > 0) { throughput = sumbits / executionTime / 1000.0 / 1000.0; latency = 1000000.0 * executionTime / ops; } directStream.close(); System.out.println("execution time " + executionTime); System.out.println("ops " + ops); System.out.println("sumbytes " + sumbytes); System.out.println("throughput " + throughput); System.out.println("latency " + latency); fs.getStatistics().print("close"); }
From source file:org.apache.bookkeeper.metadata.etcd.EtcdLedgerManagerTest.java
@Test public void testRegisterLedgerMetadataListener() throws Exception { long ledgerId = System.currentTimeMillis(); // create a ledger metadata LedgerMetadata metadata = LedgerMetadataBuilder.create().withEnsembleSize(3).withWriteQuorumSize(3) .withAckQuorumSize(2).withPassword("test-password".getBytes(UTF_8)) .withDigestType(DigestType.CRC32C.toApiDigestType()).newEnsembleEntry(0L, createNumBookies(3)) .build();//from w ww . j a v a 2s. co m result(lm.createLedgerMetadata(ledgerId, metadata)); Versioned<LedgerMetadata> readMetadata = lm.readLedgerMetadata(ledgerId).get(); log.info("Create ledger metadata : {}", readMetadata.getValue()); // register first listener LinkedBlockingQueue<Versioned<LedgerMetadata>> metadataQueue1 = new LinkedBlockingQueue<>(); LedgerMetadataListener listener1 = (lid, m) -> { log.info("[listener1] Received ledger {} metadata : {}", lid, m); metadataQueue1.add(m); }; log.info("Registered first listener for ledger {}", ledgerId); lm.registerLedgerMetadataListener(ledgerId, listener1); // we should receive a metadata notification when a ledger is created Versioned<LedgerMetadata> notifiedMetadata = metadataQueue1.take(); assertEquals(readMetadata, notifiedMetadata); ValueStream<LedgerMetadata> lms = lm.getLedgerMetadataStream(ledgerId); assertNotNull(lms.waitUntilWatched()); assertNotNull(result(lms.waitUntilWatched())); // register second listener LinkedBlockingQueue<Versioned<LedgerMetadata>> metadataQueue2 = new LinkedBlockingQueue<>(); LedgerMetadataListener listener2 = (lid, m) -> { log.info("[listener2] Received ledger {} metadata : {}", lid, m); metadataQueue2.add(m); }; log.info("Registered second listener for ledger {}", ledgerId); lm.registerLedgerMetadataListener(ledgerId, listener2); Versioned<LedgerMetadata> notifiedMetadata2 = metadataQueue2.take(); assertEquals(readMetadata, notifiedMetadata2); assertNotNull(lm.getLedgerMetadataStream(ledgerId)); // update the metadata lm.writeLedgerMetadata(ledgerId, LedgerMetadataBuilder.from(metadata).newEnsembleEntry(10L, createNumBookies(3)).build(), notifiedMetadata.getVersion()).get(); readMetadata = lm.readLedgerMetadata(ledgerId).get(); assertEquals(readMetadata, metadataQueue1.take()); assertEquals(readMetadata, metadataQueue2.take()); lms = lm.getLedgerMetadataStream(ledgerId); assertNotNull(lms); assertEquals(2, lms.getNumConsumers()); // remove listener2 lm.unregisterLedgerMetadataListener(ledgerId, listener2); lms = lm.getLedgerMetadataStream(ledgerId); assertNotNull(lms); assertEquals(1, lms.getNumConsumers()); // update the metadata again lm.writeLedgerMetadata(ledgerId, LedgerMetadataBuilder.from(metadata).newEnsembleEntry(20L, createNumBookies(3)).build(), readMetadata.getVersion()).get(); readMetadata = lm.readLedgerMetadata(ledgerId).get(); assertEquals(readMetadata, metadataQueue1.take()); assertNull(metadataQueue2.poll()); // remove listener1 lm.unregisterLedgerMetadataListener(ledgerId, listener1); // the value stream will be removed while (lm.getLedgerMetadataStream(ledgerId) != null) { TimeUnit.MILLISECONDS.sleep(100); } assertEquals(0, lms.getNumConsumers()); // update the metadata again lm.writeLedgerMetadata(ledgerId, LedgerMetadataBuilder.from(metadata).newEnsembleEntry(30L, createNumBookies(3)).build(), readMetadata.getVersion()).get(); readMetadata = lm.readLedgerMetadata(ledgerId).get(); assertNull(metadataQueue1.poll()); assertNull(metadataQueue2.poll()); log.info("Registered first listener for ledger {} again", ledgerId); lm.registerLedgerMetadataListener(ledgerId, listener1); notifiedMetadata = metadataQueue1.take(); assertEquals(readMetadata, notifiedMetadata); lms = lm.getLedgerMetadataStream(ledgerId); assertNotNull(lms); assertEquals(1, lms.getNumConsumers()); // delete the ledger lm.removeLedgerMetadata(ledgerId, readMetadata.getVersion()).get(); // the listener will eventually be removed while (lm.getLedgerMetadataStream(ledgerId) != null) { TimeUnit.MILLISECONDS.sleep(100); } assertEquals(1, lms.getNumConsumers()); assertNull(metadataQueue1.poll()); assertNull(metadataQueue2.poll()); }
From source file:com.ibm.crail.tools.CrailBenchmark.java
void collectionTest(int size, int loop) throws Exception { System.out.println("collectionTest, size " + size + ", loop " + loop); RingBuffer<Object> ringBuffer = new RingBuffer<Object>(10); ArrayBlockingQueue<Object> arrayQueue = new ArrayBlockingQueue<Object>(10); LinkedBlockingQueue<Object> listQueue = new LinkedBlockingQueue<Object>(); Object obj = new Object(); long start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { for (int j = 0; j < size; j++) { ringBuffer.add(obj);/*from w ww. jav a 2 s .co m*/ Object tmp = ringBuffer.peek(); tmp = ringBuffer.poll(); } } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)); System.out.println("ringbuffer, execution time [ms] " + executionTime); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { for (int j = 0; j < size; j++) { arrayQueue.add(obj); Object tmp = arrayQueue.peek(); tmp = arrayQueue.poll(); } } end = System.currentTimeMillis(); executionTime = ((double) (end - start)); System.out.println("arrayQueue, execution time [ms] " + executionTime); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { for (int j = 0; j < size; j++) { listQueue.add(obj); Object tmp = listQueue.peek(); tmp = listQueue.poll(); } } end = System.currentTimeMillis(); executionTime = ((double) (end - start)); System.out.println("arrayQueue, execution time [ms] " + executionTime); }
From source file:com.ibm.crail.tools.CrailBenchmark.java
void writeAsync(String filename, int size, int loop, int batch, int storageClass, int locationClass) throws Exception { System.out.println("writeAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch " + batch + ", storageClass " + storageClass + ", locationClass " + locationClass); ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>(); for (int i = 0; i < batch; i++) { CrailBuffer buf = null;/*from ww w .j a v a 2 s . c om*/ if (size == CrailConstants.BUFFER_SIZE) { buf = fs.allocateBuffer(); } else if (size < CrailConstants.BUFFER_SIZE) { CrailBuffer _buf = fs.allocateBuffer(); _buf.clear().limit(size); buf = _buf.slice(); } else { buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size)); } bufferQueue.add(buf); } //warmup warmUp(filename, warmup, bufferQueue); //benchmark System.out.println("starting benchmark..."); LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>(); HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>(); fs.getStatistics().reset(); long _loop = (long) loop; long _bufsize = (long) CrailConstants.BUFFER_SIZE; long _capacity = _loop * _bufsize; double sumbytes = 0; double ops = 0; CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass)).get().asFile(); CrailOutputStream directStream = file.getDirectOutputStream(_capacity); long start = System.currentTimeMillis(); for (int i = 0; i < batch - 1 && ops < loop; i++) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.write(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); ops = ops + 1.0; } while (ops < loop) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.write(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); future = futureQueue.poll(); future.get(); buf = futureMap.get(future.hashCode()); bufferQueue.add(buf); sumbytes = sumbytes + buf.capacity(); ops = ops + 1.0; } while (!futureQueue.isEmpty()) { Future<CrailResult> future = futureQueue.poll(); future.get(); CrailBuffer buf = futureMap.get(future.hashCode()); sumbytes = sumbytes + buf.capacity(); ops = ops + 1.0; } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; double throughput = 0.0; double latency = 0.0; double sumbits = sumbytes * 8.0; if (executionTime > 0) { throughput = sumbits / executionTime / 1000.0 / 1000.0; latency = 1000000.0 * executionTime / ops; } directStream.close(); System.out.println("execution time " + executionTime); System.out.println("ops " + ops); System.out.println("sumbytes " + sumbytes); System.out.println("throughput " + throughput); System.out.println("latency " + latency); fs.getStatistics().print("close"); }