Example usage for java.util.concurrent LinkedBlockingQueue poll

List of usage examples for java.util.concurrent LinkedBlockingQueue poll

Introduction

In this page you can find the example usage for java.util.concurrent LinkedBlockingQueue poll.

Prototype

public E poll() 

Source Link

Usage

From source file:com.ibm.crail.tools.CrailBenchmark.java

void createFileAsync(String filename, int loop, int batch) throws Exception, InterruptedException {
    System.out.println("createFileAsync, filename " + filename + ", loop " + loop + ", batch " + batch);

    //warmup/*from  w w w.jav a  2s.co  m*/
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<Future<CrailNode>> futureQueue = new LinkedBlockingQueue<Future<CrailNode>>();
    LinkedBlockingQueue<CrailFile> fileQueue = new LinkedBlockingQueue<CrailFile>();
    LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>();
    fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get()
            .syncDir();

    for (int i = 0; i < loop; i++) {
        String name = "/" + i;
        String f = filename + name;
        pathQueue.add(f);
    }

    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i += batch) {
        //single operation == loop
        for (int j = 0; j < batch; j++) {
            String path = pathQueue.poll();
            Future<CrailNode> future = fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT,
                    CrailLocationClass.DEFAULT);
            futureQueue.add(future);
        }
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = futureQueue.poll();
            CrailFile file = future.get().asFile();
            fileQueue.add(file);
        }
        for (int j = 0; j < batch; j++) {
            CrailFile file = fileQueue.poll();
            file.syncDir();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) loop);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);

    fs.delete(filename, true).get().syncDir();

    fs.getStatistics().print("close");

}

From source file:com.ibm.crail.tools.CrailBenchmark.java

void readSequentialAsync(String filename, int size, int loop, int batch) throws Exception {
    System.out.println("readSequentialAsync, filename " + filename + ", size " + size + ", loop " + loop
            + ", batch " + batch);

    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    for (int i = 0; i < batch; i++) {
        CrailBuffer buf = null;/*from  w ww .  java2  s  . c  o  m*/
        if (size == CrailConstants.BUFFER_SIZE) {
            buf = fs.allocateBuffer();
        } else if (size < CrailConstants.BUFFER_SIZE) {
            CrailBuffer _buf = fs.allocateBuffer();
            _buf.clear().limit(size);
            buf = _buf.slice();
        } else {
            buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
        }
        bufferQueue.add(buf);
    }

    //warmup
    warmUp(filename, warmup, bufferQueue);

    //benchmark
    System.out.println("starting benchmark...");
    double sumbytes = 0;
    double ops = 0;
    fs.getStatistics().reset();
    CrailFile file = fs.lookup(filename).get().asFile();
    CrailInputStream directStream = file.getDirectInputStream(file.getCapacity());
    HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>();
    LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>();
    long start = System.currentTimeMillis();
    for (int i = 0; i < batch - 1 && ops < loop; i++) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.read(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);
        ops = ops + 1.0;
    }
    while (ops < loop) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.read(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);

        future = futureQueue.poll();
        CrailResult result = future.get();
        buf = futureMap.get(future.hashCode());
        bufferQueue.add(buf);

        sumbytes = sumbytes + result.getLen();
        ops = ops + 1.0;
    }
    while (!futureQueue.isEmpty()) {
        Future<CrailResult> future = futureQueue.poll();
        CrailResult result = future.get();
        futureMap.get(future.hashCode());
        sumbytes = sumbytes + result.getLen();
        ops = ops + 1.0;
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    directStream.close();

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}

From source file:org.apache.bookkeeper.metadata.etcd.EtcdLedgerManagerTest.java

@Test
public void testRegisterLedgerMetadataListener() throws Exception {
    long ledgerId = System.currentTimeMillis();

    // create a ledger metadata
    LedgerMetadata metadata = LedgerMetadataBuilder.create().withEnsembleSize(3).withWriteQuorumSize(3)
            .withAckQuorumSize(2).withPassword("test-password".getBytes(UTF_8))
            .withDigestType(DigestType.CRC32C.toApiDigestType()).newEnsembleEntry(0L, createNumBookies(3))
            .build();//from  w  ww  .  j  a v a 2s. co m
    result(lm.createLedgerMetadata(ledgerId, metadata));
    Versioned<LedgerMetadata> readMetadata = lm.readLedgerMetadata(ledgerId).get();
    log.info("Create ledger metadata : {}", readMetadata.getValue());

    // register first listener

    LinkedBlockingQueue<Versioned<LedgerMetadata>> metadataQueue1 = new LinkedBlockingQueue<>();
    LedgerMetadataListener listener1 = (lid, m) -> {
        log.info("[listener1] Received ledger {} metadata : {}", lid, m);
        metadataQueue1.add(m);
    };
    log.info("Registered first listener for ledger {}", ledgerId);
    lm.registerLedgerMetadataListener(ledgerId, listener1);
    // we should receive a metadata notification when a ledger is created
    Versioned<LedgerMetadata> notifiedMetadata = metadataQueue1.take();
    assertEquals(readMetadata, notifiedMetadata);
    ValueStream<LedgerMetadata> lms = lm.getLedgerMetadataStream(ledgerId);
    assertNotNull(lms.waitUntilWatched());
    assertNotNull(result(lms.waitUntilWatched()));

    // register second listener

    LinkedBlockingQueue<Versioned<LedgerMetadata>> metadataQueue2 = new LinkedBlockingQueue<>();
    LedgerMetadataListener listener2 = (lid, m) -> {
        log.info("[listener2] Received ledger {} metadata : {}", lid, m);
        metadataQueue2.add(m);
    };
    log.info("Registered second listener for ledger {}", ledgerId);
    lm.registerLedgerMetadataListener(ledgerId, listener2);
    Versioned<LedgerMetadata> notifiedMetadata2 = metadataQueue2.take();
    assertEquals(readMetadata, notifiedMetadata2);
    assertNotNull(lm.getLedgerMetadataStream(ledgerId));

    // update the metadata
    lm.writeLedgerMetadata(ledgerId,
            LedgerMetadataBuilder.from(metadata).newEnsembleEntry(10L, createNumBookies(3)).build(),
            notifiedMetadata.getVersion()).get();
    readMetadata = lm.readLedgerMetadata(ledgerId).get();
    assertEquals(readMetadata, metadataQueue1.take());
    assertEquals(readMetadata, metadataQueue2.take());
    lms = lm.getLedgerMetadataStream(ledgerId);
    assertNotNull(lms);
    assertEquals(2, lms.getNumConsumers());

    // remove listener2
    lm.unregisterLedgerMetadataListener(ledgerId, listener2);
    lms = lm.getLedgerMetadataStream(ledgerId);
    assertNotNull(lms);
    assertEquals(1, lms.getNumConsumers());

    // update the metadata again
    lm.writeLedgerMetadata(ledgerId,
            LedgerMetadataBuilder.from(metadata).newEnsembleEntry(20L, createNumBookies(3)).build(),
            readMetadata.getVersion()).get();
    readMetadata = lm.readLedgerMetadata(ledgerId).get();
    assertEquals(readMetadata, metadataQueue1.take());
    assertNull(metadataQueue2.poll());

    // remove listener1
    lm.unregisterLedgerMetadataListener(ledgerId, listener1);
    // the value stream will be removed
    while (lm.getLedgerMetadataStream(ledgerId) != null) {
        TimeUnit.MILLISECONDS.sleep(100);
    }
    assertEquals(0, lms.getNumConsumers());

    // update the metadata again
    lm.writeLedgerMetadata(ledgerId,
            LedgerMetadataBuilder.from(metadata).newEnsembleEntry(30L, createNumBookies(3)).build(),
            readMetadata.getVersion()).get();
    readMetadata = lm.readLedgerMetadata(ledgerId).get();
    assertNull(metadataQueue1.poll());
    assertNull(metadataQueue2.poll());

    log.info("Registered first listener for ledger {} again", ledgerId);
    lm.registerLedgerMetadataListener(ledgerId, listener1);
    notifiedMetadata = metadataQueue1.take();
    assertEquals(readMetadata, notifiedMetadata);
    lms = lm.getLedgerMetadataStream(ledgerId);
    assertNotNull(lms);
    assertEquals(1, lms.getNumConsumers());

    // delete the ledger
    lm.removeLedgerMetadata(ledgerId, readMetadata.getVersion()).get();
    // the listener will eventually be removed
    while (lm.getLedgerMetadataStream(ledgerId) != null) {
        TimeUnit.MILLISECONDS.sleep(100);
    }
    assertEquals(1, lms.getNumConsumers());
    assertNull(metadataQueue1.poll());
    assertNull(metadataQueue2.poll());
}

From source file:com.ibm.crail.tools.CrailBenchmark.java

void collectionTest(int size, int loop) throws Exception {
    System.out.println("collectionTest, size " + size + ", loop " + loop);

    RingBuffer<Object> ringBuffer = new RingBuffer<Object>(10);
    ArrayBlockingQueue<Object> arrayQueue = new ArrayBlockingQueue<Object>(10);
    LinkedBlockingQueue<Object> listQueue = new LinkedBlockingQueue<Object>();

    Object obj = new Object();
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        for (int j = 0; j < size; j++) {
            ringBuffer.add(obj);/*from  w ww. jav a 2  s .co  m*/
            Object tmp = ringBuffer.peek();
            tmp = ringBuffer.poll();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    System.out.println("ringbuffer, execution time [ms] " + executionTime);

    start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        for (int j = 0; j < size; j++) {
            arrayQueue.add(obj);
            Object tmp = arrayQueue.peek();
            tmp = arrayQueue.poll();
        }
    }
    end = System.currentTimeMillis();
    executionTime = ((double) (end - start));
    System.out.println("arrayQueue, execution time [ms] " + executionTime);

    start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        for (int j = 0; j < size; j++) {
            listQueue.add(obj);
            Object tmp = listQueue.peek();
            tmp = listQueue.poll();
        }
    }
    end = System.currentTimeMillis();
    executionTime = ((double) (end - start));
    System.out.println("arrayQueue, execution time [ms] " + executionTime);
}

From source file:com.ibm.crail.tools.CrailBenchmark.java

void writeAsync(String filename, int size, int loop, int batch, int storageClass, int locationClass)
        throws Exception {
    System.out.println("writeAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch "
            + batch + ", storageClass " + storageClass + ", locationClass " + locationClass);

    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    for (int i = 0; i < batch; i++) {
        CrailBuffer buf = null;/*from  ww w .j a v  a 2 s . c om*/
        if (size == CrailConstants.BUFFER_SIZE) {
            buf = fs.allocateBuffer();
        } else if (size < CrailConstants.BUFFER_SIZE) {
            CrailBuffer _buf = fs.allocateBuffer();
            _buf.clear().limit(size);
            buf = _buf.slice();
        } else {
            buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
        }
        bufferQueue.add(buf);
    }

    //warmup
    warmUp(filename, warmup, bufferQueue);

    //benchmark
    System.out.println("starting benchmark...");
    LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>();
    HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>();
    fs.getStatistics().reset();
    long _loop = (long) loop;
    long _bufsize = (long) CrailConstants.BUFFER_SIZE;
    long _capacity = _loop * _bufsize;
    double sumbytes = 0;
    double ops = 0;
    CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass),
            CrailLocationClass.get(locationClass)).get().asFile();
    CrailOutputStream directStream = file.getDirectOutputStream(_capacity);
    long start = System.currentTimeMillis();
    for (int i = 0; i < batch - 1 && ops < loop; i++) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.write(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);
        ops = ops + 1.0;
    }
    while (ops < loop) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.write(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);

        future = futureQueue.poll();
        future.get();
        buf = futureMap.get(future.hashCode());
        bufferQueue.add(buf);

        sumbytes = sumbytes + buf.capacity();
        ops = ops + 1.0;
    }
    while (!futureQueue.isEmpty()) {
        Future<CrailResult> future = futureQueue.poll();
        future.get();
        CrailBuffer buf = futureMap.get(future.hashCode());
        sumbytes = sumbytes + buf.capacity();
        ops = ops + 1.0;
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    directStream.close();

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}