Example usage for java.util.concurrent ConcurrentLinkedQueue add

Introduction

In this page you can find the example usage for java.util.concurrent ConcurrentLinkedQueue add.

Prototype

public boolean add(E e) 

Source Link

Document

Inserts the specified element at the tail of this queue.

Usage

From source file:com.linkedin.pinot.perf.QueryRunner.java

/**
 * Use multiple threads to run query at an increasing target QPS.
 *
 * Use a concurrent linked queue to buffer the queries to be sent. Use the main thread to insert queries into the
 * queue at the target QPS, and start {numThreads} worker threads to fetch queries from the queue and send them.
 * We start with the start QPS, and keep adding delta QPS to the start QPS during the test. The main thread is
 * responsible for collecting the statistic information and log them periodically.
 *
 * @param conf perf benchmark driver config.
 * @param queryFile query file./*w  w w  . java  2 s  . c o  m*/
 * @param numThreads number of threads sending queries.
 * @param startQPS start QPS
 * @param deltaQPS delta QPS
 * @throws Exception
 */
@SuppressWarnings("InfiniteLoopStatement")
public static void targetQPSQueryRunner(PerfBenchmarkDriverConf conf, String queryFile, int numThreads,
        double startQPS, double deltaQPS) throws Exception {
    final long randomSeed = 123456789L;
    final Random random = new Random(randomSeed);
    final int timePerTargetQPSMillis = 60000;
    final int queueLengthThreshold = Math.max(20, (int) startQPS);

    final List<String> queries;
    try (FileInputStream input = new FileInputStream(new File(queryFile))) {
        queries = IOUtils.readLines(input);
    }
    final int numQueries = queries.size();

    final PerfBenchmarkDriver driver = new PerfBenchmarkDriver(conf);
    final AtomicInteger counter = new AtomicInteger(0);
    final AtomicLong totalResponseTime = new AtomicLong(0L);
    final ExecutorService executorService = Executors.newFixedThreadPool(numThreads);

    final ConcurrentLinkedQueue<String> queryQueue = new ConcurrentLinkedQueue<>();
    double currentQPS = startQPS;
    int intervalMillis = (int) (MILLIS_PER_SECOND / currentQPS);

    for (int i = 0; i < numThreads; i++) {
        executorService.submit(new Runnable() {
            @Override
            public void run() {
                while (true) {
                    String query = queryQueue.poll();
                    if (query == null) {
                        try {
                            Thread.sleep(1);
                            continue;
                        } catch (InterruptedException e) {
                            LOGGER.error("Interrupted.", e);
                            return;
                        }
                    }
                    long startTime = System.currentTimeMillis();
                    try {
                        driver.postQuery(query);
                        counter.getAndIncrement();
                        totalResponseTime.getAndAdd(System.currentTimeMillis() - startTime);
                    } catch (Exception e) {
                        LOGGER.error("Caught exception while running query: {}", query, e);
                        return;
                    }
                }
            }
        });
    }

    LOGGER.info("Start with QPS: {}, delta QPS: {}", startQPS, deltaQPS);
    while (true) {
        long startTime = System.currentTimeMillis();
        while (System.currentTimeMillis() - startTime <= timePerTargetQPSMillis) {
            if (queryQueue.size() > queueLengthThreshold) {
                executorService.shutdownNow();
                throw new RuntimeException("Cannot achieve target QPS of: " + currentQPS);
            }
            queryQueue.add(queries.get(random.nextInt(numQueries)));
            Thread.sleep(intervalMillis);
        }
        double timePassedSeconds = ((double) (System.currentTimeMillis() - startTime)) / MILLIS_PER_SECOND;
        int count = counter.getAndSet(0);
        double avgResponseTime = ((double) totalResponseTime.getAndSet(0)) / count;
        LOGGER.info("Target QPS: {}, Interval: {}ms, Actual QPS: {}, Avg Response Time: {}ms", currentQPS,
                intervalMillis, count / timePassedSeconds, avgResponseTime);

        // Find a new interval
        int newIntervalMillis;
        do {
            currentQPS += deltaQPS;
            newIntervalMillis = (int) (MILLIS_PER_SECOND / currentQPS);
        } while (newIntervalMillis == intervalMillis);
        intervalMillis = newIntervalMillis;
    }
}

---

From source file:com.ibm.crail.tools.CrailBenchmark.java

void write(String filename, int size, int loop, int storageClass, int locationClass, boolean buffered)
        throws Exception {
    System.out.println("write, filename " + filename + ", size " + size + ", loop " + loop + ", storageClass "
            + storageClass + ", locationClass " + locationClass + ", buffered " + buffered);

    CrailBuffer buf = null;/* w  w w.j  av a2  s  .co  m*/
    if (size == CrailConstants.BUFFER_SIZE) {
        buf = fs.allocateBuffer();
    } else if (size < CrailConstants.BUFFER_SIZE) {
        CrailBuffer _buf = fs.allocateBuffer();
        _buf.clear().limit(size);
        buf = _buf.slice();
    } else {
        buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
    }

    //warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    long _loop = (long) loop;
    long _bufsize = (long) CrailConstants.BUFFER_SIZE;
    long _capacity = _loop * _bufsize;
    double sumbytes = 0;
    double ops = 0;
    CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass),
            CrailLocationClass.get(locationClass)).get().asFile();
    CrailBufferedOutputStream bufferedStream = buffered ? file.getBufferedOutputStream(_capacity) : null;
    CrailOutputStream directStream = !buffered ? file.getDirectOutputStream(_capacity) : null;
    long start = System.currentTimeMillis();
    while (ops < loop) {
        buf.clear();
        if (buffered) {
            bufferedStream.write(buf.getByteBuffer());
        } else {
            directStream.write(buf).get();
        }
        sumbytes = sumbytes + buf.capacity();
        ops = ops + 1.0;
    }
    if (buffered) {
        bufferedStream.close();
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}

---

From source file:com.ibm.crail.tools.CrailBenchmark.java

void enumerateDir(String filename, int loop) throws Exception {
    System.out.println("reading enumarate dir, path " + filename);

    //warmup/*from w w w  .  ja va 2s .  c o m*/
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        // single operation == loop
        Iterator<String> iter = fs.lookup(filename).get().asDirectory().listEntries();
        while (iter.hasNext()) {
            iter.next();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) loop);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);

    fs.getStatistics().print("close");
}

---

From source file:com.ibm.crail.tools.CrailBenchmark.java

void getFile(String filename, int loop) throws Exception, InterruptedException {
    System.out.println("getFile, filename " + filename + ", loop " + loop);

    //warmup//ww  w  .j  a  v  a 2s. c om
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    double ops = 0;
    long start = System.currentTimeMillis();
    while (ops < loop) {
        ops = ops + 1.0;
        fs.lookup(filename).get().asFile();
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double latency = 0.0;
    if (executionTime > 0) {
        latency = 1000000.0 * executionTime / ops;
    }
    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
    fs.close();
}

---

From source file:com.ibm.crail.tools.CrailBenchmark.java

void readSequential(String filename, int size, int loop, boolean buffered) throws Exception {
    System.out.println("readSequential, filename " + filename + ", size " + size + ", loop " + loop
            + ", buffered " + buffered);

    CrailBuffer buf = null;/*from ww w  .j  av a 2s.c  om*/
    if (size == CrailConstants.BUFFER_SIZE) {
        buf = fs.allocateBuffer();
    } else if (size < CrailConstants.BUFFER_SIZE) {
        CrailBuffer _buf = fs.allocateBuffer();
        _buf.clear().limit(size);
        buf = _buf.slice();
    } else {
        buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
    }

    //warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);

    CrailFile file = fs.lookup(filename).get().asFile();
    CrailBufferedInputStream bufferedStream = file.getBufferedInputStream(file.getCapacity());
    CrailInputStream directStream = file.getDirectInputStream(file.getCapacity());

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    double sumbytes = 0;
    double ops = 0;
    long start = System.currentTimeMillis();
    while (ops < loop) {
        if (buffered) {
            buf.clear();
            double ret = (double) bufferedStream.read(buf.getByteBuffer());
            if (ret > 0) {
                sumbytes = sumbytes + ret;
                ops = ops + 1.0;
            } else {
                ops = ops + 1.0;
                if (bufferedStream.position() == 0) {
                    break;
                } else {
                    bufferedStream.seek(0);
                }
            }
        } else {
            buf.clear();
            double ret = (double) directStream.read(buf).get().getLen();
            if (ret > 0) {
                sumbytes = sumbytes + ret;
                ops = ops + 1.0;
            } else {
                ops = ops + 1.0;
                if (directStream.position() == 0) {
                    break;
                } else {
                    directStream.seek(0);
                }
            }
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    bufferedStream.close();
    directStream.close();

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}

---

From source file:com.ibm.crail.tools.CrailBenchmark.java

void readRandom(String filename, int size, int loop, boolean buffered) throws Exception {
    System.out.println("readRandom, filename " + filename + ", size " + size + ", loop " + loop + ", buffered "
            + buffered);/*from   w  w  w . ja  v  a  2  s .  c o m*/

    CrailBuffer buf = null;
    if (size == CrailConstants.BUFFER_SIZE) {
        buf = fs.allocateBuffer();
    } else if (size < CrailConstants.BUFFER_SIZE) {
        CrailBuffer _buf = fs.allocateBuffer();
        _buf.clear().limit(size);
        buf = _buf.slice();
    } else {
        buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
    }

    //warmup
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    CrailFile file = fs.lookup(filename).get().asFile();
    CrailBufferedInputStream bufferedStream = file.getBufferedInputStream(file.getCapacity());
    CrailInputStream directStream = file.getDirectInputStream(file.getCapacity());

    double sumbytes = 0;
    double ops = 0;
    long _range = file.getCapacity() - ((long) buf.capacity());
    double range = (double) _range;
    Random random = new Random();

    long start = System.currentTimeMillis();
    while (ops < loop) {
        if (buffered) {
            buf.clear();
            double _offset = range * random.nextDouble();
            long offset = (long) _offset;
            directStream.seek(offset);
            double ret = (double) directStream.read(buf).get().getLen();
            if (ret > 0) {
                sumbytes = sumbytes + ret;
                ops = ops + 1.0;
            } else {
                break;
            }
        } else {
            buf.clear();
            double _offset = range * random.nextDouble();
            long offset = (long) _offset;
            bufferedStream.seek(offset);
            double ret = (double) bufferedStream.read(buf.getByteBuffer());
            if (ret > 0) {
                sumbytes = sumbytes + ret;
                ops = ops + 1.0;
            } else {
                break;
            }
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    bufferedStream.close();
    directStream.close();

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}

---

From source file:com.ibm.crail.tools.CrailBenchmark.java

void getFileAsync(String filename, int loop, int batch) throws Exception, InterruptedException {
    System.out.println("getFileAsync, filename " + filename + ", loop " + loop + ", batch " + batch);

    //warmup//  ww w  .ja  v a  2  s  . co  m
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<Future<CrailNode>> fileQueue = new LinkedBlockingQueue<Future<CrailNode>>();
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        //single operation == loop
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = fs.lookup(filename);
            fileQueue.add(future);
        }
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = fileQueue.poll();
            future.get();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) batch);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);

    fs.getStatistics().print("close");
}

---

From source file:com.ibm.crail.tools.CrailBenchmark.java

void createFile(String filename, int loop) throws Exception, InterruptedException {
    System.out.println("createFile, filename " + filename + ", loop " + loop);

    //warmup/*from   ww  w.ja  v  a 2s  . c  o m*/
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>();
    fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get()
            .syncDir();
    int filecounter = 0;
    for (int i = 0; i < loop; i++) {
        String name = "" + filecounter++;
        String f = filename + "/" + name;
        pathQueue.add(f);
    }

    double ops = 0;
    long start = System.currentTimeMillis();
    while (!pathQueue.isEmpty()) {
        String path = pathQueue.poll();
        fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get()
                .syncDir();
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double latency = 0.0;
    if (executionTime > 0) {
        latency = 1000000.0 * executionTime / ops;
    }

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}

---

From source file:com.ibm.crail.tools.CrailBenchmark.java

void createFileAsync(String filename, int loop, int batch) throws Exception, InterruptedException {
    System.out.println("createFileAsync, filename " + filename + ", loop " + loop + ", batch " + batch);

    //warmup//from  w w  w  . jav  a2s . c o m
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<Future<CrailNode>> futureQueue = new LinkedBlockingQueue<Future<CrailNode>>();
    LinkedBlockingQueue<CrailFile> fileQueue = new LinkedBlockingQueue<CrailFile>();
    LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>();
    fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get()
            .syncDir();

    for (int i = 0; i < loop; i++) {
        String name = "/" + i;
        String f = filename + name;
        pathQueue.add(f);
    }

    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i += batch) {
        //single operation == loop
        for (int j = 0; j < batch; j++) {
            String path = pathQueue.poll();
            Future<CrailNode> future = fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT,
                    CrailLocationClass.DEFAULT);
            futureQueue.add(future);
        }
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = futureQueue.poll();
            CrailFile file = future.get().asFile();
            fileQueue.add(file);
        }
        for (int j = 0; j < batch; j++) {
            CrailFile file = fileQueue.poll();
            file.syncDir();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) loop);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);

    fs.delete(filename, true).get().syncDir();

    fs.getStatistics().print("close");

}

---

From source file:metlos.executors.batch.BatchExecutorTest.java

public void testChangesInTaskCollectionPickedUpInRepetitions() throws Exception {
    final ConcurrentLinkedQueue<Runnable> tasks = new ConcurrentLinkedQueue<Runnable>();
    final AtomicInteger reportedNofTasks = new AtomicInteger();
    final CountDownLatch waitForTask2 = new CountDownLatch(2);
    final CountDownLatch waitForTask3 = new CountDownLatch(2);

    Runnable task1 = new Runnable() {
        @Override/*  w  w  w  . j a  va  2s  .com*/
        public void run() {
        }
    };

    Runnable task2 = new Runnable() {
        @Override
        public void run() {
            if (tasks.size() == 2) {
                reportedNofTasks.set(2);
                waitForTask2.countDown();
            }
        }
    };

    Runnable task3 = new Runnable() {
        @Override
        public void run() {
            if (tasks.size() == 3) {
                reportedNofTasks.set(3);
                waitForTask3.countDown();
            }
        }
    };

    BatchExecutor ex = getExecutor(10);

    tasks.add(task1);
    tasks.add(task2);

    ex.submitWithPreferedDurationAndFixedDelay(tasks, 0, 0, 0, TimeUnit.MILLISECONDS);

    //k, now the tasks should be running and there should be just 2 of them...
    //so we should be getting the value of "2" reported by the reportedNofTasks

    waitForTask2.countDown();
    waitForTask2.await();

    int currentReportedTasks = reportedNofTasks.get();

    assert currentReportedTasks == 2 : "We should be getting 2 tasks reported but are getting "
            + currentReportedTasks + " instead.";

    //k, now let's try updating the tasks collection... this should get picked up by the
    //repeated executions 
    tasks.add(task3);

    //now the reported nof tasks should change to 3. let's wait on it first to make sure the executor has had time to 
    //register the change.        
    waitForTask3.countDown();
    waitForTask3.await();

    currentReportedTasks = reportedNofTasks.get();

    assert currentReportedTasks == 3 : "We should be getting 3 tasks reported but are getting "
            + currentReportedTasks + " instead.";

    ex.shutdown();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• »