List of usage examples for java.util.concurrent LinkedBlockingQueue isEmpty
boolean isEmpty();
From source file:Test.java
public static void main(String[] args) throws Exception { LinkedBlockingQueue<String> queue = new LinkedBlockingQueue<String>(); FileOutputStream fos = new FileOutputStream("out.log"); DataOutputStream dos = new DataOutputStream(fos); while (!queue.isEmpty()) { dos.writeUTF(queue.take());// w ww. j a va 2 s. co m } }
From source file:com.twitter.distributedlog.admin.DistributedLogAdmin.java
private static Map<String, StreamCandidate> checkStreams( final com.twitter.distributedlog.DistributedLogManagerFactory factory, final Collection<String> streams, final ExecutorService executorService, final BookKeeperClient bkc, final String digestpw, final int concurrency) throws IOException { final LinkedBlockingQueue<String> streamQueue = new LinkedBlockingQueue<String>(); streamQueue.addAll(streams);/* ww w .j a v a2 s.c o m*/ final Map<String, StreamCandidate> candidateMap = new ConcurrentSkipListMap<String, StreamCandidate>(); final AtomicInteger numPendingStreams = new AtomicInteger(streams.size()); final CountDownLatch doneLatch = new CountDownLatch(1); Runnable checkRunnable = new Runnable() { @Override public void run() { while (!streamQueue.isEmpty()) { String stream; try { stream = streamQueue.take(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } StreamCandidate candidate; try { LOG.info("Checking stream {}.", stream); candidate = checkStream(factory, stream, executorService, bkc, digestpw); LOG.info("Checked stream {} - {}.", stream, candidate); } catch (IOException e) { LOG.error("Error on checking stream {} : ", stream, e); doneLatch.countDown(); break; } if (null != candidate) { candidateMap.put(stream, candidate); } if (numPendingStreams.decrementAndGet() == 0) { doneLatch.countDown(); } } } }; Thread[] threads = new Thread[concurrency]; for (int i = 0; i < concurrency; i++) { threads[i] = new Thread(checkRunnable, "check-thread-" + i); threads[i].start(); } try { doneLatch.await(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } if (numPendingStreams.get() != 0) { throw new IOException(numPendingStreams.get() + " streams left w/o checked"); } for (int i = 0; i < concurrency; i++) { threads[i].interrupt(); try { threads[i].join(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } return candidateMap; }
From source file:com.ibm.crail.tools.CrailBenchmark.java
void createFile(String filename, int loop) throws Exception, InterruptedException { System.out.println("createFile, filename " + filename + ", loop " + loop); //warmup/* www . j av a2 s .c o m*/ ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>(); CrailBuffer buf = fs.allocateBuffer(); bufferQueue.add(buf); warmUp(filename, warmup, bufferQueue); fs.freeBuffer(buf); //benchmark System.out.println("starting benchmark..."); fs.getStatistics().reset(); LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>(); fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get() .syncDir(); int filecounter = 0; for (int i = 0; i < loop; i++) { String name = "" + filecounter++; String f = filename + "/" + name; pathQueue.add(f); } double ops = 0; long start = System.currentTimeMillis(); while (!pathQueue.isEmpty()) { String path = pathQueue.poll(); fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get() .syncDir(); } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; double latency = 0.0; if (executionTime > 0) { latency = 1000000.0 * executionTime / ops; } System.out.println("execution time " + executionTime); System.out.println("ops " + ops); System.out.println("latency " + latency); fs.getStatistics().print("close"); }
From source file:com.ibm.crail.tools.CrailBenchmark.java
void readSequentialAsync(String filename, int size, int loop, int batch) throws Exception { System.out.println("readSequentialAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch " + batch); ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>(); for (int i = 0; i < batch; i++) { CrailBuffer buf = null;/*from w ww. jav a 2 s. c o m*/ if (size == CrailConstants.BUFFER_SIZE) { buf = fs.allocateBuffer(); } else if (size < CrailConstants.BUFFER_SIZE) { CrailBuffer _buf = fs.allocateBuffer(); _buf.clear().limit(size); buf = _buf.slice(); } else { buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size)); } bufferQueue.add(buf); } //warmup warmUp(filename, warmup, bufferQueue); //benchmark System.out.println("starting benchmark..."); double sumbytes = 0; double ops = 0; fs.getStatistics().reset(); CrailFile file = fs.lookup(filename).get().asFile(); CrailInputStream directStream = file.getDirectInputStream(file.getCapacity()); HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>(); LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>(); long start = System.currentTimeMillis(); for (int i = 0; i < batch - 1 && ops < loop; i++) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.read(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); ops = ops + 1.0; } while (ops < loop) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.read(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); future = futureQueue.poll(); CrailResult result = future.get(); buf = futureMap.get(future.hashCode()); bufferQueue.add(buf); sumbytes = sumbytes + result.getLen(); ops = ops + 1.0; } while (!futureQueue.isEmpty()) { Future<CrailResult> future = futureQueue.poll(); CrailResult result = future.get(); futureMap.get(future.hashCode()); sumbytes = sumbytes + result.getLen(); ops = ops + 1.0; } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; double throughput = 0.0; double latency = 0.0; double sumbits = sumbytes * 8.0; if (executionTime > 0) { throughput = sumbits / executionTime / 1000.0 / 1000.0; latency = 1000000.0 * executionTime / ops; } directStream.close(); System.out.println("execution time " + executionTime); System.out.println("ops " + ops); System.out.println("sumbytes " + sumbytes); System.out.println("throughput " + throughput); System.out.println("latency " + latency); fs.getStatistics().print("close"); }
From source file:com.ibm.crail.tools.CrailBenchmark.java
void writeAsync(String filename, int size, int loop, int batch, int storageClass, int locationClass) throws Exception { System.out.println("writeAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch " + batch + ", storageClass " + storageClass + ", locationClass " + locationClass); ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>(); for (int i = 0; i < batch; i++) { CrailBuffer buf = null;//from w w w .j a va2 s .c o m if (size == CrailConstants.BUFFER_SIZE) { buf = fs.allocateBuffer(); } else if (size < CrailConstants.BUFFER_SIZE) { CrailBuffer _buf = fs.allocateBuffer(); _buf.clear().limit(size); buf = _buf.slice(); } else { buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size)); } bufferQueue.add(buf); } //warmup warmUp(filename, warmup, bufferQueue); //benchmark System.out.println("starting benchmark..."); LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>(); HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>(); fs.getStatistics().reset(); long _loop = (long) loop; long _bufsize = (long) CrailConstants.BUFFER_SIZE; long _capacity = _loop * _bufsize; double sumbytes = 0; double ops = 0; CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass)).get().asFile(); CrailOutputStream directStream = file.getDirectOutputStream(_capacity); long start = System.currentTimeMillis(); for (int i = 0; i < batch - 1 && ops < loop; i++) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.write(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); ops = ops + 1.0; } while (ops < loop) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.write(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); future = futureQueue.poll(); future.get(); buf = futureMap.get(future.hashCode()); bufferQueue.add(buf); sumbytes = sumbytes + buf.capacity(); ops = ops + 1.0; } while (!futureQueue.isEmpty()) { Future<CrailResult> future = futureQueue.poll(); future.get(); CrailBuffer buf = futureMap.get(future.hashCode()); sumbytes = sumbytes + buf.capacity(); ops = ops + 1.0; } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; double throughput = 0.0; double latency = 0.0; double sumbits = sumbytes * 8.0; if (executionTime > 0) { throughput = sumbits / executionTime / 1000.0 / 1000.0; latency = 1000000.0 * executionTime / ops; } directStream.close(); System.out.println("execution time " + executionTime); System.out.println("ops " + ops); System.out.println("sumbytes " + sumbytes); System.out.println("throughput " + throughput); System.out.println("latency " + latency); fs.getStatistics().print("close"); }
From source file:org.apache.distributedlog.admin.DistributedLogAdmin.java
private static Map<String, StreamCandidate> checkStreams(final Namespace namespace, final Collection<String> streams, final OrderedScheduler scheduler, final int concurrency) throws IOException { final LinkedBlockingQueue<String> streamQueue = new LinkedBlockingQueue<String>(); streamQueue.addAll(streams);// ww w .ja v a 2 s . co m final Map<String, StreamCandidate> candidateMap = new ConcurrentSkipListMap<String, StreamCandidate>(); final AtomicInteger numPendingStreams = new AtomicInteger(streams.size()); final CountDownLatch doneLatch = new CountDownLatch(1); Runnable checkRunnable = new Runnable() { @Override public void run() { while (!streamQueue.isEmpty()) { String stream; try { stream = streamQueue.take(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } StreamCandidate candidate; try { LOG.info("Checking stream {}.", stream); candidate = checkStream(namespace, stream, scheduler); LOG.info("Checked stream {} - {}.", stream, candidate); } catch (Throwable e) { LOG.error("Error on checking stream {} : ", stream, e); doneLatch.countDown(); break; } if (null != candidate) { candidateMap.put(stream, candidate); } if (numPendingStreams.decrementAndGet() == 0) { doneLatch.countDown(); } } } }; Thread[] threads = new Thread[concurrency]; for (int i = 0; i < concurrency; i++) { threads[i] = new Thread(checkRunnable, "check-thread-" + i); threads[i].start(); } try { doneLatch.await(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } if (numPendingStreams.get() != 0) { throw new IOException(numPendingStreams.get() + " streams left w/o checked"); } for (int i = 0; i < concurrency; i++) { threads[i].interrupt(); try { threads[i].join(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } return candidateMap; }
From source file:org.polymap.core.runtime.PolymapThreadPoolExecutor.java
public static PolymapThreadPoolExecutor newInstance() { // policy://from w w w .ja v a 2s . c o m // - first: spawn up to proc*6 threads (stopped when idle for 180s) // - then: queue 3 times more jobs (for faster feeding workers) // - then: block until queue can take another job // - and/or: expand corePoolSize (via bounds checkecker thread) int procs = Runtime.getRuntime().availableProcessors(); final int nThreads = procs * 6; final int maxThreads = nThreads * 5; // Proper queue bound semantics but just half the task througput of // LinkedBlockingQueue in PerfTest // BlockingQueue queue = new ArrayBlockingQueue( nThreads * 5 ); // Fastest. But unfortunatelle does not limit the number of queued // task, which might result in OOM or similar problem. final LinkedBlockingQueue queue = new LinkedBlockingQueue( /*nThreads * 3*/ ); // Slowest queue. Does not feed the workers well if pool size exeeds. // BlockingQueue queue = new SynchronousQueue(); final PolymapThreadPoolExecutor result = new PolymapThreadPoolExecutor(nThreads, maxThreads, 3 * 60, queue); result.allowCoreThreadTimeOut(true); // async queue bounds checker for unbound LinkedQueue Thread boundsChecker = new Thread("ThreadPoolBoundsChecker") { public void run() { int bound = result.getCorePoolSize() * 3; while (queue != null) { try { Thread.sleep(1000); // log.info( "queued:" + queue.size() // + " - pool:" + result.getCorePoolSize() // + " - largest:" + result.getLargestPoolSize() // + " - completed:" + result.getCompletedTaskCount() ); // need more threads? if (queue.size() > bound && result.getCorePoolSize() < maxThreads) { int n = result.getCorePoolSize() + nThreads; log.info("Thread Pool: increasing core pool size to: " + n); result.setCorePoolSize(n); } // shrink down? if (queue.isEmpty() && result.getActiveCount() == 0 && result.getCorePoolSize() > nThreads) { log.info("Thread Pool: decreasing core pool size to: " + nThreads); result.setCorePoolSize(nThreads); } } catch (Throwable e) { log.warn("", e); } } } }; boundsChecker.setPriority(Thread.MIN_PRIORITY); boundsChecker.start(); return result; }