List of usage examples for java.util.concurrent LinkedBlockingQueue size
public int size()
From source file:org.zoneproject.extractor.plugin.spotlight.App.java
public static void main(String[] args) { LinkedList<Item> itemsPending = new LinkedList<Item>(); Prop[] fr = { new Prop(ZoneOntology.PLUGIN_LANG, "\"fr\"") }; Prop[] en = { new Prop(ZoneOntology.PLUGIN_LANG, "\"en\"") }; LinkedBlockingQueue<AnnotationThread> annotationThreads; HashMap<String, ArrayList<Prop>> propsToSave; propsPendingSave = new HashMap<String, ArrayList<Prop>>(); while (true) { annotationThreads = new LinkedBlockingQueue<AnnotationThread>(); while (true) {//while we can download items Item[] enItems = Database.getItemsNotAnotatedForPluginsWithDeps(PLUGIN_URI, en, SIM_DOWNLOADS / 2); Item[] frItems = Database.getItemsNotAnotatedForPluginsWithDeps(PLUGIN_URI, fr, SIM_DOWNLOADS / 2); Item[] items = (Item[]) ArrayUtils.addAll(enItems, frItems); if (items != null && items.length > 0) { //check if the item is in annotation process for (Item i : items) { boolean exist = false; for (AnnotationThread a : annotationThreads) { if (a.item.getUri().equals(i.getUri())) { exist = true; }/* w ww . ja v a 2s.c o m*/ } if (!exist) { itemsPending.add(i); } } } if (itemsPending.isEmpty()) { break; } while (!itemsPending.isEmpty()) { //we add new thread until the limit length is thrown while (annotationThreads.size() < SIM_ANNOTATE && !itemsPending.isEmpty()) { AnnotationThread newAnnot = new AnnotationThread(itemsPending.removeFirst()); newAnnot.start(); annotationThreads.add(newAnnot); } //try{ //we try to end some terminated threads //synchronized(annotationThreads){ for (AnnotationThread a : annotationThreads) { if (!a.isAlive()) { annotationThreads.remove(a); } else if (a.getDuration() > LIMIT_TIME_FOR_DOWN) { a.interrupt(); } else if (a.getDuration() > 10) { logger.info("is alive[" + a.getDuration() + "]: " + a.item.getUri()); } //try{Thread.currentThread().sleep(1000);}catch(Exception ie){}//TODO remove } //} //}catch(java.util.ConcurrentModificationException concurrentAccess){ // logger.warn("concurrent access!"); //} if (annotationThreads.size() >= SIM_ANNOTATE) { try { Thread.currentThread().sleep(1000); } catch (Exception ie) { } } } logger.info("start saving"); synchronized (propsPendingSave) { propsToSave = (HashMap<String, ArrayList<Prop>>) propsPendingSave.clone(); propsPendingSave.clear(); } Database.addAnnotations(propsToSave); logger.info("end saving"); } logger.info("no more items to annotate"); try { Thread.currentThread().sleep(1000); } catch (Exception ie) { } } }
From source file:au.org.ala.layers.stats.ObjectsStatsGenerator.java
private static void updateArea(String fid) { try {/*w w w. jav a 2 s .c o m*/ Connection conn = getConnection(); String sql = "SELECT pid from objects where area_km is null and st_geometrytype(the_geom) <> 'Point'"; if (fid != null) { sql = sql + " and fid = '" + fid + "'"; } sql = sql + " limit 200000;"; System.out.println("loading area_km ..."); Statement s1 = conn.createStatement(); ResultSet rs1 = s1.executeQuery(sql); LinkedBlockingQueue<String> data = new LinkedBlockingQueue<String>(); while (rs1.next()) { data.put(rs1.getString("pid")); } CountDownLatch cdl = new CountDownLatch(data.size()); AreaThread[] threads = new AreaThread[CONCURRENT_THREADS]; for (int j = 0; j < CONCURRENT_THREADS; j++) { threads[j] = new AreaThread(data, cdl, getConnection().createStatement()); threads[j].start(); } cdl.await(); for (int j = 0; j < CONCURRENT_THREADS; j++) { threads[j].s.close(); threads[j].interrupt(); } rs1.close(); s1.close(); conn.close(); return; } catch (Exception e) { logger.error(e.getMessage(), e); } return; }
From source file:au.org.ala.layers.stats.ObjectsStatsGenerator.java
private static void updateBbox() { try {/*from w w w. ja va 2 s.c o m*/ Connection conn = getConnection(); String sql = "SELECT pid from objects where bbox is null limit 200000;"; logger.info("loading bbox ..."); Statement s1 = conn.createStatement(); ResultSet rs1 = s1.executeQuery(sql); LinkedBlockingQueue<String[]> data = new LinkedBlockingQueue<String[]>(); while (rs1.next()) { data.put(new String[] { rs1.getString("pid") }); } CountDownLatch cdl = new CountDownLatch(data.size()); BboxThread[] threads = new BboxThread[CONCURRENT_THREADS]; for (int j = 0; j < CONCURRENT_THREADS; j++) { threads[j] = new BboxThread(data, cdl, getConnection().createStatement()); threads[j].start(); } cdl.await(); for (int j = 0; j < CONCURRENT_THREADS; j++) { threads[j].s.close(); threads[j].interrupt(); } return; } catch (Exception e) { logger.error(e.getMessage(), e); } return; }
From source file:org.apache.hadoop.hbase.trigger.WritePrepared.java
/** * Flush current action's pending Puts.// www . ja va 2 s. c om * TODO I am pretty sure there is an error while processing the flush operations * It seems that the for statement does not work at all, and different flush may cause * interacts. I need more information about Java Multi-Thread programming! * @param action */ public static void flush(HTriggerAction action) { System.out.println("enter Flush"); int triggerId = action.getHTrigger().getTriggerId(); long round = action.getCurrentRound(); LinkedBlockingQueue<WriteUnit> writes = cachedElements.get(triggerId); //AtomicBoolean flag = flags.get(triggerId); //if other thread is flushing, we should just leave. /* * It is not necessary to stop other flushes as there is no possible other flush existing. if (!flag.compareAndSet(false, true)) return; */ try { WriteUnit w = writes.poll(); HTable ins = null; while (w != null) { System.out.println("start flush: " + w + "remain: " + writes.size()); ins = getOrNewHTableInstance(w.getTableName()); ins.put(w.getPut()); w = writes.poll(); System.out.println("end flush: " + w + "remain: " + writes.size()); } ins.flushCommits(); } catch (Exception e) { LOG.info("Exceptions While Calling HTable's Put"); } //record successful flush for future recovery. //In fact, there should be a watcher monitoring on these dir and //delete entries written by recordZKActionRound. recordZKWritesFlushed(CurrentRS, triggerId, round); //LOG.info("Trigger" + triggerId + " at round" + round + " Flush OK"); }
From source file:com.twitter.distributedlog.auditor.DLAuditor.java
static <T> void executeAction(final LinkedBlockingQueue<T> queue, final int numThreads, final Action<T> action) throws IOException { final CountDownLatch failureLatch = new CountDownLatch(1); final CountDownLatch doneLatch = new CountDownLatch(queue.size()); final AtomicInteger numFailures = new AtomicInteger(0); final AtomicInteger completedThreads = new AtomicInteger(0); ExecutorService executorService = Executors.newFixedThreadPool(numThreads); try {//from w ww. j a va 2 s. c o m for (int i = 0; i < numThreads; i++) { executorService.submit(new Runnable() { @Override public void run() { while (true) { T item = queue.poll(); if (null == item) { break; } try { action.execute(item); } catch (IOException ioe) { logger.error("Failed to execute action on item '{}'", item, ioe); numFailures.incrementAndGet(); failureLatch.countDown(); break; } doneLatch.countDown(); } if (numFailures.get() == 0 && completedThreads.incrementAndGet() == numThreads) { failureLatch.countDown(); } } }); } try { failureLatch.await(); if (numFailures.get() > 0) { throw new IOException("Encountered " + numFailures.get() + " failures on executing action."); } doneLatch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); logger.warn("Interrupted on executing action", ie); throw new DLInterruptedException("Interrupted on executing action", ie); } } finally { executorService.shutdown(); } }
From source file:cn.wanghaomiao.seimi.def.DefaultLocalQueue.java
@Override public long len(String crawlerName) { LinkedBlockingQueue<Request> queue = getQueue(crawlerName); return queue.size(); }
From source file:co.cask.cdap.filetailer.tailer.RunFromSaveStateTest.java
private void saveState(LinkedBlockingQueue<FileTailerEvent> internalQueue, FileTailerQueue queue, List<String> readLogList, FileTailerStateProcessor stateProcessor) throws InterruptedException, FileTailerStateProcessorException { while (internalQueue.size() > 0) { FileTailerEvent event = queue.take(); readLogList.add(event.getEventData()); stateProcessor.saveState(event.getState()); }//from w ww. j a va 2 s .c om }
From source file:org.polymap.core.runtime.PolymapThreadPoolExecutor.java
public static PolymapThreadPoolExecutor newInstance() { // policy:/* w w w . j ava 2 s .c o m*/ // - first: spawn up to proc*6 threads (stopped when idle for 180s) // - then: queue 3 times more jobs (for faster feeding workers) // - then: block until queue can take another job // - and/or: expand corePoolSize (via bounds checkecker thread) int procs = Runtime.getRuntime().availableProcessors(); final int nThreads = procs * 6; final int maxThreads = nThreads * 5; // Proper queue bound semantics but just half the task througput of // LinkedBlockingQueue in PerfTest // BlockingQueue queue = new ArrayBlockingQueue( nThreads * 5 ); // Fastest. But unfortunatelle does not limit the number of queued // task, which might result in OOM or similar problem. final LinkedBlockingQueue queue = new LinkedBlockingQueue( /*nThreads * 3*/ ); // Slowest queue. Does not feed the workers well if pool size exeeds. // BlockingQueue queue = new SynchronousQueue(); final PolymapThreadPoolExecutor result = new PolymapThreadPoolExecutor(nThreads, maxThreads, 3 * 60, queue); result.allowCoreThreadTimeOut(true); // async queue bounds checker for unbound LinkedQueue Thread boundsChecker = new Thread("ThreadPoolBoundsChecker") { public void run() { int bound = result.getCorePoolSize() * 3; while (queue != null) { try { Thread.sleep(1000); // log.info( "queued:" + queue.size() // + " - pool:" + result.getCorePoolSize() // + " - largest:" + result.getLargestPoolSize() // + " - completed:" + result.getCompletedTaskCount() ); // need more threads? if (queue.size() > bound && result.getCorePoolSize() < maxThreads) { int n = result.getCorePoolSize() + nThreads; log.info("Thread Pool: increasing core pool size to: " + n); result.setCorePoolSize(n); } // shrink down? if (queue.isEmpty() && result.getActiveCount() == 0 && result.getCorePoolSize() > nThreads) { log.info("Thread Pool: decreasing core pool size to: " + nThreads); result.setCorePoolSize(nThreads); } } catch (Throwable e) { log.warn("", e); } } } }; boundsChecker.setPriority(Thread.MIN_PRIORITY); boundsChecker.start(); return result; }
From source file:org.wso2.carbon.databridge.agent.thrift.lb.ReceiverGroup.java
public void resendEvents(LinkedBlockingQueue<Event> events) { if (null != events) { if (events.size() > 0) { log.info("Resending the failed events...."); }// w w w . j a v a 2s . co m while (true) { Event event = events.poll(); if (null != event) { publish(event); } else { break; } } } }
From source file:org.wso2.carbon.databridge.agent.thrift.lb.ReceiverGroup.java
public void resendPublishedData(LinkedBlockingQueue<PublishData> publishDatas) { if (null != publishDatas) { if (publishDatas.size() > 0) { log.info("Resending the failed published data..."); }/*from w ww. j a v a 2 s. co m*/ while (true) { PublishData data = publishDatas.poll(); if (null != data) { try { if (data.getStreamName() == null) { publish(data.getEvent()); } else { publish(data.getStreamName(), data.getStreamVersion(), data.getEvent()); } } catch (AgentException e) { log.error(e); } } else { break; } } } }