List of usage examples for java.util.concurrent BlockingQueue addAll
boolean addAll(Collection<? extends E> c);
From source file:com.betfair.cougar.test.socket.app.SocketCompatibilityTestingApp.java
public static void main(String[] args) throws Exception { Parser parser = new PosixParser(); Options options = new Options(); options.addOption("r", "repo", true, "Repository type to search: local|central"); options.addOption("c", "client-concurrency", true, "Max threads to allow each client tester to run tests, defaults to 10"); options.addOption("t", "test-concurrency", true, "Max client testers to run concurrently, defaults to 5"); options.addOption("m", "max-time", true, "Max time (in minutes) to allow tests to complete, defaults to 10"); options.addOption("v", "version", false, "Print version and exit"); options.addOption("h", "help", false, "This help text"); CommandLine commandLine = parser.parse(options, args); if (commandLine.hasOption("h")) { System.out.println(options); System.exit(0);/*from w ww . ja v a 2 s .c om*/ } if (commandLine.hasOption("v")) { System.out.println("How the hell should I know?"); System.exit(0); } // 1. Find all testers in given repos List<RepoSearcher> repoSearchers = new ArrayList<>(); for (String repo : commandLine.getOptionValues("r")) { if ("local".equals(repo.toLowerCase())) { repoSearchers.add(new LocalRepoSearcher()); } else if ("central".equals(repo.toLowerCase())) { repoSearchers.add(new CentralRepoSearcher()); } else { System.err.println("Unrecognized repo: " + repo); System.err.println(options); System.exit(1); } } int clientConcurrency = 10; if (commandLine.hasOption("c")) { try { clientConcurrency = Integer.parseInt(commandLine.getOptionValue("c")); } catch (NumberFormatException nfe) { System.err.println( "client-concurrency is not a valid integer: '" + commandLine.getOptionValue("c") + "'"); System.exit(1); } } int testConcurrency = 5; if (commandLine.hasOption("t")) { try { testConcurrency = Integer.parseInt(commandLine.getOptionValue("t")); } catch (NumberFormatException nfe) { System.err.println( "test-concurrency is not a valid integer: '" + commandLine.getOptionValue("t") + "'"); System.exit(1); } } int maxMinutes = 10; if (commandLine.hasOption("m")) { try { maxMinutes = Integer.parseInt(commandLine.getOptionValue("m")); } catch (NumberFormatException nfe) { System.err.println("max-time is not a valid integer: '" + commandLine.getOptionValue("m") + "'"); System.exit(1); } } Properties clientProps = new Properties(); clientProps.setProperty("client.concurrency", String.valueOf(clientConcurrency)); File baseRunDir = new File(System.getProperty("user.dir") + "/run"); baseRunDir.mkdirs(); File tmpDir = new File(baseRunDir, "jars"); tmpDir.mkdirs(); List<ServerRunner> serverRunners = new ArrayList<>(); List<ClientRunner> clientRunners = new ArrayList<>(); for (RepoSearcher searcher : repoSearchers) { List<File> jars = searcher.findAndCache(tmpDir); for (File f : jars) { ServerRunner serverRunner = new ServerRunner(f, baseRunDir); System.out.println("Found tester: " + serverRunner.getVersion()); serverRunners.add(serverRunner); clientRunners.add(new ClientRunner(f, baseRunDir, clientProps)); } } // 2. Start servers and collect ports System.out.println(); System.out.println("Starting " + serverRunners.size() + " servers..."); for (ServerRunner server : serverRunners) { server.startServer(); } System.out.println(); List<TestCombo> tests = new ArrayList<>(serverRunners.size() * clientRunners.size()); for (ServerRunner server : serverRunners) { for (ClientRunner client : clientRunners) { tests.add(new TestCombo(server, client)); } } System.out.println("Enqueued " + tests.size() + " test combos to run..."); long startTime = System.currentTimeMillis(); // 3. Run every client against every server, collecting results BlockingQueue<Runnable> workQueue = new ArrayBlockingQueue(serverRunners.size() * clientRunners.size()); ThreadPoolExecutor service = new ThreadPoolExecutor(testConcurrency, testConcurrency, 5000, TimeUnit.MILLISECONDS, workQueue); service.prestartAllCoreThreads(); workQueue.addAll(tests); while (!workQueue.isEmpty()) { Thread.sleep(1000); } service.shutdown(); service.awaitTermination(maxMinutes, TimeUnit.MINUTES); long endTime = System.currentTimeMillis(); long totalTimeSecs = Math.round((endTime - startTime) / 1000.0); for (ServerRunner server : serverRunners) { server.shutdownServer(); } System.out.println(); System.out.println("======="); System.out.println("Results"); System.out.println("-------"); // print a summary int totalTests = 0; int totalSuccess = 0; for (TestCombo combo : tests) { String clientVer = combo.getClientVersion(); String serverVer = combo.getServerVersion(); String results = combo.getClientResults(); ObjectMapper mapper = new ObjectMapper(new JsonFactory()); JsonNode node = mapper.reader().readTree(results); JsonNode resultsArray = node.get("results"); int numTests = resultsArray.size(); int numSuccess = 0; for (int i = 0; i < numTests; i++) { if ("success".equals(resultsArray.get(i).get("result").asText())) { numSuccess++; } } totalSuccess += numSuccess; totalTests += numTests; System.out.println(clientVer + "/" + serverVer + ": " + numSuccess + "/" + numTests + " succeeded - took " + String.format("%2f", combo.getRunningTime()) + " seconds"); } System.out.println("-------"); System.out.println( "Overall: " + totalSuccess + "/" + totalTests + " succeeded - took " + totalTimeSecs + " seconds"); FileWriter out = new FileWriter("results.json"); PrintWriter pw = new PrintWriter(out); // 4. Output full results pw.println("{\n \"results\": ["); for (TestCombo combo : tests) { combo.emitResults(pw, " "); } pw.println(" ],"); pw.println(" \"servers\": ["); for (ServerRunner server : serverRunners) { server.emitInfo(pw, " "); } pw.println(" ],"); pw.close(); }
From source file:com.yahoo.storm.yarn.MasterServer.java
private Thread initAndStartHeartbeat(final StormAMRMClient client, final BlockingQueue<Container> launcherQueue, final int heartBeatIntervalMs) { Thread thread = new Thread() { @Override/*from ww w . j a va 2 s . co m*/ public void run() { try { while (client.getServiceState() == Service.STATE.STARTED && !Thread.currentThread().isInterrupted()) { Thread.sleep(heartBeatIntervalMs); // We always send 50% progress. AllocateResponse allocResponse = client.allocate(0.5f); AMCommand am_command = allocResponse.getAMCommand(); if (am_command != null && (am_command == AMCommand.AM_SHUTDOWN || am_command == AMCommand.AM_RESYNC)) { LOG.info("Got AM_SHUTDOWN or AM_RESYNC from the RM"); _handler.stop(); System.exit(0); } List<Container> allocatedContainers = allocResponse.getAllocatedContainers(); if (allocatedContainers.size() > 0) { // Add newly allocated containers to the client. LOG.info("HB: Received allocated containers (" + allocatedContainers.size() + ")"); client.addAllocatedContainers(allocatedContainers); if (client.supervisorsAreToRun()) { LOG.info("HB: Supervisors are to run, so queueing (" + allocatedContainers.size() + ") containers..."); launcherQueue.addAll(allocatedContainers); } else { LOG.info("HB: Supervisors are to stop, so releasing all containers..."); client.stopAllSupervisors(); } } List<ContainerStatus> completedContainers = allocResponse.getCompletedContainersStatuses(); if (completedContainers.size() > 0 && client.supervisorsAreToRun()) { LOG.debug("HB: Containers completed (" + completedContainers.size() + "), so releasing them."); client.addSupervisors(completedContainers.size()); } } } catch (Throwable t) { // Something happened we could not handle. Make sure the AM goes // down so that we are not surprised later on that our heart // stopped.. LOG.error("Unhandled error in AM: ", t); _handler.stop(); System.exit(1); } } }; thread.start(); return thread; }
From source file:ubic.gemma.core.apps.ArrayDesignBlatCli.java
@Override protected Exception doWork(String[] args) { Exception err = this.processCommandLine(args); if (err != null) return err; final Date skipIfLastRunLaterThan = this.getLimitingDate(); if (!this.getArrayDesignsToProcess().isEmpty()) { if (this.blatResultFile != null && this.getArrayDesignsToProcess().size() > 1) { throw new IllegalArgumentException( "Cannot provide a blat result file when multiple arrays are being analyzed"); }//from w w w. j av a2s .c o m for (ArrayDesign arrayDesign : this.getArrayDesignsToProcess()) { if (!this.shouldRun(skipIfLastRunLaterThan, arrayDesign, ArrayDesignSequenceAnalysisEvent.class)) { AbstractCLI.log .warn(arrayDesign + " was last run more recently than " + skipIfLastRunLaterThan); return null; } arrayDesign = this.thaw(arrayDesign); Collection<BlatResult> persistedResults; try { if (this.blatResultFile != null) { Collection<BlatResult> blatResults = this.getBlatResultsFromFile(arrayDesign); if (blatResults == null || blatResults.size() == 0) { throw new IllegalStateException("No blat results in file!"); } AbstractCLI.log.info("Got " + blatResults.size() + " blat records"); persistedResults = arrayDesignSequenceAlignmentService.processArrayDesign(arrayDesign, taxon, blatResults); this.audit(arrayDesign, "BLAT results read from file: " + blatResultFile); this.updateMergedOrSubsumed(arrayDesign); } else { // Run blat from scratch. persistedResults = arrayDesignSequenceAlignmentService.processArrayDesign(arrayDesign, this.sensitive); this.audit(arrayDesign, "Based on a fresh alignment analysis; BLAT score threshold was " + this.blatScoreThreshold + "; sensitive mode was " + this.sensitive); this.updateMergedOrSubsumed(arrayDesign); } AbstractCLI.log.info("Persisted " + persistedResults.size() + " results"); } catch (IOException e) { this.errorObjects.add(e); } } } else if (taxon != null) { Collection<ArrayDesign> allArrayDesigns = getArrayDesignService().findByTaxon(taxon); AbstractCLI.log.warn("*** Running BLAT for all " + taxon.getCommonName() + " Array designs *** [" + allArrayDesigns.size() + " items]"); final SecurityContext context = SecurityContextHolder.getContext(); // split over multiple threads so we can multiplex. Put the array designs in a queue. /* * Here is our task runner. */ class BlatCliConsumer extends Consumer { private BlatCliConsumer(BlockingQueue<ArrayDesign> q) { super(q, context); } @Override void consume(ArrayDesign x) { if (!ArrayDesignBlatCli.this.shouldRun(skipIfLastRunLaterThan, x, ArrayDesignSequenceAnalysisEvent.class)) { return; } x = getArrayDesignService().thaw(x); ArrayDesignBlatCli.this.processArrayDesign(x); } } BlockingQueue<ArrayDesign> arrayDesigns = new ArrayBlockingQueue<>(allArrayDesigns.size()); arrayDesigns.addAll(allArrayDesigns); Collection<Thread> threads = new ArrayList<>(); for (int i = 0; i < this.numThreads; i++) { Consumer c1 = new BlatCliConsumer(arrayDesigns); Thread k = new Thread(c1); threads.add(k); k.start(); } this.waitForThreadPoolCompletion(threads); /* * All done */ this.summarizeProcessing(); } else { exitwithError(); } return null; }
From source file:ubic.gemma.loader.expression.arrayDesign.ArrayDesignProbeMapperServiceImpl.java
@Override public void processArrayDesign(ArrayDesign arrayDesign, ProbeMapperConfig config, boolean useDB) { assert config != null; if (arrayDesign.getTechnologyType().equals(TechnologyType.NONE)) { throw new IllegalArgumentException( "Do not use this service to process platforms that do not use an probe-based technology."); }/*from w w w. j ava2 s .com*/ Collection<Taxon> taxa = arrayDesignService.getTaxa(arrayDesign.getId()); Taxon taxon = arrayDesign.getPrimaryTaxon(); if (taxa.size() > 1 && taxon == null) { throw new IllegalArgumentException( "Array design has sequence from multiple taxa and has no primary taxon set: " + arrayDesign); } GoldenPathSequenceAnalysis goldenPathDb = new GoldenPathSequenceAnalysis(taxon); BlockingQueue<BlatAssociation> persistingQueue = new ArrayBlockingQueue<BlatAssociation>(QUEUE_SIZE); AtomicBoolean generatorDone = new AtomicBoolean(false); AtomicBoolean loaderDone = new AtomicBoolean(false); load(persistingQueue, generatorDone, loaderDone); if (useDB) { log.info("Removing any old associations"); arrayDesignService.deleteGeneProductAssociations(arrayDesign); } int count = 0; int hits = 0; log.info("Start processing " + arrayDesign.getCompositeSequences().size() + " probes ..."); for (CompositeSequence compositeSequence : arrayDesign.getCompositeSequences()) { Map<String, Collection<BlatAssociation>> results = processCompositeSequence(config, taxon, goldenPathDb, compositeSequence); if (results == null) continue; for (Collection<BlatAssociation> col : results.values()) { for (BlatAssociation association : col) { if (log.isDebugEnabled()) log.debug(association); } if (useDB) { // persisting is done in a separate thread. persistingQueue.addAll(col); } else { printResult(compositeSequence, col); } ++hits; } if (++count % 200 == 0) { log.info("Processed " + count + " composite sequences" + " with blat results; " + hits + " mappings found."); } } generatorDone.set(true); log.info("Waiting for loading to complete ..."); while (!loaderDone.get()) { try { Thread.sleep(1000); } catch (InterruptedException e) { throw new RuntimeException(e); } } log.info("Processed " + count + " composite sequences with blat results; " + hits + " mappings found."); try { this.deleteOldFiles(arrayDesign); } catch (IOException e) { log.error("Failed to delete all old files associated with " + arrayDesign + ", be sure to clean them up manually or regenerate them"); } arrayDesignReportService.generateArrayDesignReport(arrayDesign.getId()); }