List of usage examples for java.util.concurrent ThreadPoolExecutor invokeAll
public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks) throws InterruptedException
From source file:de.tu_dortmund.ub.data.dswarm.TaskProcessingUnit.java
private static void execute(final String serviceName, final LinkedList<Callable<String>> threads, final ThreadPoolExecutor pool) throws InterruptedException, java.util.concurrent.ExecutionException { try {//from w w w. j a v a 2s . c o m final List<Future<String>> futureList = pool.invokeAll(threads); for (final Future<String> f : futureList) { final String message = f.get(); final String message1 = String.format("[%s] %s", serviceName, message); LOG.info(message1); } } catch (final Exception e) { LOG.error("something went wrong", e); throw e; } finally { pool.shutdown(); } }
From source file:de.tu_dortmund.ub.data.util.TPUUtil.java
public static String executeInit(final String initResourceFile, final String serviceName, final Integer engineThreads, final Properties config, final int cnt) throws Exception { // create job final Callable<String> initTask = new Init(initResourceFile, config, cnt); // work on jobs final ThreadPoolExecutor pool = new ThreadPoolExecutor(engineThreads, engineThreads, 0L, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); try {/*from w ww. j a v a 2s . c om*/ final List<Callable<String>> tasks = new LinkedList<>(); tasks.add(initTask); final List<Future<String>> futureList = pool.invokeAll(tasks); final Iterator<Future<String>> iterator = futureList.iterator(); if (iterator.hasNext()) { final Future<String> f = iterator.next(); final String initResult = f.get(); final String message1 = String.format("[%s][%d] initResult = '%s'", serviceName, cnt, initResult); LOG.info(message1); return initResult; } } catch (final Exception e) { LOG.error("[{]][{}] something went wrong at init part execution", serviceName, cnt, e); throw e; } finally { pool.shutdown(); } return null; }
From source file:de.tu_dortmund.ub.data.dswarm.TaskProcessingUnit.java
private static String executeTPUTask(final String[] watchFolderFiles, final String resourceWatchFolder, final Optional<String> optionalOutputDataModelID, final Optional<String> optionalExportMimeType, final Optional<String> optionalExportFileExtension, final Integer engineThreads, final String serviceName, final Properties config) throws Exception { // create job list final LinkedList<Callable<String>> transforms = new LinkedList<>(); int cnt = 1;/*from w w w. ja v a 2 s .c o m*/ for (final String watchFolderFile : watchFolderFiles) { LOG.info("[{}][{}] do TPU task execution '{}' for file '{}'", serviceName, cnt, cnt, watchFolderFile); transforms.add(new TPUTask(config, watchFolderFile, resourceWatchFolder, optionalOutputDataModelID, optionalExportMimeType, optionalExportFileExtension, serviceName, cnt)); cnt++; } // work on jobs final ThreadPoolExecutor pool = new ThreadPoolExecutor(engineThreads, engineThreads, 0L, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); try { final List<Future<String>> futureList = pool.invokeAll(transforms); final StringBuilder resultSB = new StringBuilder(); for (final Future<String> f : futureList) { final String message = f.get(); LOG.info(message); resultSB.append(message).append("\n"); } return resultSB.toString(); } catch (final Exception e) { LOG.error("something went wrong", e); throw e; } finally { pool.shutdown(); } }
From source file:metlos.executors.batch.BatchExecutorTest.java
private long rapidFireSimpleExecutorTime(final int taskDurationMillis, int nofJobs, int nofThreads) throws Exception { ThreadPoolExecutor ex = new ThreadPoolExecutor(nofThreads, nofThreads, 0, TimeUnit.NANOSECONDS, new LinkedBlockingQueue<Runnable>()); List<Callable<Void>> payload = getCallables(taskDurationMillis, nofJobs); return measureExecutionTime(System.currentTimeMillis(), ex.invokeAll(payload)); }
From source file:org.apache.jmeter.protocol.http.sampler.HTTPSamplerBaseClassifier.java
/** * Download the resources of an HTML page. * /* w w w. java 2 s. co m*/ * @param res * result of the initial request - must contain an HTML response * @param container * for storing the results, if any * @param frameDepth * Depth of this target in the frame structure. Used only to * prevent infinite recursion. * @return res if no resources exist, otherwise the "Container" result with * one subsample per request issued */ protected HTTPSampleResult downloadPageResources(HTTPSampleResult res, HTTPSampleResult container, int frameDepth) { Iterator<URL> urls = null; try { final byte[] responseData = res.getResponseData(); if (responseData.length > 0) { // Bug 39205 String parserName = getParserClass(res); if (parserName != null) { final HTMLParser parser = parserName.length() > 0 ? // we // have // a // name HTMLParser.getParser(parserName) : HTMLParser.getParser(); // we don't; use the // default parser urls = parser.getEmbeddedResourceURLs(responseData, res.getURL(), res.getDataEncodingWithDefault()); } } } catch (HTMLParseException e) { // Don't break the world just because this failed: res.addSubResult(errorResult(e, new HTTPSampleResult(res))); setParentSampleSuccess(res, false); } // Iterate through the URLs and download each image: if (urls != null && urls.hasNext()) { if (container == null) { // TODO needed here because currently done on sample completion // in JMeterThread, // but that only catches top-level samples. res.setThreadName(Thread.currentThread().getName()); container = new HTTPSampleResult(res); container.addRawSubResult(res); } res = container; // Get the URL matcher String re = getEmbeddedUrlRE(); Perl5Matcher localMatcher = null; Pattern pattern = null; if (re.length() > 0) { try { pattern = JMeterUtils.getPattern(re); localMatcher = JMeterUtils.getMatcher();// don't fetch // unless pattern // compiles } catch (MalformedCachePatternException e) { log.warn("Ignoring embedded URL match string: " + e.getMessage()); } } // For concurrent get resources final List<Callable<AsynSamplerResultHolder>> liste = new ArrayList<Callable<AsynSamplerResultHolder>>(); while (urls.hasNext()) { Object binURL = urls.next(); // See catch clause below try { URL url = (URL) binURL; if (url == null) { log.warn("Null URL detected (should not happen)"); } else { String urlstr = url.toString(); String urlStrEnc = encodeSpaces(urlstr); if (!urlstr.equals(urlStrEnc)) {// There were some // spaces in the URL try { url = new URL(urlStrEnc); } catch (MalformedURLException e) { res.addSubResult(errorResult(new Exception(urlStrEnc + " is not a correct URI"), new HTTPSampleResult(res))); setParentSampleSuccess(res, false); continue; } } // I don't think localMatcher can be null here, but // check just in case if (pattern != null && localMatcher != null && !localMatcher.matches(urlStrEnc, pattern)) { continue; // we have a pattern and the URL does not // match, so skip it } if (isConcurrentDwn()) { // if concurrent download emb. resources, add to a // list for async gets later liste.add(new ASyncSample(url, HTTPConstants.GET, false, frameDepth + 1, getCookieManager(), this)); } else { // default: serial download embedded resources HTTPSampleResult binRes = sample(url, HTTPConstants.GET, false, frameDepth + 1); res.addSubResult(binRes); setParentSampleSuccess(res, res.isSuccessful() && binRes.isSuccessful()); } } } catch (ClassCastException e) { // TODO can this happen? res.addSubResult(errorResult(new Exception(binURL + " is not a correct URI"), new HTTPSampleResult(res))); setParentSampleSuccess(res, false); continue; } } // IF for download concurrent embedded resources if (isConcurrentDwn()) { int poolSize = CONCURRENT_POOL_SIZE; // init with default value try { poolSize = Integer.parseInt(getConcurrentPool()); } catch (NumberFormatException nfe) { log.warn("Concurrent download resources selected, "// $NON-NLS-1$ + "but pool size value is bad. Use default value");// $NON-NLS-1$ } // Thread pool Executor to get resources // use a LinkedBlockingQueue, note: max pool size doesn't effect final ThreadPoolExecutor exec = new ThreadPoolExecutor(poolSize, poolSize, KEEPALIVETIME, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new ThreadFactory() { public Thread newThread(final Runnable r) { Thread t = new CleanerThread(new Runnable() { public void run() { try { r.run(); } finally { ((CleanerThread) Thread.currentThread()).notifyThreadEnd(); } } }); return t; } }); boolean tasksCompleted = false; try { // sample all resources with threadpool final List<Future<AsynSamplerResultHolder>> retExec = exec.invokeAll(liste); // call normal shutdown (wait ending all tasks) exec.shutdown(); // put a timeout if tasks couldn't terminate exec.awaitTermination(AWAIT_TERMINATION_TIMEOUT, TimeUnit.SECONDS); CookieManager cookieManager = getCookieManager(); // add result to main sampleResult for (Future<AsynSamplerResultHolder> future : retExec) { AsynSamplerResultHolder binRes; try { binRes = future.get(1, TimeUnit.MILLISECONDS); if (cookieManager != null) { CollectionProperty cookies = binRes.getCookies(); PropertyIterator iter = cookies.iterator(); while (iter.hasNext()) { Cookie cookie = (Cookie) iter.next().getObjectValue(); cookieManager.add(cookie); } } res.addSubResult(binRes.getResult()); setParentSampleSuccess(res, res.isSuccessful() && binRes.getResult().isSuccessful()); } catch (TimeoutException e) { errorResult(e, res); } } tasksCompleted = exec.awaitTermination(1, TimeUnit.MILLISECONDS); // did all the tasks finish? } catch (InterruptedException ie) { log.warn("Interruped fetching embedded resources", ie); // $NON-NLS-1$ } catch (ExecutionException ee) { log.warn("Execution issue when fetching embedded resources", ee); // $NON-NLS-1$ } finally { if (!tasksCompleted) { exec.shutdownNow(); // kill any remaining tasks } } } } return res; }
From source file:org.opentripplanner.routing.algorithm.strategies.WeightTable.java
/** * Build the weight table, parallelized according to the number of processors */// ww w .ja v a2 s . com public void buildTable() { ArrayList<TransitStop> stopVertices; LOG.debug("Number of vertices: " + g.getVertices().size()); stopVertices = new ArrayList<TransitStop>(); for (Vertex gv : g.getVertices()) if (gv instanceof TransitStop) stopVertices.add((TransitStop) gv); int nStops = stopVertices.size(); stopIndices = new IdentityHashMap<Vertex, Integer>(nStops); for (int i = 0; i < nStops; i++) stopIndices.put(stopVertices.get(i), i); LOG.debug("Number of stops: " + nStops); table = new float[nStops][nStops]; for (float[] row : table) Arrays.fill(row, Float.POSITIVE_INFINITY); LOG.debug("Performing search at each transit stop."); int nThreads = Runtime.getRuntime().availableProcessors(); LOG.debug("number of threads: " + nThreads); ArrayBlockingQueue<Runnable> taskQueue = new ArrayBlockingQueue<Runnable>(nStops); ThreadPoolExecutor threadPool = new ThreadPoolExecutor(nThreads, nThreads, 10, TimeUnit.SECONDS, taskQueue); GenericObjectPool heapPool = new GenericObjectPool( new PoolableBinHeapFactory<State>(g.getVertices().size()), nThreads); // make one heap and recycle it RoutingRequest options = new RoutingRequest(); // TODO LG Check this change: options.setWalkSpeed(maxWalkSpeed); final double MAX_WEIGHT = 60 * 60 * options.walkReluctance; final double OPTIMISTIC_BOARD_COST = options.getBoardCostLowerBound(); // create a task for each transit stop in the graph ArrayList<Callable<Void>> tasks = new ArrayList<Callable<Void>>(); for (TransitStop origin : stopVertices) { SPTComputer task = new SPTComputer(heapPool, options, MAX_WEIGHT, OPTIMISTIC_BOARD_COST, origin); tasks.add(task); } try { //invoke all of tasks. threadPool.invokeAll(tasks); threadPool.shutdown(); } catch (InterruptedException e) { throw new RuntimeException(e); } floyd(); }