List of usage examples for java.util.concurrent ExecutorService shutdown
void shutdown();
From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepositoryTest.java
/** * Conflict cache creation./* w w w.j av a 2 s . c o m*/ * @throws Exception if failed */ @Test public void conflict() throws Exception { File source = folder.newFile(); byte[] bytes = new byte[1024 * 1024]; try (OutputStream output = new FileOutputStream(source)) { for (int i = 0, n = 50; i < n; i++) { output.write(bytes); } } Path path = path(source); File cacheRepo = folder.newFolder(); Configuration configuration = new ConfigurationProvider().newInstance(); LockProvider<Path> locks = new LocalFileLockProvider<>(folder.newFolder()); RetryStrategy retrier = new ConstantRetryStrategy(30, 100, 200); FileCacheRepository cache = new HadoopFileCacheRepository(configuration, path(cacheRepo), locks, retrier); List<Future<Path>> futures = new ArrayList<>(); int count = 10; CountDownLatch latch = new CountDownLatch(count); ExecutorService executor = Executors.newFixedThreadPool(count); try { for (int i = 0; i < count; i++) { String label = String.format("thread-%d", i); futures.add(executor.submit(() -> { LOG.info("Wait: resolve @" + label); latch.countDown(); if (latch.await(5, TimeUnit.SECONDS) == false) { throw new TimeoutException(); } LOG.info("Start: resolve @" + label); Path result = cache.resolve(path); LOG.info("Finish: resolve @" + label); return result; })); } executor.shutdown(); if (executor.awaitTermination(30, TimeUnit.SECONDS) == false) { throw new TimeoutException(); } } finally { executor.shutdownNow(); } for (Future<Path> future : futures) { future.get(); } }
From source file:com.linemetrics.monk.api.ApiClient.java
public List<DataItem> getRangeOptimized(final Number dataStreamId, long time_from, long time_to, TDB tdb, TimeZone tz) throws ApiException { try {/* w w w.ja v a2s . c o m*/ long timeDiff = time_to - time_from; long maxTimeRange = tdb.getQueryLimit(); long queryRange = tdb.getQueryRange(); if (timeDiff < maxTimeRange) { return this.getRange(dataStreamId, time_from, time_to, tdb, tz); } long millis = System.currentTimeMillis(); ExecutorService executorService = Executors.newSingleThreadExecutor(); Set<Future<List<DataItem>>> callables = new HashSet<Future<List<DataItem>>>(); long queryStart = time_from; long queryEnd = time_from + queryRange; while (queryStart < time_to) { callables.add(executorService .submit(new CallableRangeQuery(dataStreamId, queryStart, queryEnd, tdb, tz))); queryStart += queryRange; queryEnd += queryRange; } executorService.shutdown(); List<DataItem> list = new ArrayList<>(); for (Future<List<DataItem>> future : callables) { List<DataItem> slice = future.get(); if (slice == null) { throw new ApiException("Error while retrieving slice :("); } else { list.addAll(slice); } } executorService.awaitTermination(60 * 60 * 1000L, TimeUnit.MILLISECONDS); System.out.print("Optimized Range Query takes: " + (System.currentTimeMillis() - millis) + "ms "); // System.out.println(list.size()); // Collections.sort(list, DataItemComparator.getInstance()); DataItem prevItem = null, currItem; DataItem beginSlice = null; Iterator<DataItem> itemIterator = list.iterator(); while (itemIterator.hasNext()) { currItem = itemIterator.next(); if (prevItem != null) { if (prevItem.getTimestamp().equals(currItem.getTimestamp())) { itemIterator.remove(); continue; } if (beginSlice == null) { if (currItem.getTimestamp() - prevItem.getTimestamp() > tdb.getMilliseconds()) { beginSlice = prevItem; } } else { if (currItem.getTimestamp() - prevItem.getTimestamp() == tdb.getMilliseconds()) { System.out.println("TimeRange " + beginSlice.getTimestamp() + " - " + prevItem.getTimestamp() + " " + (prevItem.getTimestamp() - beginSlice.getTimestamp()) + " ms missing!"); beginSlice = null; } } } prevItem = currItem; } if (beginSlice != null) { System.out.println("TimeRange " + beginSlice.getTimestamp() + " - " + prevItem.getTimestamp() + " " + (prevItem.getTimestamp() - beginSlice.getTimestamp()) + " ms missing!"); } long expectedItems = ((time_to - time_from) / tdb.getMilliseconds()) - 1; System.out.println(" (" + (list.size() - expectedItems) + ")"); return list; } catch (Exception e) { throw new ApiException(e.getMessage()); } }
From source file:net.jotel.ws.client.WebSocketClientTest.java
@Test public void reconnect() throws Exception { final List<Exception> exceptions = new ArrayList<Exception>(); URI uri = new URI("ws://not-existing-domain-name:8080/websocket/ws/subscribe"); final WebSocketClient c = new WebSocketClient(); c.setWebSocketUri(uri);/* w w w . ja va2 s . com*/ c.setReconnectEnabled(true); c.setReconnectInterval(100L); c.setReconnectAttempts(2); c.addListener(new WebSocketListener() { @Override public void onMessage(String message) { } @Override public void onMessage(byte[] message) { } @Override public void onError(Exception ex) { exceptions.add(ex); } @Override public void onClose(Integer statusCode, String message) { // TODO Auto-generated method stub } @Override public void onConnect() { // TODO Auto-generated method stub } }); try { c.connect(); fail("Expected WebSocketException"); } catch (WebSocketException ex) { // expected assertEquals(3, exceptions.size()); for (Exception e : exceptions) { Throwable rootCause = ExceptionUtils.getRootCause(e); if (rootCause == null) { rootCause = e; } assertTrue(rootCause instanceof UnknownHostException); } } exceptions.clear(); c.setReconnectAttempts(0); try { c.connect(); fail("Expected WebSocketException"); } catch (WebSocketException ex) { // expected assertEquals(1, exceptions.size()); for (Exception e : exceptions) { Throwable rootCause = ExceptionUtils.getRootCause(e); if (rootCause == null) { rootCause = e; } assertTrue(rootCause instanceof UnknownHostException); } } exceptions.clear(); c.setReconnectAttempts(-1); ExecutorService executor = Executors.newSingleThreadExecutor(); Future<?> future = executor.submit(new Runnable() { @Override public void run() { try { c.connect(); fail("Expected WebSocketException"); } catch (WebSocketException ex) { throw new UnhandledException(ex); } } }); Thread.sleep(2000L); c.setReconnectEnabled(false); Thread.sleep(2000L); executor.shutdown(); assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); try { future.get(); fail("Expected WebSocketException"); } catch (Exception ex) { // expected assertTrue(exceptions.size() > 1); for (Exception e : exceptions) { Throwable rootCause = ExceptionUtils.getRootCause(e); if (rootCause == null) { rootCause = e; } assertTrue(rootCause instanceof UnknownHostException); } } }
From source file:com.qwazr.library.test.CassandraTest.java
@Test public void test_10_transaction() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(10); try {// w w w . j av a2 s. c o m cassandra.execute("SELECT count(*) FROM qwazr_connector_test.test").all(); finalTime = System.currentTimeMillis() + 10000; List<ProcedureExceptionCatcher> threadList = new ArrayList<ProcedureExceptionCatcher>(); for (int i = 0; i < 50; i++) { threadList.add(new InsertThread()); threadList.add(new SelectUpdateThread()); } ThreadUtils.invokeAndJoin(executor, threadList); for (CallableExceptionCatcher<?> callable : threadList) callable.checkException(); } catch (NoHostAvailableException e) { logger.warning("Bypass (no cassandra host is running)"); } finally { executor.shutdown(); } }
From source file:com.thoughtworks.go.server.security.LdapAuthenticationTest.java
@Test public void shouldAuthenticateConcurrently() throws Exception { ldapServer.addUser(employeesOrgUnit, "foleys", "some-password", "Shilpa Foley", "foleys@somecompany.com"); ExecutorService pool = Executors.newFixedThreadPool(100); List<Callable<String>> allCallables = new ArrayList<Callable<String>>(); for (int i = 0; i < 100; i++) { final boolean even = i % 2 == 0; allCallables.add(new Callable<String>() { @Override// w w w . ja va 2 s . c om public String call() throws Exception { if (even) { assertAuthenticationOfValidAdminUser("foleys", "some-password"); } else { assertFailedAuthentication("invalid_user", ""); } return ""; } }); } List<Future<String>> futures = pool.invokeAll(allCallables); pool.shutdown(); boolean finishedWithoutTimeout = pool.awaitTermination(10, TimeUnit.SECONDS); assertThat(finishedWithoutTimeout, is(true)); // Assert no exceptions, by getting result. for (Future<String> future : futures) { future.get(); } }
From source file:ok.MyService2.java
@Override protected Task<BlockingQueue> createTask() { final Task<BlockingQueue> task; task = new Task<BlockingQueue>() { @Override/*from w w w .j a va 2s . c om*/ protected BlockingQueue call() throws Exception { BlockingQueue result = new LinkedBlockingQueue<String>(); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); cm.setMaxTotal(100); CloseableHttpClient httpclient = HttpClients.custom().setConnectionManager(cm).build(); try { ExecutorService executor = Executors.newFixedThreadPool(sites.size()); List<Future<String>> results = new ArrayList<Future<String>>(); for (int i = 0; i < sites.size(); i++) { HttpGet httpget = new HttpGet(sites.get(i)); Callable worker = new MyCallable(httpclient, httpget); Future<String> res = executor.submit(worker); results.add(res); // String url = hostList[i]; // Runnable worker = new MyRunnable(url); // executor.execute(worker); // executor.submit(null); } executor.shutdown(); // Wait until all threads are finish // while (!executor.isTerminated()) { // // } for (Future<String> element : results) { result.add(element.get()); } System.out.println("\nFinished all threads"); } finally { httpclient.close(); } return result; } }; return task; }
From source file:com.cloud.hypervisor.kvm.resource.wrapper.LibvirtMigrateCommandWrapper.java
@Override public Answer execute(final MigrateCommand command, final LibvirtComputingResource libvirtComputingResource) { final String vmName = command.getVmName(); String result = null;/*from w ww .j a v a2 s .c o m*/ List<InterfaceDef> ifaces = null; List<DiskDef> disks; Domain dm = null; Connect dconn = null; Domain destDomain = null; Connect conn = null; String xmlDesc = null; List<Ternary<String, Boolean, String>> vmsnapshots = null; try { final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource .getLibvirtUtilitiesHelper(); conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName); ifaces = libvirtComputingResource.getInterfaces(conn, vmName); disks = libvirtComputingResource.getDisks(conn, vmName); dm = conn.domainLookupByName(vmName); /* We replace the private IP address with the address of the destination host. This is because the VNC listens on the private IP address of the hypervisor, but that address is of course different on the target host. MigrateCommand.getDestinationIp() returns the private IP address of the target hypervisor. So it's safe to use. The Domain.migrate method from libvirt supports passing a different XML description for the instance to be used on the target host. This is supported by libvirt-java from version 0.50.0 CVE-2015-3252: Get XML with sensitive information suitable for migration by using VIR_DOMAIN_XML_MIGRATABLE flag (value = 8) https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainXMLFlags Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0. */ final int xmlFlag = conn.getLibVirVersion() >= 1000000 ? 8 : 1; // 1000000 equals v1.0.0 final String target = command.getDestinationIp(); xmlDesc = dm.getXMLDesc(xmlFlag); xmlDesc = replaceIpForVNCInDescFile(xmlDesc, target); // delete the metadata of vm snapshots before migration vmsnapshots = libvirtComputingResource.cleanVMSnapshotMetadata(dm); Map<String, MigrateCommand.MigrateDiskInfo> mapMigrateStorage = command.getMigrateStorage(); // migrateStorage is declared as final because the replaceStorage method may mutate mapMigrateStorage, but // migrateStorage's value should always only be associated with the initial state of mapMigrateStorage. final boolean migrateStorage = MapUtils.isNotEmpty(mapMigrateStorage); if (migrateStorage) { xmlDesc = replaceStorage(xmlDesc, mapMigrateStorage); } dconn = libvirtUtilitiesHelper .retrieveQemuConnection("qemu+tcp://" + command.getDestinationIp() + "/system"); //run migration in thread so we can monitor it s_logger.info("Live migration of instance " + vmName + " initiated"); final ExecutorService executor = Executors.newFixedThreadPool(1); final Callable<Domain> worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc, migrateStorage, command.isAutoConvergence(), vmName, command.getDestinationIp()); final Future<Domain> migrateThread = executor.submit(worker); executor.shutdown(); long sleeptime = 0; while (!executor.isTerminated()) { Thread.sleep(100); sleeptime += 100; if (sleeptime == 1000) { // wait 1s before attempting to set downtime on migration, since I don't know of a VIR_DOMAIN_MIGRATING state final int migrateDowntime = libvirtComputingResource.getMigrateDowntime(); if (migrateDowntime > 0) { try { final int setDowntime = dm.migrateSetMaxDowntime(migrateDowntime); if (setDowntime == 0) { s_logger.debug("Set max downtime for migration of " + vmName + " to " + String.valueOf(migrateDowntime) + "ms"); } } catch (final LibvirtException e) { s_logger.debug( "Failed to set max downtime for migration, perhaps migration completed? Error: " + e.getMessage()); } } } if (sleeptime % 1000 == 0) { s_logger.info( "Waiting for migration of " + vmName + " to complete, waited " + sleeptime + "ms"); } // pause vm if we meet the vm.migrate.pauseafter threshold and not already paused final int migratePauseAfter = libvirtComputingResource.getMigratePauseAfter(); if (migratePauseAfter > 0 && sleeptime > migratePauseAfter) { DomainState state = null; try { state = dm.getInfo().state; } catch (final LibvirtException e) { s_logger.info("Couldn't get VM domain state after " + sleeptime + "ms: " + e.getMessage()); } if (state != null && state == DomainState.VIR_DOMAIN_RUNNING) { try { s_logger.info( "Pausing VM " + vmName + " due to property vm.migrate.pauseafter setting to " + migratePauseAfter + "ms to complete migration"); dm.suspend(); } catch (final LibvirtException e) { // pause could be racy if it attempts to pause right when vm is finished, simply warn s_logger.info("Failed to pause vm " + vmName + " : " + e.getMessage()); } } } } s_logger.info("Migration thread for " + vmName + " is done"); destDomain = migrateThread.get(10, TimeUnit.SECONDS); if (destDomain != null) { for (final DiskDef disk : disks) { libvirtComputingResource.cleanupDisk(disk); } } } catch (final LibvirtException e) { s_logger.debug("Can't migrate domain: " + e.getMessage()); result = e.getMessage(); } catch (final InterruptedException e) { s_logger.debug("Interrupted while migrating domain: " + e.getMessage()); result = e.getMessage(); } catch (final ExecutionException e) { s_logger.debug("Failed to execute while migrating domain: " + e.getMessage()); result = e.getMessage(); } catch (final TimeoutException e) { s_logger.debug("Timed out while migrating domain: " + e.getMessage()); result = e.getMessage(); } catch (final IOException e) { s_logger.debug("IOException: " + e.getMessage()); result = e.getMessage(); } catch (final ParserConfigurationException e) { s_logger.debug("ParserConfigurationException: " + e.getMessage()); result = e.getMessage(); } catch (final SAXException e) { s_logger.debug("SAXException: " + e.getMessage()); result = e.getMessage(); } catch (final TransformerConfigurationException e) { s_logger.debug("TransformerConfigurationException: " + e.getMessage()); result = e.getMessage(); } catch (final TransformerException e) { s_logger.debug("TransformerException: " + e.getMessage()); result = e.getMessage(); } finally { try { if (dm != null && result != null) { // restore vm snapshots in case of failed migration if (vmsnapshots != null) { libvirtComputingResource.restoreVMSnapshotMetadata(dm, vmName, vmsnapshots); } } if (dm != null) { if (dm.isPersistent() == 1) { dm.undefine(); } dm.free(); } if (dconn != null) { dconn.close(); } if (destDomain != null) { destDomain.free(); } } catch (final LibvirtException e) { s_logger.trace("Ignoring libvirt error.", e); } } if (result != null) { } else { libvirtComputingResource.destroyNetworkRulesForVM(conn, vmName); for (final InterfaceDef iface : ifaces) { // We don't know which "traffic type" is associated with // each interface at this point, so inform all vif drivers final List<VifDriver> allVifDrivers = libvirtComputingResource.getAllVifDrivers(); for (final VifDriver vifDriver : allVifDrivers) { vifDriver.unplug(iface); } } } return new MigrateAnswer(command, result == null, result, null); }
From source file:net.arp7.HdfsPerfTest.WriteFile.java
private static void writeFiles(final Configuration conf, final FileIoStats stats) throws InterruptedException, IOException { final FileSystem fs = FileSystem.get(conf); final AtomicLong filesLeft = new AtomicLong(params.getNumFiles()); final long runId = abs(rand.nextLong()); final byte[] data = new byte[params.getIoSize()]; Arrays.fill(data, (byte) 65); // Start the writers. final ExecutorService executor = Executors.newFixedThreadPool((int) params.getNumThreads()); final CompletionService<Object> ecs = new ExecutorCompletionService<>(executor); LOG.info("NumFiles=" + params.getNumFiles() + ", FileSize=" + FileUtils.byteCountToDisplaySize(params.getFileSize()) + ", IoSize=" + FileUtils.byteCountToDisplaySize(params.getIoSize()) + ", BlockSize=" + FileUtils.byteCountToDisplaySize(params.getBlockSize()) + ", ReplicationFactor=" + params.getReplication() + ", isThrottled=" + (params.maxWriteBps() > 0)); LOG.info("Starting " + params.getNumThreads() + " writer thread" + (params.getNumThreads() > 1 ? "s" : "") + "."); final long startTime = System.nanoTime(); for (long t = 0; t < params.getNumThreads(); ++t) { final long threadIndex = t; Callable<Object> c = new Callable<Object>() { @Override/* w ww . j a va2 s. c o m*/ public Object call() throws Exception { long fileIndex = 0; while (filesLeft.addAndGet(-1) >= 0) { final String fileName = "WriteFile-" + runId + "-" + (threadIndex + 1) + "-" + (++fileIndex); writeOneFile(new Path(params.getOutputDir(), fileName), fs, data, stats); } return null; } }; ecs.submit(c); } // And wait for all writers to complete. for (long t = 0; t < params.getNumThreads(); ++t) { ecs.take(); } final long endTime = System.nanoTime(); stats.setElapsedTime(endTime - startTime); executor.shutdown(); }
From source file:com.blacklocus.jres.request.index.JresUpdateDocumentTest.java
@Test public void testRetryOnConflict() throws InterruptedException { final String index = "JresUpdateDocumentTest.testRetryOnConflict".toLowerCase(); final String type = "test"; final String id = "warzone"; final AtomicReference<String> error = new AtomicReference<String>(); final int numThreads = 16, numIterations = 100; ExecutorService x = Executors.newFixedThreadPool(numThreads); for (int i = 0; i < numThreads; i++) { x.submit(new Runnable() { @Override//from www .j a va 2 s . c o m public void run() { try { for (int j = 0; j < numIterations; j++) { JresUpdateDocument req = new JresUpdateDocument(index, type, id, ImmutableMap.of("value", 0)); req.setRetryOnConflict(numIterations * 10); jres.quest(req); } } catch (Exception e) { error.set(e.getMessage()); } } }); } x.shutdown(); x.awaitTermination(1, TimeUnit.MINUTES); Assert.assertNull("With so many retries, all of these should have gotten through without conflict error", error.get()); jres.quest(new JresRefresh(index)); JresGetDocumentReply getReply = jres.quest(new JresGetDocument(index, type, id)); Map<String, Integer> doc = getReply.getSourceAsType(new TypeReference<Map<String, Integer>>() { }); Assert.assertEquals("Should have been numThreads * numIterations versions committed", (Object) (numThreads * numIterations), getReply.getVersion()); }
From source file:com.concursive.connect.web.modules.common.social.images.jobs.ImageResizerJob.java
public void execute(JobExecutionContext context) throws JobExecutionException { LOG.debug("Starting..."); SchedulerContext schedulerContext = null; Connection db = null;/* w w w . ja v a 2 s. c o m*/ // Initial setup try { schedulerContext = context.getScheduler().getContext(); } catch (Exception e) { LOG.error("ImageResizerJob Exception due to scheduler", e); throw new JobExecutionException(e); } // Process the arrays Vector exportList = (Vector) schedulerContext.get(IMAGE_RESIZER_ARRAY); while (exportList.size() > 0) { // Holds the transactions to be threaded List<TransactionTask> renderTasks = new ArrayList<TransactionTask>(); // Pre-process the files using a database connection try { db = SchedulerUtils.getConnection(schedulerContext); // The imageResizerBean contains the image handle to be processed ImageResizerBean bean = (ImageResizerBean) exportList.remove(0); LOG.debug("Preparing thumbnails for FileItem (" + bean.getFileItemId() + ")... " + bean.getWidth() + "x" + bean.getHeight()); // Load the fileItem FileItem fileItem = new FileItem(db, bean.getFileItemId()); if (bean.getWidth() > 0 || bean.getHeight() > 0) { // A specific size needs to be rendered renderTasks.add(new TransactionTask(bean, fileItem, bean.getWidth(), bean.getHeight(), false)); } else { // No specific size so for each fileItem, generate several sizes of the image renderTasks.add(new TransactionTask(bean, fileItem, 640, 480, false)); renderTasks.add(new TransactionTask(bean, fileItem, 210, 150, false)); renderTasks.add(new TransactionTask(bean, fileItem, 200, 200, false)); renderTasks.add(new TransactionTask(bean, fileItem, 133, 133, true)); renderTasks.add(new TransactionTask(bean, fileItem, 100, 100, false)); renderTasks.add(new TransactionTask(bean, fileItem, 75, 75, false)); renderTasks.add(new TransactionTask(bean, fileItem, 50, 50, false)); renderTasks.add(new TransactionTask(bean, fileItem, 45, 45, false)); renderTasks.add(new TransactionTask(bean, fileItem, 30, 30, false)); } } catch (Exception e) { LOG.error("ImageResizerJob Exception", e); continue; } finally { SchedulerUtils.freeConnection(schedulerContext, db); } int threads = 2; // Process the files ExecutorService executor = null; List<Future<Thumbnail>> futures = null; try { executor = Executors.newFixedThreadPool(threads); // NOTE: this wrapper fix is for Java 1.5 final Collection<Callable<Thumbnail>> wrapper = Collections .<Callable<Thumbnail>>unmodifiableCollection(renderTasks); LOG.debug("Generating thumbnails... " + renderTasks.size()); futures = executor.invokeAll(wrapper); } catch (InterruptedException e) { LOG.error("ImageResizerJob executor exception", e); if (executor != null) { executor.shutdown(); } throw new JobExecutionException(e); } // Insert the thumbnails using the database connection try { db = SchedulerUtils.getConnection(schedulerContext); LOG.debug("Inserting thumbnails into database... " + futures.size()); // Process the executor results for (Future<Thumbnail> f : futures) { Thumbnail thumbnail = f.get(); thumbnail.insert(db); } } catch (Exception e) { LOG.error("ImageResizerJob insert thumbnails into database exception", e); throw new JobExecutionException(e); } finally { SchedulerUtils.freeConnection(schedulerContext, db); if (executor != null) { executor.shutdown(); } } } }