List of usage examples for com.google.common.util.concurrent ListeningExecutorService submit
@Override ListenableFuture<?> submit(Runnable task);
From source file:com.facebook.buck.distributed.DistBuildService.java
public ListenableFuture<Void> uploadTargetGraph(final BuildJobState buildJobStateArg, final BuildId buildId, ListeningExecutorService executorService) { // TODO(shivanker): We shouldn't be doing this. Fix after we stop reading all files into memory. final BuildJobState buildJobState = buildJobStateArg.deepCopy(); return executorService.submit(new Callable<Void>() { @Override//from w w w . j a v a 2s . c o m public Void call() throws IOException { // Get rid of file contents from buildJobState for (BuildJobStateFileHashes cell : buildJobState.getFileHashes()) { if (!cell.isSetEntries()) { continue; } for (BuildJobStateFileHashEntry file : cell.getEntries()) { file.unsetContents(); } } // Now serialize and send the whole buildJobState StoreBuildGraphRequest storeBuildGraphRequest = new StoreBuildGraphRequest(); storeBuildGraphRequest.setBuildId(buildId); storeBuildGraphRequest.setBuildGraph(BuildJobStateSerializer.serialize(buildJobState)); FrontendRequest request = new FrontendRequest(); request.setType(FrontendRequestType.STORE_BUILD_GRAPH); request.setStoreBuildGraphRequest(storeBuildGraphRequest); makeRequestChecked(request); // No response expected. return null; } }); }
From source file:co.cask.cdap.internal.app.deploy.pipeline.ProgramGenerationStage.java
@Override public void process(final ApplicationDeployable input) throws Exception { ImmutableList.Builder<Program> programs = ImmutableList.builder(); final ApplicationSpecification appSpec = input.getSpecification(); final String applicationName = appSpec.getName(); final ArchiveBundler bundler = new ArchiveBundler(input.getLocation()); // Make sure the namespace directory exists Id.Namespace namespaceId = input.getId().getNamespace(); Location namespacedLocation = namespacedLocationFactory.get(namespaceId); // Note: deployApplication/deployAdapters have already checked for namespaceDir existence, so not checking again // Make sure we have a directory to store the original artifact. final Location appFabricDir = namespacedLocation.append(configuration.get(Constants.AppFabric.OUTPUT_DIR)); // Check exists, create, check exists again to avoid failure due to race condition. if (!appFabricDir.exists() && !appFabricDir.mkdirs() && !appFabricDir.exists()) { throw new IOException(String.format("Failed to create directory %s", appFabricDir.toURI().getPath())); }//from www . j a v a 2s. c o m // Now, we iterate through all ProgramSpecification and generate programs Iterable<ProgramSpecification> specifications = Iterables.concat(appSpec.getMapReduce().values(), appSpec.getFlows().values(), appSpec.getWorkflows().values(), appSpec.getServices().values(), appSpec.getSpark().values(), appSpec.getWorkers().values()); // Generate webapp program if required Set<String> servingHostNames = WebappProgramRunner .getServingHostNames(Locations.newInputSupplier(input.getLocation())); if (!servingHostNames.isEmpty()) { specifications = Iterables.concat(specifications, ImmutableList.of(createWebappSpec(ProgramType.WEBAPP.toString().toLowerCase()))); } ListeningExecutorService executorService = MoreExecutors.listeningDecorator( Executors.newFixedThreadPool(10, Threads.createDaemonThreadFactory("program-gen-%d"))); try { List<ListenableFuture<Location>> futures = Lists.newArrayList(); for (final ProgramSpecification spec : specifications) { ListenableFuture<Location> future = executorService.submit(new Callable<Location>() { @Override public Location call() throws Exception { ProgramType type = ProgramTypes.fromSpecification(spec); String name = String.format(Locale.ENGLISH, "%s/%s", applicationName, type); Location programDir = appFabricDir.append(name); if (!programDir.exists()) { programDir.mkdirs(); } Location output = programDir.append(String.format("%s.jar", spec.getName())); Id.Program programId = Id.Program.from(input.getId(), type, spec.getName()); return ProgramBundle.create(programId, bundler, output, spec.getClassName(), appSpec); } }); futures.add(future); } for (Location jarLocation : Futures.allAsList(futures).get()) { programs.add(Programs.create(jarLocation, null)); } } finally { executorService.shutdown(); } // moves the <appfabricdir>/archive/<app-name>.jar to <appfabricdir>/<app-name>/archive/<app-name>.jar // Cannot do this before starting the deploy pipeline because appId could be null at that time. // However, it is guaranteed to be non-null from VerificationsStage onwards Location newArchiveLocation = appFabricDir.append(applicationName).append(Constants.ARCHIVE_DIR); moveAppArchiveUnderAppDirectory(input.getLocation(), newArchiveLocation); Location programLocation = newArchiveLocation.append(input.getLocation().getName()); ApplicationDeployable updatedAppDeployable = new ApplicationDeployable(input.getId(), input.getSpecification(), input.getExistingAppSpec(), input.getApplicationDeployScope(), programLocation); // Emits the received specification with programs. emit(new ApplicationWithPrograms(updatedAppDeployable, programs.build())); }
From source file:org.n52.lod.triplestore.AbstractWorkerTripleSink.java
@Override protected int addRecordsToModel(Map<String, GetRecordByIdResponseDocument> records, final Model m, final Report report) { final MutableInt counter = new MutableInt(0); final long modelSizeBefore = m.size(); ListeningExecutorService executorService = MoreExecutors .listeningDecorator(Executors.newFixedThreadPool(NUM_THREADS)); for (Entry<String, GetRecordByIdResponseDocument> entry : records.entrySet()) { final String id = entry.getKey(); log.debug("Adding {} to the model", id); CallableMapper c = new CallableMapper(this.mapper.replicate(), entry.getValue()); ListenableFuture<Model> modelFuture = executorService.submit(c); Futures.addCallback(modelFuture, new FutureCallback<Model>() { @Override/*from ww w . ja va2 s. co m*/ public void onFailure(Throwable t) { log.error("Error mapping xml to model", t); report.issues.put(id, "Error while adding to model: " + t.getMessage()); } @Override public void onSuccess(Model result) { log.trace("Adding result to model: {}", result); m.add(result); log.trace("ADDED result to mode."); counter.increment(); report.added++; report.addedIds.add(id); } }); } executorService.shutdown(); while (!executorService.isTerminated()) { try { executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e) { log.error("Could not await termination", e); } } log.debug("Added {} of {} records to model {}, which now has size {} (before {})", counter, records.size(), m.getClass(), m.size(), modelSizeBefore); return counter.intValue(); }
From source file:com.cloudera.oryx.kmeans.computation.local.ClusteringEvaluation.java
@Override public List<KMeansEvaluationData> call() throws InterruptedException, ExecutionException { Config config = ConfigUtils.getDefaultConfig(); EvaluationSettings settings = EvaluationSettings.create(config); ListeningExecutorService exec = MoreExecutors .listeningDecorator(Executors.newFixedThreadPool(config.getInt("model.parallelism"), new ThreadFactoryBuilder().setNameFormat("KMEANS-%d").setDaemon(true).build())); List<ListenableFuture<KMeansEvaluationData>> futures = Lists.newArrayList(); for (Integer nc : settings.getKValues()) { int loops = nc == 1 ? 1 : settings.getReplications(); for (int i = 0; i < loops; i++) { futures.add(exec.submit(new EvaluationRun(weightedPoints, nc, i, settings))); }//from w w w. j a v a 2 s . co m } try { List<KMeansEvaluationData> evalData = Futures.allAsList(futures).get(); KMeansEvalStrategy evalStrategy = settings.getEvalStrategy(); if (evalStrategy != null) { List<ClusterValidityStatistics> best = evalStrategy.evaluate( Lists.transform(evalData, new Function<KMeansEvaluationData, ClusterValidityStatistics>() { @Override public ClusterValidityStatistics apply(KMeansEvaluationData input) { return input.getClusterValidityStatistics(); } })); if (best.size() == 1) { ClusterValidityStatistics cvs = best.get(0); for (KMeansEvaluationData ed : evalData) { if (cvs.getK() == ed.getK() && cvs.getReplica() == ed.getReplica()) { return ImmutableList.of(ed); } } } } return evalData; } finally { ExecutorUtils.shutdownAndAwait(exec); } }
From source file:qa.qcri.nadeef.core.pipeline.Iterator.java
/** * Iterator operator execution./* www . j a v a 2 s .com*/ * * @param blocks input tables. * @return iteration output. */ @Override @SuppressWarnings("unchecked") public Boolean execute(Collection<Table> blocks) { Tracer tracer = Tracer.getTracer(Iterator.class); ThreadFactory factory = new ThreadFactoryBuilder().setNameFormat("iterator-pool-%d").build(); ExecutorService executor = Executors.newFixedThreadPool(MAX_THREAD_NUM, factory); ListeningExecutorService service = MoreExecutors.listeningDecorator(executor); Stopwatch stopwatch = Stopwatch.createStarted(); blockCount = 0; ExecutionContext context = getCurrentContext(); Rule rule = context.getRule(); try { if (rule.supportTwoTables()) { // Rule runs on two tables. ListenableFuture<Integer> future = service .submit(new IteratorCallable(blocks, rule, context.getNewTuples())); blockCount++; setPercentage(1.0f); } else { // Rule runs on each table. for (Table table : blocks) { ListenableFuture<Integer> future = service .submit(new IteratorCallable(table, rule, context.getNewTuples())); Futures.addCallback(future, new IteratorCallback(blocks.size())); } } // wait until all the tasks are finished service.shutdown(); while (!service.awaitTermination(10l, TimeUnit.MINUTES)) ; // recycle the collection when dealing with pairs. This is mainly used to remove refs. if (rule instanceof PairTupleRule) { for (Table block : blocks) { block.recycle(); } } // mark the end of the iteration output IteratorBlockingQueue.markEnd(); } catch (InterruptedException ex) { tracer.err("Iterator is interrupted.", ex); } finally { executor.shutdown(); } PerfReport.appendMetric(PerfReport.Metric.IteratorTime, stopwatch.elapsed(TimeUnit.MILLISECONDS)); stopwatch.stop(); return true; }
From source file:com.skcraft.launcher.install.HttpDownloader.java
/** * Prevent further downloads from being queued and download queued files. * * @throws InterruptedException thrown on interruption * @throws IOException thrown on I/O error *///from www .j a v a2s . c o m public void execute() throws InterruptedException, IOException { synchronized (this) { queue = Collections.unmodifiableList(queue); } ListeningExecutorService executor = MoreExecutors .listeningDecorator(Executors.newFixedThreadPool(threadCount)); try { List<ListenableFuture<?>> futures = new ArrayList<ListenableFuture<?>>(); synchronized (this) { for (HttpDownloadJob job : queue) { futures.add(executor.submit(job)); } } try { Futures.allAsList(futures).get(); } catch (ExecutionException e) { throw new IOException("Something went wrong", e); } synchronized (this) { if (failed.size() > 0) { throw new IOException(failed.size() + " file(s) could not be downloaded"); } } } finally { executor.shutdownNow(); } }
From source file:com.google.cloud.dataflow.sdk.io.FileBasedSource.java
private ListenableFuture<List<? extends FileBasedSource<T>>> createFutureForFileSplit(final String file, final long desiredBundleSizeBytes, final PipelineOptions options, ListeningExecutorService service) { return service.submit(new Callable<List<? extends FileBasedSource<T>>>() { @Override//from w ww. j ava 2 s . c o m public List<? extends FileBasedSource<T>> call() throws Exception { return createForSubrangeOfFile(file, 0, Long.MAX_VALUE).splitIntoBundles(desiredBundleSizeBytes, options); } }); }
From source file:org.apache.qpid.server.security.SiteSpecificTrustStoreImpl.java
private ListenableFuture<X509Certificate> downloadCertificate(final String url) { final ListeningExecutorService workerService = MoreExecutors.listeningDecorator( Executors.newSingleThreadExecutor(getThreadFactory("download-certificate-worker-" + getName()))); try {//from w ww . ja v a2s .c o m return workerService.submit(new Callable<X509Certificate>() { @Override public X509Certificate call() { try { final URL siteUrl = new URL(url); final int port = siteUrl.getPort() == -1 ? siteUrl.getDefaultPort() : siteUrl.getPort(); SSLContext sslContext = SSLUtil.tryGetSSLContext(); sslContext.init(new KeyManager[0], new TrustManager[] { new AlwaysTrustManager() }, null); try (SSLSocket socket = (SSLSocket) sslContext.getSocketFactory().createSocket()) { socket.setSoTimeout(_readTimeout); socket.connect(new InetSocketAddress(siteUrl.getHost(), port), _connectTimeout); socket.startHandshake(); final Certificate[] certificateChain = socket.getSession().getPeerCertificates(); if (certificateChain != null && certificateChain.length != 0 && certificateChain[0] instanceof X509Certificate) { final X509Certificate x509Certificate = (X509Certificate) certificateChain[0]; LOGGER.debug( "Successfully downloaded X509Certificate with DN {} certificate from {}", x509Certificate.getSubjectDN(), url); return x509Certificate; } else { throw new IllegalConfigurationException(String.format( "TLS handshake for '%s' from '%s' " + "did not provide a X509Certificate", getName(), url)); } } } catch (IOException | GeneralSecurityException e) { throw new IllegalConfigurationException( String.format("Unable to get certificate for '%s' from '%s'", getName(), url), e); } } }); } finally { workerService.shutdown(); } }
From source file:org.apache.crunch.io.impl.FileTargetImpl.java
@Override public void handleOutputs(Configuration conf, Path workingPath, int index) throws IOException { FileSystem srcFs = workingPath.getFileSystem(conf); Path src = getSourcePattern(workingPath, index); Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(src), src); FileSystem dstFs = path.getFileSystem(conf); if (!dstFs.exists(path)) { dstFs.mkdirs(path);/* w ww . j av a 2 s. c o m*/ } boolean sameFs = isCompatible(srcFs, path); List<ListenableFuture<Boolean>> renameFutures = Lists.newArrayList(); ListeningExecutorService executorService = MoreExecutors.listeningDecorator( Executors.newFixedThreadPool(conf.getInt(RuntimeParameters.FILE_TARGET_MAX_THREADS, 1))); for (Path s : srcs) { Path d = getDestFile(conf, s, path, s.getName().contains("-m-")); renameFutures.add(executorService.submit(new WorkingPathFileMover(conf, s, d, srcFs, dstFs, sameFs))); } LOG.debug("Renaming " + renameFutures.size() + " files."); ListenableFuture<List<Boolean>> future = Futures.successfulAsList(renameFutures); List<Boolean> renameResults = null; try { renameResults = future.get(); } catch (InterruptedException | ExecutionException e) { Throwables.propagate(e); } finally { executorService.shutdownNow(); } if (renameResults != null && !renameResults.contains(false)) { dstFs.create(getSuccessIndicator(), true).close(); LOG.debug("Renamed " + renameFutures.size() + " files."); } }
From source file:dk.ilios.spanner.internal.ExperimentingSpannerRun.java
/** * Schedule all the trials.//www .j a va 2 s. c o m * <p> * <p>This method arranges all the {@link ScheduledTrial trials} to run according to their * scheduling criteria. The executor instance is responsible for enforcing max parallelism. */ private List<ListenableFuture<Trial.Result>> scheduleTrials(List<ScheduledTrial> trials, final ListeningExecutorService executor) { List<ListenableFuture<Trial.Result>> pendingTrials = Lists.newArrayList(); List<ScheduledTrial> serialTrials = Lists.newArrayList(); for (final ScheduledTrial scheduledTrial : trials) { if (scheduledTrial.policy() == TrialSchedulingPolicy.PARALLEL) { pendingTrials.add(executor.submit(scheduledTrial.trialTask())); } else { serialTrials.add(scheduledTrial); } } // A future representing the completion of all prior tasks. Futures.successfulAsList allows us // to ignore failure. ListenableFuture<?> previous = Futures.successfulAsList(pendingTrials); for (final ScheduledTrial scheduledTrial : serialTrials) { // each of these trials can only start after all prior trials have finished, so we use // Futures.transform to force the sequencing. ListenableFuture<Trial.Result> current = Futures.transform(previous, new AsyncFunction<Object, Trial.Result>() { @Override public ListenableFuture<Trial.Result> apply(Object ignored) { return executor.submit(scheduledTrial.trialTask()); } }); pendingTrials.add(current); // ignore failure of the prior task. previous = Futures.withFallback(current, FALLBACK_TO_NULL); } return pendingTrials; }