List of usage examples for com.google.common.util.concurrent MoreExecutors listeningDecorator
@GwtIncompatible("TODO") public static ListeningScheduledExecutorService listeningDecorator(ScheduledExecutorService delegate)
From source file:org.apache.hadoop.hive.llap.AsyncPbRpcProxy.java
public AsyncPbRpcProxy(String name, int numThreads, Configuration conf, Token<TokenType> token, long connectionTimeoutMs, long retrySleepMs, int expectedNodes, int maxPerNode) { super(name);/*from w w w. j ava2 s . c o m*/ // Note: we may make size/etc. configurable later. CacheBuilder<String, ProtocolType> cb = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.HOURS) .removalListener(new RemovalListener<String, ProtocolType>() { @Override public void onRemoval(RemovalNotification<String, ProtocolType> arg) { if (arg == null) return; shutdownProtocolImpl(arg.getValue()); } }); if (expectedNodes > 0) { cb.maximumSize(expectedNodes * 2); } this.hostProxies = cb.build(); this.socketFactory = NetUtils.getDefaultSocketFactory(conf); this.token = token; if (token != null) { String tokenUser = getTokenUser(token); if (tokenUser == null) { try { tokenUser = UserGroupInformation.getCurrentUser().getShortUserName(); } catch (IOException e) { throw new RuntimeException(e); } LOG.warn("Cannot determine token user from the token; using {}", tokenUser); } this.tokenUser = tokenUser; } else { this.tokenUser = null; } this.retryPolicy = RetryPolicies.retryUpToMaximumTimeWithFixedSleep(connectionTimeoutMs, retrySleepMs, TimeUnit.MILLISECONDS); this.requestManager = new RequestManager(numThreads, maxPerNode); ExecutorService localExecutor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setNameFormat("RequestManagerExecutor").build()); this.requestManagerExecutor = MoreExecutors.listeningDecorator(localExecutor); LOG.info("Setting up AsyncPbRpcProxy with" + "numThreads=" + numThreads + "retryTime(millis)=" + connectionTimeoutMs + "retrySleep(millis)=" + retrySleepMs); }
From source file:org.apache.druid.server.lookup.cache.LookupCoordinatorManager.java
public void start() { synchronized (lifecycleLock) { if (!lifecycleLock.canStart()) { throw new ISE("LookupCoordinatorManager can't start."); }//w ww . j a v a 2s. c o m try { LOG.debug("Starting."); if (lookupNodeDiscovery == null) { lookupNodeDiscovery = new LookupNodeDiscovery(druidNodeDiscoveryProvider); } //first ensure that previous executorService from last cycle of start/stop has finished completely. //so that we don't have multiple live executorService instances lying around doing lookup management. if (executorService != null && !executorService.awaitTermination( lookupCoordinatorManagerConfig.getHostTimeout().getMillis() * 10, TimeUnit.MILLISECONDS)) { throw new ISE( "WTF! LookupCoordinatorManager executor from last start() hasn't finished. Failed to Start."); } executorService = MoreExecutors.listeningDecorator( Executors.newScheduledThreadPool(lookupCoordinatorManagerConfig.getThreadPoolSize(), Execs.makeThreadFactory("LookupCoordinatorManager--%s"))); initializeLookupsConfigWatcher(); this.backgroundManagerExitedLatch = new CountDownLatch(1); this.backgroundManagerFuture = executorService.scheduleWithFixedDelay(this::lookupManagementLoop, lookupCoordinatorManagerConfig.getInitialDelay(), lookupCoordinatorManagerConfig.getPeriod(), TimeUnit.MILLISECONDS); Futures.addCallback(backgroundManagerFuture, new FutureCallback<Object>() { @Override public void onSuccess(@Nullable Object result) { backgroundManagerExitedLatch.countDown(); LOG.debug("Exited background lookup manager"); } @Override public void onFailure(Throwable t) { backgroundManagerExitedLatch.countDown(); if (backgroundManagerFuture.isCancelled()) { LOG.debug("Exited background lookup manager due to cancellation."); } else { LOG.makeAlert(t, "Background lookup manager exited with error!").emit(); } } }); LOG.debug("Started"); } catch (Exception ex) { LOG.makeAlert(ex, "Got Exception while start()").emit(); } finally { //so that subsequent stop() would happen, even if start() failed with exception lifecycleLock.started(); lifecycleLock.exitStart(); } } }
From source file:com.infinities.skyport.registrar.ConfigurationHomeImpl.java
@Override public synchronized void initialize() throws Exception { if (driverHome == null) { throw new NullPointerException("driverHome cannot be null"); }/*from w w w. java 2 s . c om*/ if (isInitialized.compareAndSet(false, true)) { logger.debug("AccessConfigHomeService Activate"); logger.debug("AccessConfigHomeService load AccessConfigs"); this.worker = MoreExecutors.listeningDecorator( Executors.newCachedThreadPool(new NamedThreadFactory("ordinary", "Cache-Worker"))); this.scheduler = MoreExecutors.listeningDecorator( Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("ordinary", "Scheduler"))); // dispatcher = new EventBusSecondLevelDispatcher(new // AsyncEventBus("eventbus", worker)); Profile profile = profileHome.get(); for (Configuration configuration : profile.getConfigurations()) { if (!this.registeredServiceProviders.containsKey(configuration.getId())) { try { CachedServiceProvider asyncDriver = setUpCachedServiceProvider(configuration); for (ConfigurationLifeCycleListener service : listeners) { service.persist(asyncDriver.getConfiguration()); } this.registeredServiceProviders.put(asyncDriver.getConfiguration().getId(), asyncDriver); logger.info("setup Service Provider success, config: {}", new Object[] { configuration.getCloudName() }); } catch (Exception e) { logger.error("setup Service Provider failed: {}", configuration.getCloudName(), e); } } } } else { throw new IllegalStateException("object has been initialized"); } }
From source file:org.apache.druid.indexing.kafka.supervisor.KafkaSupervisor.java
public KafkaSupervisor(final TaskStorage taskStorage, final TaskMaster taskMaster, final IndexerMetadataStorageCoordinator indexerMetadataStorageCoordinator, final KafkaIndexTaskClientFactory taskClientFactory, final ObjectMapper mapper, final KafkaSupervisorSpec spec, final RowIngestionMetersFactory rowIngestionMetersFactory) { this.taskStorage = taskStorage; this.taskMaster = taskMaster; this.indexerMetadataStorageCoordinator = indexerMetadataStorageCoordinator; this.sortingMapper = mapper.copy().configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); this.spec = spec; this.emitter = spec.getEmitter(); this.monitorSchedulerConfig = spec.getMonitorSchedulerConfig(); this.rowIngestionMetersFactory = rowIngestionMetersFactory; this.dataSource = spec.getDataSchema().getDataSource(); this.ioConfig = spec.getIoConfig(); this.tuningConfig = spec.getTuningConfig(); this.taskTuningConfig = KafkaTuningConfig.copyOf(this.tuningConfig); this.supervisorId = StringUtils.format("KafkaSupervisor-%s", dataSource); this.exec = Execs.singleThreaded(supervisorId); this.scheduledExec = Execs.scheduledSingleThreaded(supervisorId + "-Scheduler-%d"); this.reportingExec = Execs.scheduledSingleThreaded(supervisorId + "-Reporting-%d"); int workerThreads = (this.tuningConfig.getWorkerThreads() != null ? this.tuningConfig.getWorkerThreads() : Math.min(10, this.ioConfig.getTaskCount())); this.workerExec = MoreExecutors .listeningDecorator(Execs.multiThreaded(workerThreads, supervisorId + "-Worker-%d")); log.info("Created worker pool with [%d] threads for dataSource [%s]", workerThreads, this.dataSource); this.taskInfoProvider = new TaskInfoProvider() { @Override//from ww w .j a v a 2 s .c om public TaskLocation getTaskLocation(final String id) { Preconditions.checkNotNull(id, "id"); Optional<TaskRunner> taskRunner = taskMaster.getTaskRunner(); if (taskRunner.isPresent()) { Optional<? extends TaskRunnerWorkItem> item = Iterables.tryFind( taskRunner.get().getRunningTasks(), (Predicate<TaskRunnerWorkItem>) taskRunnerWorkItem -> id .equals(taskRunnerWorkItem.getTaskId())); if (item.isPresent()) { return item.get().getLocation(); } } else { log.error("Failed to get task runner because I'm not the leader!"); } return TaskLocation.unknown(); } @Override public Optional<TaskStatus> getTaskStatus(String id) { return taskStorage.getStatus(id); } }; this.futureTimeoutInSeconds = Math.max(MINIMUM_FUTURE_TIMEOUT_IN_SECONDS, tuningConfig.getChatRetries() * (tuningConfig.getHttpTimeout().getStandardSeconds() + KafkaIndexTaskClient.MAX_RETRY_WAIT_SECONDS)); int chatThreads = (this.tuningConfig.getChatThreads() != null ? this.tuningConfig.getChatThreads() : Math.min(10, this.ioConfig.getTaskCount() * this.ioConfig.getReplicas())); this.taskClient = taskClientFactory.build(taskInfoProvider, dataSource, chatThreads, this.tuningConfig.getHttpTimeout(), this.tuningConfig.getChatRetries()); log.info("Created taskClient with dataSource[%s] chatThreads[%d] httpTimeout[%s] chatRetries[%d]", dataSource, chatThreads, this.tuningConfig.getHttpTimeout(), this.tuningConfig.getChatRetries()); }
From source file:org.apache.tez.runtime.library.common.shuffle.orderedgrouped.ShuffleScheduler.java
public ShuffleScheduler(InputContext inputContext, Configuration conf, int numberOfInputs, ExceptionReporter exceptionReporter, MergeManager mergeManager, FetchedInputAllocatorOrderedGrouped allocator, long startTime, CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, String srcNameTrimmed) throws IOException { this.inputContext = inputContext; this.conf = conf; this.exceptionReporter = exceptionReporter; this.allocator = allocator; this.mergeManager = mergeManager; this.numInputs = numberOfInputs; int abortFailureLimitConf = conf.getInt( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_SOURCE_ATTEMPT_ABORT_LIMIT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_SOURCE_ATTEMPT_ABORT_LIMIT_DEFAULT); if (abortFailureLimitConf <= -1) { abortFailureLimit = Math.max(15, numberOfInputs / 10); } else {//from www . j a v a 2s . c om //No upper cap, as user is setting this intentionally abortFailureLimit = abortFailureLimitConf; } remainingMaps = new AtomicInteger(numberOfInputs); finishedMaps = new BitSet(numberOfInputs); this.ifileReadAhead = ifileReadAhead; this.ifileReadAheadLength = ifileReadAheadLength; this.srcNameTrimmed = srcNameTrimmed; this.codec = codec; int configuredNumFetchers = conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES_DEFAULT); numFetchers = Math.min(configuredNumFetchers, numInputs); localDiskFetchEnabled = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH_DEFAULT); this.minFailurePerHost = conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MIN_FAILURES_PER_HOST, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MIN_FAILURES_PER_HOST_DEFAULT); Preconditions.checkArgument(minFailurePerHost >= 0, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MIN_FAILURES_PER_HOST + "=" + minFailurePerHost + " should not be negative"); this.hostFailureFraction = conf.getFloat( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ACCEPTABLE_HOST_FETCH_FAILURE_FRACTION, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ACCEPTABLE_HOST_FETCH_FAILURE_FRACTION_DEFAULT); this.maxStallTimeFraction = conf.getFloat( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MAX_STALL_TIME_FRACTION, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MAX_STALL_TIME_FRACTION_DEFAULT); Preconditions.checkArgument(maxStallTimeFraction >= 0, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MAX_STALL_TIME_FRACTION + "=" + maxStallTimeFraction + " should not be negative"); this.minReqProgressFraction = conf.getFloat( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MIN_REQUIRED_PROGRESS_FRACTION, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MIN_REQUIRED_PROGRESS_FRACTION_DEFAULT); Preconditions.checkArgument(minReqProgressFraction >= 0, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MIN_REQUIRED_PROGRESS_FRACTION + "=" + minReqProgressFraction + " should not be negative"); this.maxAllowedFailedFetchFraction = conf.getFloat( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MAX_ALLOWED_FAILED_FETCH_ATTEMPT_FRACTION, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MAX_ALLOWED_FAILED_FETCH_ATTEMPT_FRACTION_DEFAULT); Preconditions.checkArgument(maxAllowedFailedFetchFraction >= 0, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MAX_ALLOWED_FAILED_FETCH_ATTEMPT_FRACTION + "=" + maxAllowedFailedFetchFraction + " should not be negative"); this.checkFailedFetchSinceLastCompletion = conf.getBoolean( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION_DEFAULT); this.applicationId = inputContext.getApplicationId().toString(); this.dagId = inputContext.getDagIdentifier(); this.localHostname = inputContext.getExecutionContext().getHostName(); String auxiliaryService = conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT); final ByteBuffer shuffleMetadata = inputContext.getServiceProviderMetaData(auxiliaryService); this.shufflePort = ShuffleUtils.deserializeShuffleProviderMetaData(shuffleMetadata); this.referee = new Referee(); // Counters used by the ShuffleScheduler this.shuffledInputsCounter = inputContext.getCounters().findCounter(TaskCounter.NUM_SHUFFLED_INPUTS); this.reduceShuffleBytes = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES); this.reduceBytesDecompressed = inputContext.getCounters() .findCounter(TaskCounter.SHUFFLE_BYTES_DECOMPRESSED); this.failedShuffleCounter = inputContext.getCounters().findCounter(TaskCounter.NUM_FAILED_SHUFFLE_INPUTS); this.bytesShuffledToDisk = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES_TO_DISK); this.bytesShuffledToDiskDirect = inputContext.getCounters() .findCounter(TaskCounter.SHUFFLE_BYTES_DISK_DIRECT); this.bytesShuffledToMem = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES_TO_MEM); // Counters used by Fetchers ioErrsCounter = inputContext.getCounters().findCounter(SHUFFLE_ERR_GRP_NAME, ShuffleErrors.IO_ERROR.toString()); wrongLengthErrsCounter = inputContext.getCounters().findCounter(SHUFFLE_ERR_GRP_NAME, ShuffleErrors.WRONG_LENGTH.toString()); badIdErrsCounter = inputContext.getCounters().findCounter(SHUFFLE_ERR_GRP_NAME, ShuffleErrors.BAD_ID.toString()); wrongMapErrsCounter = inputContext.getCounters().findCounter(SHUFFLE_ERR_GRP_NAME, ShuffleErrors.WRONG_MAP.toString()); connectionErrsCounter = inputContext.getCounters().findCounter(SHUFFLE_ERR_GRP_NAME, ShuffleErrors.CONNECTION.toString()); wrongReduceErrsCounter = inputContext.getCounters().findCounter(SHUFFLE_ERR_GRP_NAME, ShuffleErrors.WRONG_REDUCE.toString()); this.startTime = startTime; this.lastProgressTime = startTime; this.sslShuffle = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL_DEFAULT); this.asyncHttp = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_USE_ASYNC_HTTP, false); this.httpConnectionParams = ShuffleUtils.getHttpConnectionParams(conf); SecretKey jobTokenSecret = ShuffleUtils .getJobTokenSecretFromTokenBytes(inputContext.getServiceConsumerMetaData(auxiliaryService)); this.jobTokenSecretManager = new JobTokenSecretManager(jobTokenSecret); final ExecutorService fetcherRawExecutor; if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCHER_USE_SHARED_POOL, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCHER_USE_SHARED_POOL_DEFAULT)) { fetcherRawExecutor = inputContext.createTezFrameworkExecutorService(numFetchers, "Fetcher_O {" + srcNameTrimmed + "} #%d"); } else { fetcherRawExecutor = Executors.newFixedThreadPool(numFetchers, new ThreadFactoryBuilder() .setDaemon(true).setNameFormat("Fetcher_O {" + srcNameTrimmed + "} #%d").build()); } this.fetcherExecutor = MoreExecutors.listeningDecorator(fetcherRawExecutor); this.maxFailedUniqueFetches = Math.min(numberOfInputs, 5); referee.start(); this.maxFetchFailuresBeforeReporting = conf.getInt( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT_DEFAULT); this.reportReadErrorImmediately = conf.getBoolean( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR_DEFAULT); this.verifyDiskChecksum = conf.getBoolean( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_VERIFY_DISK_CHECKSUM, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_VERIFY_DISK_CHECKSUM_DEFAULT); /** * Setting to very high val can lead to Http 400 error. Cap it to 75; every attempt id would * be approximately 48 bytes; 48 * 75 = 3600 which should give some room for other info in URL. */ this.maxTaskOutputAtOnce = Math.max(1, Math.min(75, conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE_DEFAULT))); this.skippedInputCounter = inputContext.getCounters().findCounter(TaskCounter.NUM_SKIPPED_INPUTS); this.firstEventReceived = inputContext.getCounters().findCounter(TaskCounter.FIRST_EVENT_RECEIVED); this.lastEventReceived = inputContext.getCounters().findCounter(TaskCounter.LAST_EVENT_RECEIVED); this.compositeFetch = ShuffleUtils.isTezShuffleHandler(conf); this.maxPenaltyTime = conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_HOST_PENALTY_TIME_LIMIT_MS, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_HOST_PENALTY_TIME_LIMIT_MS_DEFAULT); pipelinedShuffleInfoEventsMap = Maps.newConcurrentMap(); LOG.info("ShuffleScheduler running for sourceVertex: " + inputContext.getSourceVertexName() + " with configuration: " + "maxFetchFailuresBeforeReporting=" + maxFetchFailuresBeforeReporting + ", reportReadErrorImmediately=" + reportReadErrorImmediately + ", maxFailedUniqueFetches=" + maxFailedUniqueFetches + ", abortFailureLimit=" + abortFailureLimit + ", maxTaskOutputAtOnce=" + maxTaskOutputAtOnce + ", numFetchers=" + numFetchers + ", hostFailureFraction=" + hostFailureFraction + ", minFailurePerHost=" + minFailurePerHost + ", maxAllowedFailedFetchFraction=" + maxAllowedFailedFetchFraction + ", maxStallTimeFraction=" + maxStallTimeFraction + ", minReqProgressFraction=" + minReqProgressFraction + ", checkFailedFetchSinceLastCompletion=" + checkFailedFetchSinceLastCompletion); }
From source file:com.microsoft.azuretools.azureexplorer.forms.createrediscache.CreateRedisCacheForm.java
@Override protected void okPressed() { try {//ww w.j a v a2 s. c om if (!RedisCacheUtil.doValidate(azureManager, currentSub, dnsNameValue, selectedLocationValue, selectedResGrpValue, selectedPriceTierValue)) { return; } Azure azure = azureManager.getAzure(currentSub.getSubscriptionId()); ProcessingStrategy processor = RedisCacheUtil.doGetProcessor(azure, skus, dnsNameValue, selectedLocationValue, selectedResGrpValue, selectedPriceTierValue, noSSLPort, newResGrp); ExecutorService executor = Executors.newSingleThreadExecutor(); ListeningExecutorService executorService = MoreExecutors.listeningDecorator(executor); ListenableFuture<Void> futureTask = executorService.submit(new CreateRedisCacheCallable(processor)); final ProcessingStrategy processorInner = processor; Futures.addCallback(futureTask, new FutureCallback<Void>() { @Override public void onSuccess(Void arg0) { if (onCreate != null) { onCreate.run(); } } @Override public void onFailure(Throwable throwable) { JOptionPane.showMessageDialog(null, throwable.getMessage(), String.format(MessageHandler.getResourceString(resourceBundle, CREATING_ERROR_INDICATOR_FORMAT), dnsNameValue), JOptionPane.ERROR_MESSAGE, null); try { processorInner.notifyCompletion(); } catch (InterruptedException ex) { LOG.log("Error occurred while notifyCompletion in RedisCache.", ex); } } }); } catch (Exception ex) { JOptionPane.showMessageDialog(null, ex.getMessage(), String.format(MessageHandler.getResourceString(resourceBundle, CREATING_ERROR_INDICATOR_FORMAT), dnsNameValue), JOptionPane.ERROR_MESSAGE, null); LOG.log(String.format(MessageHandler.getResourceString(resourceBundle, CREATING_ERROR_INDICATOR_FORMAT), dnsNameValue), ex); } super.okPressed(); }
From source file:org.apache.abdera2.common.protocol.Session.java
/** * Processes requests asynchronously.. will return a Future * whose value will be set once the call completes *//*from w w w.j av a 2s. c om*/ public <X extends ClientResponse> Future<X> process(ExecutorService executor, Callable<X> resp) { ListeningExecutorService exec = MoreExecutors.listeningDecorator(executor); return exec.submit(resp); }
From source file:org.apache.abdera2.common.protocol.Session.java
/** * Processes requests asynchronously.. the listener will * be invoked once the call completes//from w w w . j a v a 2s . c om */ public <X extends ClientResponse> void process(ExecutorService executor, Callable<X> resp, final Listener<X> listener) { ListeningExecutorService exec = MoreExecutors.listeningDecorator(executor); final ListenableFuture<X> lf = exec.submit(resp); lf.addListener(new Runnable() { public void run() { X resp = null; try { resp = lf.get(); listener.onResponse(resp); } catch (Throwable t) { throw ExceptionHelper.propogate(t); } finally { // auto release since by this point we know we're done with it if (resp != null) resp.release(); } } }, executor); }
From source file:org.n52.lod.csw.CSWLoDEnabler.java
protected Map<String, GetRecordByIdResponseDocument> retrieveRecordsThreaded(int startPos, int maxRecords, long recordsInTotal) { log.info("Retrieve {} records, starting from {} of {}", maxRecords, startPos, recordsInTotal); // one thread for getting ids List<String> recordIdList = getRecordIds(startPos, maxRecords); // many threads getting records descriptions final Map<String, GetRecordByIdResponseDocument> recordDescriptions = Maps.newConcurrentMap(); ListeningExecutorService executorService = MoreExecutors .listeningDecorator(Executors.newFixedThreadPool(maxRecords)); for (String id : recordIdList) { final String recordId = id; log.debug("Adding {} to the model", recordId); CallableRecordDescription c = new CallableRecordDescription(id, csw); ListenableFuture<GetRecordByIdResponseDocument> responseFuture = executorService.submit(c); Futures.addCallback(responseFuture, new FutureCallback<GetRecordByIdResponseDocument>() { private final Logger logger = LoggerFactory.getLogger("Record Downloader"); @Override//from w w w. j a v a2 s . c o m public void onFailure(Throwable t) { logger.error("Error retrieving and parsing record {}", t); report.retrievalIssues.put(recordId, t); } @Override public void onSuccess(GetRecordByIdResponseDocument result) { logger.trace("SUCCESS with {}", result); recordDescriptions.put(recordId, result); report.added++; report.addedIds.add(recordId); } }); } executorService.shutdown(); while (!executorService.isTerminated()) { try { executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e) { log.error("Could not await termination", e); } } log.info("Done with requests and parsing, have {} GetRecordById documents.", recordDescriptions.size()); return recordDescriptions; }
From source file:org.apache.beam.sdk.util.GcsUtil.java
private static void executeBatches(List<BatchRequest> batches) throws IOException { ListeningExecutorService executor = MoreExecutors.listeningDecorator( MoreExecutors.getExitingExecutorService(new ThreadPoolExecutor(MAX_CONCURRENT_BATCHES, MAX_CONCURRENT_BATCHES, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>()))); List<ListenableFuture<Void>> futures = new LinkedList<>(); for (final BatchRequest batch : batches) { futures.add(executor.submit(new Callable<Void>() { public Void call() throws IOException { batch.execute();/*ww w . j av a 2 s . c om*/ return null; } })); } try { Futures.allAsList(futures).get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException("Interrupted while executing batch GCS request", e); } catch (ExecutionException e) { if (e.getCause() instanceof FileNotFoundException) { throw (FileNotFoundException) e.getCause(); } throw new IOException("Error executing batch GCS request", e); } finally { executor.shutdown(); } }