List of usage examples for java.util.concurrent Future isDone
boolean isDone();
From source file:hudson.slaves.SlaveComputer.java
@Override public boolean isConnecting() { Future<?> l = lastConnectActivity; return isOffline() && l != null && !l.isDone(); }
From source file:org.apache.hadoop.yarn.util.TestFSDownload.java
@Test(timeout = 10000) public void testUniqueDestinationPath() throws Exception { Configuration conf = new Configuration(); FileContext files = FileContext.getLocalFSFileContext(conf); final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName())); files.mkdir(basedir, null, true);// w ww.j a v a 2 s . c o m conf.setStrings(TestFSDownload.class.getName(), basedir.toString()); ExecutorService singleThreadedExec = Executors.newSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName()); Path destPath = dirs.getLocalPathForWrite(basedir.toString(), conf); destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet())); Path p = new Path(basedir, "dir" + 0 + ".jar"); LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE; LocalResource rsrc = createJar(files, p, vis); FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc); Future<Path> rPath = singleThreadedExec.submit(fsd); singleThreadedExec.shutdown(); while (!singleThreadedExec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ; Assert.assertTrue(rPath.isDone()); // Now FSDownload will not create a random directory to localize the // resource. Therefore the final localizedPath for the resource should be // destination directory (passed as an argument) + file name. Assert.assertEquals(destPath, rPath.get().getParent()); }
From source file:org.mariotaku.twidere.loader.MicroBlogAPIStatusesLoader.java
private void saveCachedData(final List<ParcelableStatus> data) { final String key = getSerializationKey(); if (key == null || data == null) return;//from w ww . j av a 2 s.co m final int databaseItemLimit = mPreferences.getInt(KEY_DATABASE_ITEM_LIMIT, DEFAULT_DATABASE_ITEM_LIMIT); try { final List<ParcelableStatus> statuses = data.subList(0, Math.min(databaseItemLimit, data.size())); final PipedOutputStream pos = new PipedOutputStream(); final PipedInputStream pis = new PipedInputStream(pos); final Future<Object> future = pool.submit(new Callable<Object>() { @Override public Object call() throws Exception { LoganSquareMapperFinder.mapperFor(ParcelableStatus.class).serialize(statuses, pos); return null; } }); final boolean saved = mFileCache.save(key, pis, new IoUtils.CopyListener() { @Override public boolean onBytesCopied(int current, int total) { return !future.isDone(); } }); if (BuildConfig.DEBUG) { Log.v(LOGTAG, key + " saved: " + saved); } } catch (final Exception e) { // Ignore if (BuildConfig.DEBUG && !(e instanceof IOException)) { Log.w(LOGTAG, e); } } }
From source file:com.baifendian.swordfish.execserver.runner.flow.FlowRunner.java
/** * kill //from www .ja v a 2 s. c o m */ private void kill() { synchronized (this) { if (activeNodeRunners.isEmpty()) { return; } logger.info("Kill has been called on exec id: {}, num: {}", executionFlow.getId(), activeNodeRunners.size()); // ? for (Map.Entry<NodeRunner, Future<Boolean>> entry : activeNodeRunners.entrySet()) { NodeRunner nodeRunner = entry.getKey(); Future<Boolean> future = entry.getValue(); if (!future.isDone()) { // kill ? logger.info("kill exec, id: {}, node: {}", executionFlow.getId(), nodeRunner.getNodename()); // ? nodeRunner.kill(); // , future.cancel(true); } } } }
From source file:org.apache.hadoop.hdfs.notifier.server.ServerDispatcher.java
private void sendNotifications() { if (LOG.isDebugEnabled()) { LOG.debug("Sending notifications ..."); }/*from www.ja v a 2 s.c o m*/ List<Long> shuffledClients = new ArrayList<Long>(assignedClients); Collections.shuffle(shuffledClients); for (Long clientId : shuffledClients) { Future<DispatchStatus> clientFuture = clientsFuture.get(clientId); DispatchStatus status; if (!shouldSendNotifications(clientId)) { continue; } // Get the results of the last batch notification sending // for this client. if (clientFuture != null && clientFuture.isDone()) { try { status = clientFuture.get(); } catch (InterruptedException e) { LOG.error("Got interrupted for client " + clientId, e); core.shutdown(); return; } catch (ExecutionException e) { LOG.error("got exception in clientFuture.get() for client " + clientId, e); core.shutdown(); return; } if (status.action == DispatchStatus.Action.DISPATCH_SUCCESSFUL) { handleSuccessfulDispatch(clientId, status.time); } if (status.action == DispatchStatus.Action.DISPATCH_FAILED) { handleFailedDispatch(clientId, status.time); } } // Submit a new batch notification sending callable for this client if (clientFuture == null || (clientFuture != null && clientFuture.isDone())) { if (LOG.isDebugEnabled()) { LOG.debug("Submiting for client " + clientId); } try { clientFuture = workers.submit(new NotificationsSender(clientId)); } catch (RejectedExecutionException e) { LOG.warn("Failed to submit task", e); continue; } clientsFuture.put(clientId, clientFuture); } } if (LOG.isDebugEnabled()) { LOG.debug("Done sending notifications."); } }
From source file:org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher.java
@Override public PigStats launchPig(PhysicalPlan php, String grpName, PigContext pc) throws Exception { synchronized (this) { if (executor == null) { executor = Executors.newSingleThreadExecutor(namedThreadFactory); }// w w w .j av a 2 s . c o m } if (pc.getExecType().isLocal()) { pc.getProperties().setProperty(TezConfiguration.TEZ_LOCAL_MODE, "true"); pc.getProperties().setProperty(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, "true"); pc.getProperties().setProperty(TezConfiguration.TEZ_IGNORE_LIB_URIS, "true"); pc.getProperties().setProperty(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, DAGSchedulerNaturalOrderControlled.class.getName()); } Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties(), true); // Make sure MR counter does not exceed limit if (conf.get(TezConfiguration.TEZ_COUNTERS_MAX) != null) { conf.setInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTERS_MAX_KEY, Math.max(conf.getInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTERS_MAX_KEY, 0), conf.getInt(TezConfiguration.TEZ_COUNTERS_MAX, 0))); } if (conf.get(TezConfiguration.TEZ_COUNTERS_MAX_GROUPS) != null) { conf.setInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTER_GROUPS_MAX_KEY, Math.max(conf.getInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTER_GROUPS_MAX_KEY, 0), conf.getInt(TezConfiguration.TEZ_COUNTERS_MAX_GROUPS, 0))); } // This is hacky, but Limits cannot be initialized twice try { Field f = Limits.class.getDeclaredField("isInited"); f.setAccessible(true); f.setBoolean(null, false); Limits.init(conf); } catch (Throwable e) { log.warn("Error when setting counter limit: " + e.getMessage()); } if (pc.defaultParallel == -1 && !conf.getBoolean(PigConfiguration.PIG_TEZ_AUTO_PARALLELISM, true)) { pc.defaultParallel = 1; } aggregateWarning = conf.getBoolean("aggregate.warning", false); TezResourceManager tezResourceManager = TezResourceManager.getInstance(); tezResourceManager.init(pc, conf); String stagingDir = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR); String resourcesDir = tezResourceManager.getResourcesDir().toString(); if (stagingDir == null) { // If not set in tez-site.xml, use Pig's tez resources directory as staging directory // instead of TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT stagingDir = resourcesDir; conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, resourcesDir); } log.info("Tez staging directory is " + stagingDir + " and resources directory is " + resourcesDir); List<TezOperPlan> processedPlans = new ArrayList<TezOperPlan>(); tezScriptState = TezScriptState.get(); tezStats = new TezPigScriptStats(pc); PigStats.start(tezStats); conf.set(TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS, "true"); TezJobCompiler jc = new TezJobCompiler(pc, conf); TezPlanContainer tezPlanContainer = compile(php, pc); tezStats.initialize(tezPlanContainer); tezScriptState.emitInitialPlanNotification(tezPlanContainer); tezScriptState.emitLaunchStartedNotification(tezPlanContainer.size()); //number of DAGs to Launch boolean stop_on_failure = Boolean.valueOf(pc.getProperties().getProperty("stop.on.failure", "false")); boolean stoppedOnFailure = false; TezPlanContainerNode tezPlanContainerNode; TezOperPlan tezPlan; int processedDAGs = 0; while ((tezPlanContainerNode = tezPlanContainer.getNextPlan(processedPlans)) != null) { tezPlan = tezPlanContainerNode.getTezOperPlan(); processLoadAndParallelism(tezPlan, pc); processedPlans.add(tezPlan); ProgressReporter reporter = new ProgressReporter(tezPlanContainer.size(), processedDAGs); if (tezPlan.size() == 1 && tezPlan.getRoots().get(0) instanceof NativeTezOper) { // Native Tez Plan NativeTezOper nativeOper = (NativeTezOper) tezPlan.getRoots().get(0); tezScriptState.emitJobsSubmittedNotification(1); nativeOper.runJob(tezPlanContainerNode.getOperatorKey().toString()); } else { TezPOPackageAnnotator pkgAnnotator = new TezPOPackageAnnotator(tezPlan); pkgAnnotator.visit(); runningJob = jc.compile(tezPlanContainerNode, tezPlanContainer); //TODO: Exclude vertex groups from numVerticesToLaunch ?? tezScriptState.dagLaunchNotification(runningJob.getName(), tezPlan, tezPlan.size()); runningJob.setPigStats(tezStats); // Set the thread UDFContext so registered classes are available. final UDFContext udfContext = UDFContext.getUDFContext(); Runnable task = new Runnable() { @Override public void run() { Thread.currentThread().setContextClassLoader(PigContext.getClassLoader()); UDFContext.setUdfContext(udfContext.clone()); runningJob.run(); } }; // Mark the times that the jobs were submitted so it's reflected in job // history props. TODO: Fix this. unused now long scriptSubmittedTimestamp = System.currentTimeMillis(); // Job.getConfiguration returns the shared configuration object Configuration jobConf = runningJob.getConfiguration(); jobConf.set("pig.script.submitted.timestamp", Long.toString(scriptSubmittedTimestamp)); jobConf.set("pig.job.submitted.timestamp", Long.toString(System.currentTimeMillis())); Future<?> future = executor.submit(task); tezScriptState.emitJobsSubmittedNotification(1); boolean jobStarted = false; while (!future.isDone()) { if (!jobStarted && runningJob.getApplicationId() != null) { jobStarted = true; String appId = runningJob.getApplicationId().toString(); //For Oozie Pig action job id matching compatibility with MR mode log.info("HadoopJobId: " + appId.replace("application", "job")); tezScriptState.emitJobStartedNotification(appId); tezScriptState.dagStartedNotification(runningJob.getName(), appId); } reporter.notifyUpdate(); Thread.sleep(1000); } // For tez_local mode where PigProcessor destroys all UDFContext UDFContext.setUdfContext(udfContext); try { // In case of FutureTask there is no uncaught exception // Need to do future.get() to get any exception future.get(); } catch (ExecutionException e) { setJobException(e.getCause()); } } processedDAGs++; if (tezPlanContainer.size() == processedDAGs) { tezScriptState.emitProgressUpdatedNotification(100); } else { tezScriptState.emitProgressUpdatedNotification( ((tezPlanContainer.size() - processedDAGs) / tezPlanContainer.size()) * 100); } handleUnCaughtException(pc); boolean tezDAGSucceeded = reporter.notifyFinishedOrFailed(); tezPlanContainer.updatePlan(tezPlan, tezDAGSucceeded); // if stop_on_failure is enabled, we need to stop immediately when any job has failed if (!tezDAGSucceeded) { if (stop_on_failure) { stoppedOnFailure = true; break; } else { log.warn("Ooops! Some job has failed! Specify -stop_on_failure if you " + "want Pig to stop immediately on failure."); } } } tezStats.finish(); tezScriptState.emitLaunchCompletedNotification(tezStats.getNumberSuccessfulJobs()); for (OutputStats output : tezStats.getOutputStats()) { POStore store = output.getPOStore(); try { if (!output.isSuccessful()) { store.getStoreFunc().cleanupOnFailure(store.getSFile().getFileName(), Job.getInstance(output.getConf())); } else { store.getStoreFunc().cleanupOnSuccess(store.getSFile().getFileName(), Job.getInstance(output.getConf())); } } catch (IOException e) { throw new ExecException(e); } catch (AbstractMethodError nsme) { // Just swallow it. This means we're running against an // older instance of a StoreFunc that doesn't implement // this method. } } if (stoppedOnFailure) { throw new ExecException("Stopping execution on job failure with -stop_on_failure option", 6017, PigException.REMOTE_ENVIRONMENT); } return tezStats; }
From source file:com.alibaba.otter.shared.communication.core.impl.DefaultCommunicationClientImpl.java
public Object call(final String[] addrs, final Event event) { Assert.notNull(this.factory, "No factory specified"); if (addrs == null || addrs.length == 0) { throw new IllegalArgumentException("addrs example: 127.0.0.1:1099"); }/* w w w . ja v a2 s . c om*/ ExecutorCompletionService completionService = new ExecutorCompletionService(executor); List<Future<Object>> futures = new ArrayList<Future<Object>>(addrs.length); List result = new ArrayList(10); for (final String addr : addrs) { futures.add(completionService.submit((new Callable<Object>() { @Override public Object call() throws Exception { return DefaultCommunicationClientImpl.this.call(addr, event); } }))); } Exception ex = null; int errorIndex = 0; while (errorIndex < futures.size()) { try { Future future = completionService.take();// ? future.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); ex = e; break; } catch (ExecutionException e) { ex = e; break; } errorIndex++; } if (errorIndex < futures.size()) { for (int index = 0; index < futures.size(); index++) { Future<Object> future = futures.get(index); if (future.isDone() == false) { future.cancel(true); } } } else { for (int index = 0; index < futures.size(); index++) { Future<Object> future = futures.get(index); try { result.add(future.get()); } catch (InterruptedException e) { // ignore Thread.currentThread().interrupt(); } catch (ExecutionException e) { // ignore } } } if (ex != null) { throw new CommunicationException( String.format("call addr[%s] error by %s", addrs[errorIndex], ex.getMessage()), ex); } else { return result; } }
From source file:com.alibaba.otter.node.etl.extract.extractor.DatabaseExtractor.java
@Override public void extract(DbBatch dbBatch) throws ExtractException { Assert.notNull(dbBatch);//from w w w. j av a 2s. c om Assert.notNull(dbBatch.getRowBatch()); // ?? Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId()); boolean mustDb = pipeline.getParameters().getSyncConsistency().isMedia(); boolean isRow = pipeline.getParameters().getSyncMode().isRow();// ??? // ?? adjustPoolSize(pipeline.getParameters().getExtractPoolSize()); // Extractor? ExecutorCompletionService completionService = new ExecutorCompletionService(executor); // ??? ExtractException exception = null; // ?? List<DataItem> items = new ArrayList<DataItem>(); List<Future> futures = new ArrayList<Future>(); List<EventData> eventDatas = dbBatch.getRowBatch().getDatas(); for (EventData eventData : eventDatas) { if (eventData.getEventType().isDdl()) { continue; } DataItem item = new DataItem(eventData); // row??????row??? boolean flag = mustDb || (eventData.getSyncConsistency() != null && eventData.getSyncConsistency().isMedia()); // ?case, oracle erosa?????? if (!flag && CollectionUtils.isEmpty(eventData.getUpdatedColumns())) { DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId()); if (dataMedia.getSource().getType().isOracle()) { flag |= true; eventData.setRemedy(true);// ???erosa????? } } if (isRow && !flag) { // ????? // view?? flag = checkNeedDbForRowMode(pipeline, eventData); } if (flag && (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate())) {// ???? Future future = completionService.submit(new DatabaseExtractWorker(pipeline, item), null); // ?? if (future.isDone()) { // ?CallerRun???? try { future.get(); } catch (InterruptedException e) { cancel(futures);// ?? throw new ExtractException(e); } catch (ExecutionException e) { cancel(futures); // ?? throw new ExtractException(e); } } futures.add(future);// } items.add(item);// ? } // ? int index = 0; while (index < futures.size()) { // ?? try { Future future = completionService.take();// ? future.get(); } catch (InterruptedException e) { exception = new ExtractException(e); break;// future } catch (ExecutionException e) { exception = new ExtractException(e); break;// future } index++; } if (index < futures.size()) { // ???cancel????? cancel(futures); throw exception; } else { // ?, ???? for (int i = 0; i < items.size(); i++) { DataItem item = items.get(i); if (item.filter) { // ??????? eventDatas.remove(item.getEventData()); } } } }
From source file:com.tc.stats.DSO.java
@Override public Map<ObjectName, Map> getPrimaryClientStatistics() { Map<ObjectName, Map> result = new HashMap<ObjectName, Map>(); List<Callable<Map>> tasks = new ArrayList<Callable<Map>>(); synchronized (clientObjectNames) { Iterator<ObjectName> iter = clientObjectNames.iterator(); while (iter.hasNext()) { ObjectName clientBeanName = iter.next(); tasks.add(new PrimaryClientStatWorker(clientBeanName, clientMap.get(clientBeanName))); }/*ww w . j a v a 2s.c om*/ } try { List<Future<Map>> results = pool.invokeAll(tasks, 2, TimeUnit.SECONDS); Iterator<Future<Map>> resultIter = results.iterator(); while (resultIter.hasNext()) { Future<Map> future = resultIter.next(); if (future.isDone()) { try { Map statsMap = future.get(); if (statsMap != null) { result.put((ObjectName) statsMap.remove("clientBeanName"), statsMap); } } catch (InterruptedException e) { /**/ } catch (ExecutionException e) { /**/ } } } } catch (InterruptedException ie) {/**/ } return result; }