List of usage examples for java.util.concurrent.atomic AtomicInteger AtomicInteger
public AtomicInteger(int initialValue)
From source file:com.baidu.asynchttpclient.AsyncHttpClient.java
/** * Creates a new AsyncHttpClient./*from w ww. j a v a 2s . c o m*/ */ public AsyncHttpClient() { BasicHttpParams httpParams = new BasicHttpParams(); ConnManagerParams.setTimeout(httpParams, socketTimeout); ConnManagerParams.setMaxConnectionsPerRoute(httpParams, new ConnPerRouteBean(maxConnections)); ConnManagerParams.setMaxTotalConnections(httpParams, DEFAULT_MAX_CONNECTIONS); HttpConnectionParams.setSoTimeout(httpParams, socketTimeout); HttpConnectionParams.setConnectionTimeout(httpParams, socketTimeout); HttpConnectionParams.setTcpNoDelay(httpParams, true); HttpConnectionParams.setSocketBufferSize(httpParams, DEFAULT_SOCKET_BUFFER_SIZE); HttpProtocolParams.setVersion(httpParams, HttpVersion.HTTP_1_1); HttpProtocolParams.setUserAgent(httpParams, String.format("android-async-http/%s (http://loopj.com/android-async-http)", VERSION)); HttpProtocolParams.setUseExpectContinue(httpParams, false); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); // schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(), 443)); ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(httpParams, schemeRegistry); httpContext = new SyncBasicHttpContext(new BasicHttpContext()); httpClient = new DefaultHttpClient(cm, httpParams); httpClient.addRequestInterceptor(new HttpRequestInterceptor() { public void process(HttpRequest request, HttpContext context) { if (!request.containsHeader(HEADER_ACCEPT_ENCODING)) { request.addHeader(HEADER_ACCEPT_ENCODING, ENCODING_GZIP); } for (String header : clientHeaderMap.keySet()) { request.addHeader(header, clientHeaderMap.get(header)); } } }); httpClient.addResponseInterceptor(new HttpResponseInterceptor() { public void process(HttpResponse response, HttpContext context) { final HttpEntity entity = response.getEntity(); final Header encoding = entity.getContentEncoding(); if (encoding != null) { for (HeaderElement element : encoding.getElements()) { if (element.getName().equalsIgnoreCase(ENCODING_GZIP)) { response.setEntity(new InflatingEntity(response.getEntity())); break; } } } } }); httpClient.setHttpRequestRetryHandler(new RetryHandler(DEFAULT_MAX_RETRIES)); threadPool = (ThreadPoolExecutor) Executors.newCachedThreadPool(new ThreadFactory() { private final AtomicInteger mCount = new AtomicInteger(1); public Thread newThread(Runnable r) { Thread t = new Thread(r, "AsyncHttpClient #" + mCount.getAndIncrement()); if (t.isDaemon()) t.setDaemon(false); if (t.getPriority() != (Thread.NORM_PRIORITY - 1)) t.setPriority((Thread.NORM_PRIORITY - 1)); return t; } }); requestMap = new WeakHashMap<Context, List<WeakReference<Future<?>>>>(); clientHeaderMap = new HashMap<String, String>(); }
From source file:com.indeed.imhotep.builder.tsv.TsvConverter.java
public void run() { final List<FileToIndex> files = findNewFilesToIndex(); boolean first = true; final Map<FileToIndex, Future<Boolean>> futures = Maps.newHashMap(); final int toProcessCount = files.size(); final AtomicInteger processedCount = new AtomicInteger(0); for (final FileToIndex fileToIndex : files) { if (DEBUG_BUILD_ONE && !first) { break; }/*from w ww. j a v a 2s. c o m*/ first = false; futures.put(fileToIndex, executor.submit(new Callable<Boolean>() { @Override public Boolean call() throws Exception { final ShardInfo.DateTimeRange fileTimeRange = getTimeRangeFromFileName(fileToIndex.name); final String indexName = fileToIndex.index; final File localShardParentDir = localBuildDir; final String shardName = getShardName(fileTimeRange); final File localShardDir = new File(localShardParentDir, shardName); final String[] commandToRun = new String[] { "-o", localShardParentDir.getAbsolutePath(), "--start", fileTimeRange.start.toString().replace("T", " ").substring(0, 19), "--end", fileTimeRange.end.toString().replace("T", " ").substring(0, 19), "--overwrite", "--extra", fileToIndex.fsPath.toString().replaceFirst("hdfs://[^/]*", "hdfs:") }; final EasyIndexBuilderOptions builderOptions = new EasyIndexBuilderOptions(); builderOptions.baseDir = localShardParentDir.getAbsolutePath(); builderOptions.start = fileTimeRange.start.getMillis(); builderOptions.end = fileTimeRange.end.getMillis(); builderOptions.overwrite = true; builderOptions.extra = fileToIndex.fsPath.toString().replaceFirst("hdfs://[^/]*", "hdfs:"); log.info(StringUtils.join(commandToRun, " ")); final EasyIndexBuilderFromTSV builder = new EasyIndexBuilderFromTSV(); builder.setOptions(builderOptions); final int buildExitCode = builder.build(); // alternatively run in a separate process instead of in the same thread // final String bashCommand = "\"" + org.apache.commons.lang.StringUtils.join(commandToRun, "\" \"") + "\" 2>&1"; // log.info(bashCommand); // final Process runningBuild = Runtime.getRuntime().exec(new String[] {"bash", "-c", bashCommand}); // BufferedReader output = new BufferedReader(new InputStreamReader(runningBuild.getInputStream())); // for(String line = output.readLine(); line != null; line = output.readLine()) { // log.info(line); // TODO // } // final int buildExitCode = runningBuild.waitFor(); if (buildExitCode != 0) { throw new RuntimeException("Build exited with a non 0 code: " + buildExitCode); } if (!localShardDir.exists() || !localShardDir.isDirectory()) { throw new RuntimeException( "Build completed but a shard directory is not found at " + localShardDir); } // add build timestamp (version) final File localShardDirWithTimestamp = new File(localShardParentDir, localShardDir.getName() + "." + yyyymmddhhmmss.print(DateTime.now())); if (!localShardDir.renameTo(localShardDirWithTimestamp)) { throw new RuntimeException( "Failed to append timestamp to completed shard dir: " + localShardDir); } final String scheme = finalFS.getUri().getScheme(); if (scheme.equals("hdfs") || scheme.equals("s3n")) { final boolean uploadSucceeded = uploadShard(localShardDirWithTimestamp.getParent(), localShardDirWithTimestamp.getName(), indexName, finalShardPath, finalFS, qaMode); if (uploadSucceeded) { try { PosixFileOperations.rmrf(localShardDirWithTimestamp); } catch (IOException e) { log.warn("Failed to delete temp dir: " + localShardDirWithTimestamp, e); } } else { log.error( "Shard upload failed. Local shard copy left in: " + localShardDirWithTimestamp); throw new RuntimeException("Failed to upload the built shard to " + scheme); } } else { final Path localShardLoc; final Path finalIndexPath; final Path finalShardLoc; localShardLoc = new Path("file:" + localShardDirWithTimestamp.getAbsolutePath()); finalIndexPath = new Path(finalShardPath, indexName); finalFS.mkdirs(finalIndexPath); finalShardLoc = new Path(finalIndexPath, localShardDirWithTimestamp.getName()); if (!finalFS.rename(localShardLoc, finalShardLoc)) { throw new RuntimeException("Failed to move the completed shard dir " + localShardDir + " to the final location at " + finalIndexPath); } } moveToProcessed(fileToIndex, processedSuccessPath); log.info("Progress: " + (processedCount.incrementAndGet() * 100 / toProcessCount) + "%"); return true; } })); } for (FileToIndex fileToIndex : files) { final Future<Boolean> future = futures.get(fileToIndex); if (future == null) { log.warn("Build failed for: " + fileToIndex.fsPath); continue; } boolean success; try { success = future.get(); } catch (Exception e) { log.warn("Failed to build " + fileToIndex.fsPath, e); success = false; try { writeExceptionLog(fileToIndex, processedFailedPath, e); } catch (Exception e2) { log.error("Failed to write the error log to HDFS", e2); } } if (!success) { // mark the file as failed moveToProcessed(fileToIndex, processedFailedPath); } } }
From source file:com.comcast.cdn.traffic_control.traffic_router.core.router.StatelessTrafficRouterPerformanceTest.java
public URL routeTest(final HTTPRequest request, String[] rstrs) throws TrafficRouterException, GeolocationException { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Attempting to route HTTPRequest: " + request.getRequestedUrl()); }/*from ww w. j a v a2 s .c om*/ final String ip = request.getClientIP(); final DeliveryService ds = selectDeliveryService(request, true); if (ds == null) { return null; } final StatTracker.Track track = StatTracker.getTrack(); List<Cache> caches = selectCache(request, ds, track); Dispersion dispersion = ds.getDispersion(); Cache cache = dispersion.getCache(consistentHash(caches, request.getPath())); try { if (cache != null) { return new URL(ds.createURIString(request, cache)); } } catch (final MalformedURLException e) { LOGGER.error(e.getMessage(), e); throw new TrafficRouterException(URL_ERR_STR, e); } LOGGER.warn("No Cache found in CoverageZoneMap for HTTPRequest.getClientIP: " + ip); final String zoneId = null; Geolocation clientLocation = getGeolocationService().location(request.getClientIP()); final List<CacheLocation> cacheLocations = orderCacheLocations(getCacheRegister().getCacheLocations(zoneId), ds, clientLocation); for (final CacheLocation location : cacheLocations) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Trying location: " + location.getId()); } caches = getSupportingCaches(location.getCaches(), ds); if (caches.isEmpty()) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("No online, supporting caches were found at location: " + location.getId()); } return null; } cache = dispersion.getCache(consistentHash(caches, request.getPath())); LOGGER.warn("cache selected: " + cache.getId()); Map<String, AtomicInteger> m = new HashMap<String, AtomicInteger>(); long time = System.currentTimeMillis(); for (String str : rstrs) { cache = dispersion.getCache(consistentHash(caches, str)); AtomicInteger i = m.get(cache.getId()); if (i == null) { i = new AtomicInteger(0); m.put(cache.getId(), i); } i.incrementAndGet(); } time = System.currentTimeMillis() - time; LOGGER.warn(String.format("time: %d", time)); for (String id : m.keySet()) { LOGGER.warn(String.format("cache(%s): %d", id, m.get(id).get())); } m = new HashMap<String, AtomicInteger>(); time = System.currentTimeMillis(); for (String str : rstrs) { cache = consistentHashOld(caches, str); AtomicInteger i = m.get(cache.getId()); if (i == null) { i = new AtomicInteger(0); m.put(cache.getId(), i); } i.incrementAndGet(); } time = System.currentTimeMillis() - time; LOGGER.warn(String.format("time: %d", time)); for (String id : m.keySet()) { LOGGER.warn(String.format("cache(%s): %d", id, m.get(id).get())); } if (cache != null) { try { return new URL(ds.createURIString(request, cache)); } catch (final MalformedURLException e) { LOGGER.error(e.getMessage(), e); throw new TrafficRouterException(URL_ERR_STR, e); } } } LOGGER.info(UNABLE_TO_ROUTE_REQUEST); throw new TrafficRouterException(UNABLE_TO_ROUTE_REQUEST); }
From source file:io.codis.nedis.codis.RoundRobinNedisClientPool.java
@Override public Future<Void> close() { if (!closed.compareAndSet(false, true)) { return closePromise; }//from w w w .j a va2 s . c o m new Thread(getClass().getSimpleName() + "-Closer") { @Override public void run() { try { watcher.close(); } catch (IOException e) { LOG.warn("IOException should not have been thrown", e); } if (closeCurator) { curatorClient.close(); } synchronized (RoundRobinNedisClientPool.this) { List<PooledObject> toClose = pools; if (toClose.isEmpty()) { closePromise.trySuccess(null); return; } final AtomicInteger remaining = new AtomicInteger(toClose.size()); FutureListener<Void> listener = new FutureListener<Void>() { @Override public void operationComplete(Future<Void> future) throws Exception { if (remaining.decrementAndGet() == 0) { closePromise.trySuccess(null); } } }; for (PooledObject pool : toClose) { pool.pool.close().addListener(listener); } } } }.start(); return closePromise; }
From source file:eu.stratosphere.nephele.executiongraph.ExecutionVertex.java
/** * Private constructor used to duplicate execution vertices. * /*ww w. jav a 2 s. co m*/ * @param vertexID * the ID of the new execution vertex. * @param executionGraph * the execution graph the new vertex belongs to * @param groupVertex * the group vertex the new vertex belongs to * @param numberOfOutputGates * the number of output gates attached to this vertex * @param numberOfInputGates * the number of input gates attached to this vertex */ private ExecutionVertex(final ExecutionVertexID vertexID, final ExecutionGraph executionGraph, final ExecutionGroupVertex groupVertex, final int numberOfOutputGates, final int numberOfInputGates) { this.vertexID = vertexID; this.executionGraph = executionGraph; this.groupVertex = groupVertex; this.retriesLeft = new AtomicInteger(groupVertex.getNumberOfExecutionRetries()); this.outputGates = new ExecutionGate[numberOfOutputGates]; this.inputGates = new ExecutionGate[numberOfInputGates]; // Register vertex with execution graph this.executionGraph.registerExecutionVertex(this); // Register the vertex itself as a listener for state changes registerExecutionListener(this.executionGraph); }
From source file:com.netflix.curator.framework.recipes.queue.TestDistributedQueue.java
@Test public void testErrorMode() throws Exception { Timing timing = new Timing(); CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1)); client.start();/*from www. j a v a 2 s. c o m*/ try { final AtomicReference<CountDownLatch> latch = new AtomicReference<CountDownLatch>( new CountDownLatch(1)); final AtomicInteger count = new AtomicInteger(0); QueueConsumer<TestQueueItem> consumer = new QueueConsumer<TestQueueItem>() { @Override public void consumeMessage(TestQueueItem message) throws Exception { if (count.incrementAndGet() < 2) { throw new Exception(); } latch.get().countDown(); } @Override public void stateChanged(CuratorFramework client, ConnectionState newState) { } }; DistributedQueue<TestQueueItem> queue = QueueBuilder.builder(client, consumer, serializer, QUEUE_PATH) .lockPath("/locks").buildQueue(); try { queue.start(); TestQueueItem item = new TestQueueItem("1"); queue.put(item); Assert.assertTrue(timing.awaitLatch(latch.get())); Assert.assertEquals(count.get(), 2); queue.setErrorMode(ErrorMode.DELETE); count.set(0); latch.set(new CountDownLatch(1)); item = new TestQueueItem("1"); queue.put(item); Assert.assertFalse(latch.get().await(5, TimeUnit.SECONDS)); // consumer should get called only once Assert.assertEquals(count.get(), 1); } finally { queue.close(); } } finally { client.close(); } }
From source file:com.cyngn.vertx.bosun.BosunReporterTests.java
@Test public void testSendMany(TestContext context) throws Exception { JsonObject metric = new JsonObject(); metric.put("action", BosunReporter.INDEX_COMMAND); metric.put("metric", "test.value"); metric.put("value", "34.4"); metric.put("tags", new JsonObject().put("foo", "bar")); int totalMessages = 10000; AtomicInteger count = new AtomicInteger(0); Async async = context.async();/* w ww . j a v a 2s.c o m*/ AtomicInteger okCount = new AtomicInteger(0); Handler<AsyncResult<Message<JsonObject>>> handler = result -> { if (result.failed()) { context.fail(); } String response = result.result().body().getString(BosunReporter.RESULT_FIELD); if (StringUtils.equals(BosunResponse.OK_MSG, response)) { okCount.incrementAndGet(); } else if (StringUtils.equals(BosunResponse.EXISTS_MSG, response)) { } else { context.fail(); } if (count.incrementAndGet() == totalMessages) { if (okCount.get() != 1) { context.fail(); return; } async.complete(); } }; for (int i = 0; i < totalMessages; i++) { eb.send(topic, metric, new DeliveryOptions(), handler); } }
From source file:com.linkedin.pinot.core.data.manager.offline.OfflineTableDataManager.java
@Override public void addSegment(final IndexSegment indexSegmentToAdd) { LOGGER.info("Trying to add a new segment to table : " + _tableName); synchronized (getGlobalLock()) { if (!_segmentsMap.containsKey(indexSegmentToAdd.getSegmentName())) { LOGGER.info("Trying to add segment - " + indexSegmentToAdd.getSegmentName()); _segmentsMap.put(indexSegmentToAdd.getSegmentName(), new OfflineSegmentDataManager(indexSegmentToAdd)); markSegmentAsLoaded(indexSegmentToAdd.getSegmentName()); _referenceCounts.put(indexSegmentToAdd.getSegmentName(), new AtomicInteger(1)); } else {/* ww w. j av a2 s .c o m*/ LOGGER.info("Trying to refresh segment - " + indexSegmentToAdd.getSegmentName()); OfflineSegmentDataManager segment = _segmentsMap.get(indexSegmentToAdd.getSegmentName()); _segmentsMap.put(indexSegmentToAdd.getSegmentName(), new OfflineSegmentDataManager(indexSegmentToAdd)); if (segment != null) { _currentNumberOfDocuments.dec(segment.getSegment().getTotalDocs()); _currentNumberOfDocuments.inc(indexSegmentToAdd.getTotalDocs()); segment.getSegment().destroy(); } } } }
From source file:com.twitter.distributedlog.service.balancer.ClusterBalancer.java
public void balance(int rebalanceWaterMark, double rebalanceTolerancePercentage, int rebalanceConcurrency, Optional<String> source, Optional<RateLimiter> rebalanceRateLimiter) { Map<SocketAddress, Set<String>> distribution = monitor.getStreamOwnershipDistribution(); if (distribution.size() <= 1) { return;/*from ww w . j ava2 s. c o m*/ } SocketAddress sourceAddr = null; if (source.isPresent()) { sourceAddr = DLSocketAddress.parseSocketAddress(source.get()); logger.info("Balancer source is {}", sourceAddr); if (!distribution.containsKey(sourceAddr)) { return; } } // Get the list of hosts ordered by number of streams in DESC order List<Host> hosts = new ArrayList<Host>(distribution.size()); for (Map.Entry<SocketAddress, Set<String>> entry : distribution.entrySet()) { Host host = new Host(entry.getKey(), entry.getValue(), clientBuilder); hosts.add(host); } Collections.sort(hosts, new HostComparator()); try { // find the host to move streams from. int hostIdxMoveFrom = -1; if (null != sourceAddr) { for (Host host : hosts) { ++hostIdxMoveFrom; if (sourceAddr.equals(host.address)) { break; } } } // compute the average load. int totalStream = 0; for (Host host : hosts) { totalStream += host.streams.size(); } double averageLoad; if (hostIdxMoveFrom >= 0) { averageLoad = ((double) totalStream / (hosts.size() - 1)); } else { averageLoad = ((double) totalStream / hosts.size()); } int moveFromLowWaterMark; int moveToHighWaterMark = Math.max(1, (int) (averageLoad + averageLoad * rebalanceTolerancePercentage / 100.0f)); if (hostIdxMoveFrom >= 0) { moveFromLowWaterMark = Math.max(0, rebalanceWaterMark); moveStreams(hosts, new AtomicInteger(hostIdxMoveFrom), moveFromLowWaterMark, new AtomicInteger(hosts.size() - 1), moveToHighWaterMark, rebalanceRateLimiter); moveRemainingStreamsFromSource(hosts.get(hostIdxMoveFrom), hosts, rebalanceRateLimiter); } else { moveFromLowWaterMark = Math.max((int) Math.ceil(averageLoad), rebalanceWaterMark); AtomicInteger moveFrom = new AtomicInteger(0); AtomicInteger moveTo = new AtomicInteger(hosts.size() - 1); while (moveFrom.get() < moveTo.get()) { moveStreams(hosts, moveFrom, moveFromLowWaterMark, moveTo, moveToHighWaterMark, rebalanceRateLimiter); moveFrom.incrementAndGet(); } } } finally { for (Host host : hosts) { host.close(); } } }
From source file:io.fabric8.kubernetes.client.dsl.internal.RollingUpdater.java
/** * Lets wait until there are enough Ready pods of the given RC *///from w ww . java 2 s . com private void waitUntilPodsAreReady(final T obj, final String namespace, final int requiredPodCount) { final CountDownLatch countDownLatch = new CountDownLatch(1); final AtomicInteger podCount = new AtomicInteger(0); final Runnable readyPodsPoller = new Runnable() { public void run() { PodList podList = listSelectedPods(obj); int count = 0; List<Pod> items = podList.getItems(); for (Pod item : items) { for (PodCondition c : item.getStatus().getConditions()) { if (c.getType().equals("Ready") && c.getStatus().equals("True")) { count++; } } } podCount.set(count); if (count == requiredPodCount) { countDownLatch.countDown(); } } }; ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); ScheduledFuture poller = executor.scheduleWithFixedDelay(readyPodsPoller, 0, 1, TimeUnit.SECONDS); ScheduledFuture logger = executor.scheduleWithFixedDelay(new Runnable() { @Override public void run() { LOG.debug("Only {}/{} pod(s) ready for {}: {} in namespace: {} seconds so waiting...", podCount.get(), requiredPodCount, obj.getKind(), obj.getMetadata().getName(), namespace); } }, 0, loggingIntervalMillis, TimeUnit.MILLISECONDS); try { countDownLatch.await(rollingTimeoutMillis, TimeUnit.MILLISECONDS); executor.shutdown(); } catch (InterruptedException e) { poller.cancel(true); logger.cancel(true); executor.shutdown(); LOG.warn( "Only {}/{} pod(s) ready for {}: {} in namespace: {} after waiting for {} seconds so giving up", podCount.get(), requiredPodCount, obj.getKind(), obj.getMetadata().getName(), namespace, TimeUnit.MILLISECONDS.toSeconds(rollingTimeoutMillis)); } }