List of usage examples for java.util.concurrent.atomic AtomicLong decrementAndGet
public final long decrementAndGet()
From source file:org.apache.druid.client.cache.MemcachedCache.java
public static MemcachedCache create(final MemcachedCacheConfig config) { final ConcurrentMap<String, AtomicLong> counters = new ConcurrentHashMap<>(); final ConcurrentMap<String, AtomicLong> meters = new ConcurrentHashMap<>(); final AbstractMonitor monitor = new AbstractMonitor() { final AtomicReference<Map<String, Long>> priorValues = new AtomicReference<Map<String, Long>>( new HashMap<String, Long>()); @Override//w ww . j av a 2 s . com public boolean doMonitor(ServiceEmitter emitter) { final Map<String, Long> priorValues = this.priorValues.get(); final Map<String, Long> currentValues = getCurrentValues(); final ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder(); for (Map.Entry<String, Long> entry : currentValues.entrySet()) { emitter.emit(builder.setDimension("memcached metric", entry.getKey()) .build("query/cache/memcached/total", entry.getValue())); final Long prior = priorValues.get(entry.getKey()); if (prior != null) { emitter.emit(builder.setDimension("memcached metric", entry.getKey()) .build("query/cache/memcached/delta", entry.getValue() - prior)); } } if (!this.priorValues.compareAndSet(priorValues, currentValues)) { log.error("Prior value changed while I was reporting! updating anyways"); this.priorValues.set(currentValues); } return true; } private Map<String, Long> getCurrentValues() { final ImmutableMap.Builder<String, Long> builder = ImmutableMap.builder(); for (Map.Entry<String, AtomicLong> entry : counters.entrySet()) { builder.put(entry.getKey(), entry.getValue().get()); } for (Map.Entry<String, AtomicLong> entry : meters.entrySet()) { builder.put(entry.getKey(), entry.getValue().get()); } return builder.build(); } }; try { LZ4Transcoder transcoder = new LZ4Transcoder(config.getMaxObjectSize()); // always use compression transcoder.setCompressionThreshold(0); OperationQueueFactory opQueueFactory; long maxQueueBytes = config.getMaxOperationQueueSize(); if (maxQueueBytes > 0) { opQueueFactory = new MemcachedOperationQueueFactory(maxQueueBytes); } else { opQueueFactory = new LinkedOperationQueueFactory(); } final Predicate<String> interesting = new Predicate<String>() { // See net.spy.memcached.MemcachedConnection.registerMetrics() private final Set<String> interestingMetrics = ImmutableSet.of( "[MEM] Reconnecting Nodes (ReconnectQueue)", //"[MEM] Shutting Down Nodes (NodesToShutdown)", // Busted "[MEM] Request Rate: All", "[MEM] Average Bytes written to OS per write", "[MEM] Average Bytes read from OS per read", "[MEM] Average Time on wire for operations (s)", "[MEM] Response Rate: All (Failure + Success + Retry)", "[MEM] Response Rate: Retry", "[MEM] Response Rate: Failure", "[MEM] Response Rate: Success"); @Override public boolean apply(@Nullable String input) { return input != null && interestingMetrics.contains(input); } }; final MetricCollector metricCollector = new MetricCollector() { @Override public void addCounter(String name) { if (!interesting.apply(name)) { return; } counters.putIfAbsent(name, new AtomicLong(0L)); if (log.isDebugEnabled()) { log.debug("Add Counter [%s]", name); } } @Override public void removeCounter(String name) { if (log.isDebugEnabled()) { log.debug("Ignoring request to remove [%s]", name); } } @Override public void incrementCounter(String name) { if (!interesting.apply(name)) { return; } AtomicLong counter = counters.get(name); if (counter == null) { counters.putIfAbsent(name, new AtomicLong(0)); counter = counters.get(name); } counter.incrementAndGet(); if (log.isDebugEnabled()) { log.debug("Increment [%s]", name); } } @Override public void incrementCounter(String name, int amount) { if (!interesting.apply(name)) { return; } AtomicLong counter = counters.get(name); if (counter == null) { counters.putIfAbsent(name, new AtomicLong(0)); counter = counters.get(name); } counter.addAndGet(amount); if (log.isDebugEnabled()) { log.debug("Increment [%s] %d", name, amount); } } @Override public void decrementCounter(String name) { if (!interesting.apply(name)) { return; } AtomicLong counter = counters.get(name); if (counter == null) { counters.putIfAbsent(name, new AtomicLong(0)); counter = counters.get(name); } counter.decrementAndGet(); if (log.isDebugEnabled()) { log.debug("Decrement [%s]", name); } } @Override public void decrementCounter(String name, int amount) { if (!interesting.apply(name)) { return; } AtomicLong counter = counters.get(name); if (counter == null) { counters.putIfAbsent(name, new AtomicLong(0L)); counter = counters.get(name); } counter.addAndGet(-amount); if (log.isDebugEnabled()) { log.debug("Decrement [%s] %d", name, amount); } } @Override public void addMeter(String name) { if (!interesting.apply(name)) { return; } meters.putIfAbsent(name, new AtomicLong(0L)); if (log.isDebugEnabled()) { log.debug("Adding meter [%s]", name); } } @Override public void removeMeter(String name) { if (!interesting.apply(name)) { return; } if (log.isDebugEnabled()) { log.debug("Ignoring request to remove meter [%s]", name); } } @Override public void markMeter(String name) { if (!interesting.apply(name)) { return; } AtomicLong meter = meters.get(name); if (meter == null) { meters.putIfAbsent(name, new AtomicLong(0L)); meter = meters.get(name); } meter.incrementAndGet(); if (log.isDebugEnabled()) { log.debug("Increment counter [%s]", name); } } @Override public void addHistogram(String name) { log.debug("Ignoring add histogram [%s]", name); } @Override public void removeHistogram(String name) { log.debug("Ignoring remove histogram [%s]", name); } @Override public void updateHistogram(String name, int amount) { log.debug("Ignoring update histogram [%s]: %d", name, amount); } }; final ConnectionFactory connectionFactory = new MemcachedCustomConnectionFactoryBuilder() // 1000 repetitions gives us good distribution with murmur3_128 // (approx < 5% difference in counts across nodes, with 5 cache nodes) .setKetamaNodeRepetitions(1000).setHashAlg(MURMUR3_128) .setProtocol(ConnectionFactoryBuilder.Protocol .valueOf(StringUtils.toUpperCase(config.getProtocol()))) .setLocatorType( ConnectionFactoryBuilder.Locator.valueOf(StringUtils.toUpperCase(config.getLocator()))) .setDaemon(true).setFailureMode(FailureMode.Cancel).setTranscoder(transcoder) .setShouldOptimize(true).setOpQueueMaxBlockTime(config.getTimeout()) .setOpTimeout(config.getTimeout()).setReadBufferSize(config.getReadBufferSize()) .setOpQueueFactory(opQueueFactory).setMetricCollector(metricCollector) .setEnableMetrics(MetricType.DEBUG) // Not as scary as it sounds .build(); final List<InetSocketAddress> hosts = AddrUtil.getAddresses(config.getHosts()); final Supplier<ResourceHolder<MemcachedClientIF>> clientSupplier; if (config.getNumConnections() > 1) { clientSupplier = new MemcacheClientPool(config.getNumConnections(), new Supplier<MemcachedClientIF>() { @Override public MemcachedClientIF get() { try { return new MemcachedClient(connectionFactory, hosts); } catch (IOException e) { log.error(e, "Unable to create memcached client"); throw Throwables.propagate(e); } } }); } else { clientSupplier = Suppliers .ofInstance(StupidResourceHolder.create(new MemcachedClient(connectionFactory, hosts))); } return new MemcachedCache(clientSupplier, config, monitor); } catch (IOException e) { throw Throwables.propagate(e); } }
From source file:org.apache.solr.cloud.TestStressInPlaceUpdates.java
@Test @ShardsFixed(num = 3)//from ww w.java 2 s .com public void stressTest() throws Exception { waitForRecoveriesToFinish(true); this.leaderClient = getClientForLeader(); assertNotNull("Couldn't obtain client for the leader of the shard", this.leaderClient); final int commitPercent = 5 + random().nextInt(20); final int softCommitPercent = 30 + random().nextInt(75); // what percent of the commits are soft final int deletePercent = 4 + random().nextInt(25); final int deleteByQueryPercent = random().nextInt(8); final int ndocs = atLeast(5); int nWriteThreads = 5 + random().nextInt(25); int fullUpdatePercent = 5 + random().nextInt(50); // query variables final int percentRealtimeQuery = 75; // number of cumulative read/write operations by all threads final AtomicLong operations = new AtomicLong(25000); int nReadThreads = 5 + random().nextInt(25); /** // testing final int commitPercent = 5; final int softCommitPercent = 100; // what percent of the commits are soft final int deletePercent = 0; final int deleteByQueryPercent = 50; final int ndocs = 10; int nWriteThreads = 10; final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers // query variables final int percentRealtimeQuery = 101; final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total int nReadThreads = 10; int fullUpdatePercent = 20; **/ log.info("{}", Arrays.asList("commitPercent", commitPercent, "softCommitPercent", softCommitPercent, "deletePercent", deletePercent, "deleteByQueryPercent", deleteByQueryPercent, "ndocs", ndocs, "nWriteThreads", nWriteThreads, "percentRealtimeQuery", percentRealtimeQuery, "operations", operations, "nReadThreads", nReadThreads)); initModel(ndocs); List<Thread> threads = new ArrayList<>(); for (int i = 0; i < nWriteThreads; i++) { Thread thread = new Thread("WRITER" + i) { Random rand = new Random(random().nextInt()); @Override public void run() { try { while (operations.decrementAndGet() > 0) { int oper = rand.nextInt(100); if (oper < commitPercent) { Map<Integer, DocInfo> newCommittedModel; long version; synchronized (TestStressInPlaceUpdates.this) { // take a snapshot of the model // this is safe to do w/o synchronizing on the model because it's a ConcurrentHashMap newCommittedModel = new HashMap<>(model); version = snapshotCount++; int chosenClientIndex = rand.nextInt(clients.size()); if (rand.nextInt(100) < softCommitPercent) { log.info("softCommit start"); clients.get(chosenClientIndex).commit(true, true, true); log.info("softCommit end"); } else { log.info("hardCommit start"); clients.get(chosenClientIndex).commit(); log.info("hardCommit end"); } // install this model snapshot only if it's newer than the current one if (version >= committedModelClock) { if (VERBOSE) { log.info("installing new committedModel version={}", committedModelClock); } clientIndexUsedForCommit = chosenClientIndex; committedModel = newCommittedModel; committedModelClock = version; } } continue; } int id; if (rand.nextBoolean()) { id = rand.nextInt(ndocs); } else { id = lastId; // reuse the last ID half of the time to force more race conditions } // set the lastId before we actually change it sometimes to try and // uncover more race conditions between writing and reading boolean before = rand.nextBoolean(); if (before) { lastId = id; } DocInfo info = model.get(id); // yield after getting the next version to increase the odds of updates happening out of order if (rand.nextBoolean()) Thread.yield(); if (oper < commitPercent + deletePercent + deleteByQueryPercent) { final boolean dbq = (oper >= commitPercent + deletePercent); final String delType = dbq ? "DBI" : "DBQ"; log.info("{} id {}: {}", delType, id, info); Long returnedVersion = null; try { returnedVersion = deleteDocAndGetVersion(Integer.toString(id), params("_version_", Long.toString(info.version)), dbq); log.info(delType + ": Deleting id=" + id + ", version=" + info.version + ". Returned version=" + returnedVersion); } catch (RuntimeException e) { if (e.getMessage() != null && e.getMessage().contains("version conflict") || e.getMessage() != null && e.getMessage().contains("Conflict")) { // Its okay for a leader to reject a concurrent request log.warn("Conflict during {}, rejected id={}, {}", delType, id, e); returnedVersion = null; } else { throw e; } } // only update model if update had no conflict & the version is newer synchronized (model) { DocInfo currInfo = model.get(id); if (null != returnedVersion && (Math.abs(returnedVersion.longValue()) > Math .abs(currInfo.version))) { model.put(id, new DocInfo(returnedVersion.longValue(), 0, 0)); } } } else { int val1 = info.intFieldValue; long val2 = info.longFieldValue; int nextVal1 = val1; long nextVal2 = val2; int addOper = rand.nextInt(100); Long returnedVersion; if (addOper < fullUpdatePercent || info.version <= 0) { // if document was never indexed or was deleted // FULL UPDATE nextVal1 = Primes.nextPrime(val1 + 1); nextVal2 = nextVal1 * 1000000000l; try { returnedVersion = addDocAndGetVersion("id", id, "title_s", "title" + id, "val1_i_dvo", nextVal1, "val2_l_dvo", nextVal2, "_version_", info.version); log.info("FULL: Writing id=" + id + ", val=[" + nextVal1 + "," + nextVal2 + "], version=" + info.version + ", Prev was=[" + val1 + "," + val2 + "]. Returned version=" + returnedVersion); } catch (RuntimeException e) { if (e.getMessage() != null && e.getMessage().contains("version conflict") || e.getMessage() != null && e.getMessage().contains("Conflict")) { // Its okay for a leader to reject a concurrent request log.warn("Conflict during full update, rejected id={}, {}", id, e); returnedVersion = null; } else { throw e; } } } else { // PARTIAL nextVal2 = val2 + val1; try { returnedVersion = addDocAndGetVersion("id", id, "val2_l_dvo", map("inc", String.valueOf(val1)), "_version_", info.version); log.info("PARTIAL: Writing id=" + id + ", val=[" + nextVal1 + "," + nextVal2 + "], version=" + info.version + ", Prev was=[" + val1 + "," + val2 + "]. Returned version=" + returnedVersion); } catch (RuntimeException e) { if (e.getMessage() != null && e.getMessage().contains("version conflict") || e.getMessage() != null && e.getMessage().contains("Conflict")) { // Its okay for a leader to reject a concurrent request log.warn("Conflict during partial update, rejected id={}, {}", id, e); } else if (e.getMessage() != null && e.getMessage().contains("Document not found for update.") && e.getMessage().contains("id=" + id)) { log.warn( "Attempted a partial update for a recently deleted document, rejected id={}, {}", id, e); } else { throw e; } returnedVersion = null; } } // only update model if update had no conflict & the version is newer synchronized (model) { DocInfo currInfo = model.get(id); if (null != returnedVersion && (Math.abs(returnedVersion.longValue()) > Math .abs(currInfo.version))) { model.put(id, new DocInfo(returnedVersion.longValue(), nextVal1, nextVal2)); } } } if (!before) { lastId = id; } } } catch (Throwable e) { operations.set(-1L); log.error("", e); throw new RuntimeException(e); } } }; threads.add(thread); } // Read threads for (int i = 0; i < nReadThreads; i++) { Thread thread = new Thread("READER" + i) { Random rand = new Random(random().nextInt()); @SuppressWarnings("unchecked") @Override public void run() { try { while (operations.decrementAndGet() >= 0) { // bias toward a recently changed doc int id = rand.nextInt(100) < 25 ? lastId : rand.nextInt(ndocs); // when indexing, we update the index, then the model // so when querying, we should first check the model, and then the index boolean realTime = rand.nextInt(100) < percentRealtimeQuery; DocInfo expected; if (realTime) { expected = model.get(id); } else { synchronized (TestStressInPlaceUpdates.this) { expected = committedModel.get(id); } } if (VERBOSE) { log.info("querying id {}", id); } ModifiableSolrParams params = new ModifiableSolrParams(); if (realTime) { params.set("wt", "json"); params.set("qt", "/get"); params.set("ids", Integer.toString(id)); } else { params.set("wt", "json"); params.set("q", "id:" + Integer.toString(id)); params.set("omitHeader", "true"); } int clientId = rand.nextInt(clients.size()); if (!realTime) clientId = clientIndexUsedForCommit; QueryResponse response = clients.get(clientId).query(params); if (response.getResults().size() == 0) { // there's no info we can get back with a delete, so not much we can check without further synchronization } else if (response.getResults().size() == 1) { final SolrDocument actual = response.getResults().get(0); final String msg = "Realtime=" + realTime + ", expected=" + expected + ", actual=" + actual; assertNotNull(msg, actual); final Long foundVersion = (Long) actual.getFieldValue("_version_"); assertNotNull(msg, foundVersion); assertTrue(msg + "... solr doc has non-positive version???", 0 < foundVersion.longValue()); final Integer intVal = (Integer) actual.getFieldValue("val1_i_dvo"); assertNotNull(msg, intVal); final Long longVal = (Long) actual.getFieldValue("val2_l_dvo"); assertNotNull(msg, longVal); assertTrue(msg + " ...solr returned older version then model. " + "should not be possible given the order of operations in writer threads", Math.abs(expected.version) <= foundVersion.longValue()); if (foundVersion.longValue() == expected.version) { assertEquals(msg, expected.intFieldValue, intVal.intValue()); assertEquals(msg, expected.longFieldValue, longVal.longValue()); } // Some things we can assert about any Doc returned from solr, // even if it's newer then our (expected) model information... assertTrue(msg + " ...how did a doc in solr get a non positive intVal?", 0 < intVal); assertTrue(msg + " ...how did a doc in solr get a non positive longVal?", 0 < longVal); assertEquals(msg + " ...intVal and longVal in solr doc are internally (modulo) inconsistent w/eachother", 0, (longVal % intVal)); // NOTE: when foundVersion is greater then the version read from the model, // it's not possible to make any assertions about the field values in solr relative to the // field values in the model -- ie: we can *NOT* assert expected.longFieldVal <= doc.longVal // // it's tempting to think that this would be possible if we changed our model to preserve the // "old" valuess when doing a delete, but that's still no garuntee because of how oportunistic // concurrency works with negative versions: When adding a doc, we can assert that it must not // exist with version<0, but we can't assert that the *reason* it doesn't exist was because of // a delete with the specific version of "-42". // So a wrtier thread might (1) prep to add a doc for the first time with "intValue=1,_version_=-1", // and that add may succeed and (2) return some version X which is put in the model. but // inbetween #1 and #2 other threads may have added & deleted the doc repeatedly, updating // the model with intValue=7,_version_=-42, and a reader thread might meanwhile read from the // model before #2 and expect intValue=5, but get intValue=1 from solr (with a greater version) } else { fail(String.format(Locale.ENGLISH, "There were more than one result: {}", response)); } } } catch (Throwable e) { operations.set(-1L); log.error("", e); throw new RuntimeException(e); } } }; threads.add(thread); } // Start all threads for (Thread thread : threads) { thread.start(); } for (Thread thread : threads) { thread.join(); } { // final pass over uncommitted model with RTG for (SolrClient client : clients) { for (Map.Entry<Integer, DocInfo> entry : model.entrySet()) { final Integer id = entry.getKey(); final DocInfo expected = entry.getValue(); final SolrDocument actual = client.getById(id.toString()); String msg = "RTG: " + id + "=" + expected; if (null == actual) { // a deleted or non-existent document // sanity check of the model agrees... assertTrue(msg + " is deleted/non-existent in Solr, but model has non-neg version", expected.version < 0); assertEquals(msg + " is deleted/non-existent in Solr", expected.intFieldValue, 0); assertEquals(msg + " is deleted/non-existent in Solr", expected.longFieldValue, 0); } else { msg = msg + " <==VS==> " + actual; assertEquals(msg, expected.intFieldValue, actual.getFieldValue("val1_i_dvo")); assertEquals(msg, expected.longFieldValue, actual.getFieldValue("val2_l_dvo")); assertEquals(msg, expected.version, actual.getFieldValue("_version_")); assertTrue(msg + " doc exists in solr, but version is negative???", 0 < expected.version); } } } } { // do a final search and compare every result with the model // because commits don't provide any sort of concrete versioning (or optimistic concurrency constraints) // there's no way to garuntee that our committedModel matches what was in Solr at the time of the last commit. // It's possible other threads made additional writes to solr before the commit was processed, but after // the committedModel variable was assigned it's new value. // // what we can do however, is commit all completed updates, and *then* compare solr search results // against the (new) committed model.... waitForThingsToLevelOut(30); // NOTE: this does an automatic commit for us & ensures replicas are up to date committedModel = new HashMap<>(model); // first, prune the model of any docs that have negative versions // ie: were never actually added, or were ultimately deleted. for (int i = 0; i < ndocs; i++) { DocInfo info = committedModel.get(i); if (info.version < 0) { // first, a quick sanity check of the model itself... assertEquals("Inconsistent int value in model for deleted doc" + i + "=" + info, 0, info.intFieldValue); assertEquals("Inconsistent long value in model for deleted doc" + i + "=" + info, 0L, info.longFieldValue); committedModel.remove(i); } } for (SolrClient client : clients) { QueryResponse rsp = client.query(params("q", "*:*", "sort", "id asc", "rows", ndocs + "")); for (SolrDocument actual : rsp.getResults()) { final Integer id = Integer.parseInt(actual.getFieldValue("id").toString()); final DocInfo expected = committedModel.get(id); assertNotNull("Doc found but missing/deleted from model: " + actual, expected); final String msg = "Search: " + id + "=" + expected + " <==VS==> " + actual; assertEquals(msg, expected.intFieldValue, actual.getFieldValue("val1_i_dvo")); assertEquals(msg, expected.longFieldValue, actual.getFieldValue("val2_l_dvo")); assertEquals(msg, expected.version, actual.getFieldValue("_version_")); assertTrue(msg + " doc exists in solr, but version is negative???", 0 < expected.version); // also sanity check the model (which we already know matches the doc) assertEquals("Inconsistent (modulo) values in model for id " + id + "=" + expected, 0, (expected.longFieldValue % expected.intFieldValue)); } assertEquals(committedModel.size(), rsp.getResults().getNumFound()); } } }
From source file:org.petalslink.dsb.federation.core.server.DefaultPropagationStrategy.java
/** * @param query/*from ww w. j a v a 2 s.c om*/ * @param id * @param counter * @param federationClient */ private void submitLookup(final EndpointQuery query, final String id, final AtomicLong counter, final org.petalslink.dsb.federation.core.api.FederationClient federationClient) { Thread t = new Thread() { @Override public void run() { if (logger.isDebugEnabled()) { logger.debug("Client " + federationClient.getName() + " with callback " + federationClient.getCallbackURL()); } try { // TODO : Only the federation server can talk to this // client! DefaultPropagationStrategy.this.federationServer.getClientManager() .getClient(federationClient.getCallbackURL()) .lookup(query, DefaultPropagationStrategy.this.federationServer.getName(), id); } catch (FederationException e) { counter.decrementAndGet(); } } }; this.executorService.submit(t); }
From source file:org.petalslink.dsb.federation.core.server.DefaultPropagationStrategy.java
/** * {@inheritDoc}/*ww w.j av a 2 s .c om*/ */ public void lookupReply(Set<ServiceEndpoint> endpoints, String clientId, String id) throws FederationException { if (logger.isInfoEnabled()) { logger.info("Got a lookupReply call from client = '" + clientId + "'"); } // aggregate response... if (endpoints != null) { // update the endpoints location... for (ServiceEndpoint serviceEndpoint : endpoints) { // add the current client ID from response to the domain path so // that the call to the endpoint can be routed to the right // federation client. serviceEndpoint.setSubdomainLocation(clientId + "/" + serviceEndpoint.getSubdomainLocation()); } Set<ServiceEndpoint> endpointsBuffer = this.endpoints.get(id); if (endpointsBuffer != null) { endpointsBuffer.addAll(endpoints); } } AtomicLong counter = this.latches.get(id); long remain = counter.decrementAndGet(); if (logger.isInfoEnabled()) { logger.info("Waiting for " + remain + " more response"); } if (remain <= 0) { this.latches.remove(id); if (this.idClientMap.get(id) != null) { // this is a response... So get the initial client and send it // back // the response String initialClient = this.idClientMap.remove(id); if (logger.isInfoEnabled()) { logger.info("This is a response by " + clientId + " for initial client " + initialClient); } // send the response to the client this.lookupReply2(this.endpoints.remove(id), initialClient, id); } else { // Failure, this is a one way call! if (logger.isInfoEnabled()) { logger.info("Failure, can not find a client..."); } } } }