Java tutorial
/** * Copyright (C) 2016 Instacount Inc. (developers@instacount.io) * <p/> * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * <p/> * http://www.apache.org/licenses/LICENSE-2.0 * <p/> * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package io.instacount.appengine.counter.service; import java.math.BigInteger; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Random; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; import io.instacount.appengine.counter.Counter; import io.instacount.appengine.counter.CounterBuilder; import io.instacount.appengine.counter.CounterOperation; import io.instacount.appengine.counter.CounterOperation.CounterOperationType; import io.instacount.appengine.counter.data.CounterData; import io.instacount.appengine.counter.data.CounterData.CounterIndexes; import io.instacount.appengine.counter.data.CounterData.CounterStatus; import io.instacount.appengine.counter.data.CounterShardData; import io.instacount.appengine.counter.data.CounterShardOperationData; import io.instacount.appengine.counter.exceptions.CounterExistsException; import io.instacount.appengine.counter.exceptions.CounterNotMutableException; import io.instacount.appengine.counter.exceptions.InvalidCounterNameException; import io.instacount.appengine.counter.exceptions.NoCounterExistsException; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; import org.apache.commons.lang3.StringUtils; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import com.google.appengine.api.capabilities.CapabilitiesService; import com.google.appengine.api.capabilities.CapabilitiesServiceFactory; import com.google.appengine.api.datastore.DatastoreFailureException; import com.google.appengine.api.datastore.DatastoreTimeoutException; import com.google.appengine.api.memcache.MemcacheService; import com.google.appengine.api.memcache.MemcacheService.IdentifiableValue; import com.google.appengine.api.memcache.MemcacheService.SetPolicy; import com.google.appengine.api.memcache.MemcacheServiceException; import com.google.appengine.api.memcache.MemcacheServiceFactory; import com.google.appengine.api.taskqueue.Queue; import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.api.taskqueue.TaskOptions; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.math.LongMath; import com.googlecode.objectify.Key; import com.googlecode.objectify.ObjectifyService; import com.googlecode.objectify.VoidWork; import com.googlecode.objectify.Work; /** * <p> * A durable implementation of a {@link ShardedCounterService} that provides counter increment, decrement, accessor, and * delete functionality. This implementation is is backed by one or more Datastore "shard" entities which each hold a * discrete count in order to provide for high throughput. When aggregated, the sum total of all CounterShard entity * counts is the value of the counter. See the google link below for more details on Sharded Counters in Appengine. Note * that CounterShards may not go negative depending on the configuration of the counter. Also, note that there is no * difference between a counter being "zero" and a counter not existing. As such, there is no concept of "creating" a * counter, and deleting a counter will actually just remove all shards for a given counter, thereby resetting the * counter to 0. * </p> * <p> * <b>Shard Number Adjustments</b> This implementation is capable of incrementing/decrementing various counter shard * counts but does not automatically increase or reduce the <b>number</b> of shards for a given counter in response to * load. * </p> * <p> * <b>Incrementing a Counter</b><br/> * When incrementing, a random shard is selected to prevent a single shard from being written to too frequently.<br/> * </p> * <p> * <b>Decrementing a Counter</b><br/> * This implementation does not support negative counts, so CounterShard counts can not go below zero. Thus, when * decrementing, a random shard is selected to prevent a single shard from being written to too frequently. However, if * a particular shard cannot be decremented then other shards are tried until all shards have been tried. If no shard * can be decremented, then the decrement function is considered complete, even though nothing was decremented. Because * of this, a request to reduce a counter by more than its available count will succeed with a lesser count having been * reduced. * </p> * <p> * <b>Getting the Count</b> <br/> * Aggregate ounter lookups are first attempted using Memcache. If the counter value is not in the cache, then the * shards are read from the datastore and accumulated to reconstruct the current count. This operation has a cost of * O(numShards), or O(N). Increase the number of shards to improve counter increment throughput, but beware that this * has a cost - it makes counter lookups from the Datastore more expensive.<br/> * </p> * <p> * <b>Throughput</b><br/> * As an upper-bound calculation of throughput as it relates to shard-count, the Psy "Gangham Style" youtube video * (arguably one of the most viral videos of all time) reached 750m views in approximately 60 days. If that video's * 'hit-counter' was using appengine-counter as its underlying implementation, then the counter would have needed to * sustain an increment rate of 145 updates per second for 60 days. Since each CounterShard could have provided up to 5 * updates per second (this seems to be the average indicated by the appengine team and in various documentation), then * the counter would have required at least 29 CounterShard entities. While something like a high-traffic hit-counter * could be be implemented using appengine-counter, a HyperLogLog counter would be a better choice (see * http://antirez.com/news/75 for more details). * </p> * <p> * All datastore operations are performed using Objectify. * </p> * <b>Future Improvements</b><br/> * <ul> * <li><b>CounterShard Expansion</b>: A shard-expansion mechanism can be envisioned to increase the number of * CounterShard entities for a particular Counter when load increases to a specified incrementAmount for a given * Counter.</li> * <li><b>CounterShard Contraction</b>: A shard-reduction mechanism can be envisioned to aggregate multiple shards (and * their counts) into fewer shards to improve datastore counter lookup performance when Counter load falls below some * threshold.</li> * <li><b>Counter Reset</b>: Reset a counter to zero by resetting all counter shards 'counts' to zero. This would need * to be, by nature of this implementation, async.</li> * </ul> * * @author David Fuelling * @see "https://developers.google.com/appengine/articles/sharding_counters" */ public class ShardedCounterServiceImpl implements ShardedCounterService { private static final Logger logger = Logger.getLogger(ShardedCounterServiceImpl.class.getName()); // Helper constant for counterName keys. public static final String COUNTER_NAME = "counterName"; // This indicates that cache should be used, and is equivalent to skipCache=false private static final boolean USE_CACHE = false; private static final boolean SKIP_CACHE = true; /** * A random number generating, for distributing writes across shards. */ protected final Random generator = new Random(); protected final MemcacheService memcacheService; protected final ShardedCounterServiceConfiguration config; protected final CounterNameValidator counterNameValidator; // ///////////////////////////// // Constructors // ///////////////////////////// /** * Default Constructor for Dependency-Injection that uses {@link MemcacheServiceFactory} to construct the * {@link MemcacheService} and {@link CapabilitiesServiceFactory} to construct the {@link CapabilitiesService}. * dependency for this service. */ public ShardedCounterServiceImpl() { this(MemcacheServiceFactory.getMemcacheService()); } /** * Default Constructor for Dependency-Injection that uses a default number of counter shards (set to 1) and a * default configuration per {@link ShardedCounterServiceConfiguration#defaultConfiguration}. * * @param memcacheService */ public ShardedCounterServiceImpl(final MemcacheService memcacheService) { this(memcacheService, ShardedCounterServiceConfiguration.defaultConfiguration()); } /** * Default Constructor for Dependency-Injection. * * @param memcacheService * @param config The configuration for this service */ public ShardedCounterServiceImpl(final MemcacheService memcacheService, final ShardedCounterServiceConfiguration config) { this(memcacheService, config, new DefaultCounterNameValidator()); } /** * Default Constructor for Dependency-Injection. * * @param memcacheService * @param config The configuration for this service */ public ShardedCounterServiceImpl(final MemcacheService memcacheService, final ShardedCounterServiceConfiguration config, final CounterNameValidator counterNameValidator) { Preconditions.checkNotNull(memcacheService, "Invalid memcacheService!"); this.memcacheService = memcacheService; Preconditions.checkNotNull(config); this.config = config; Preconditions.checkArgument(config.getNumInitialShards() > 0, "Number of Shards for a new CounterData must be greater than 0!"); if (config.getRelativeUrlPathForDeleteTaskQueue() != null) { // The relativeUrlPathForDeleteTaskQueue may be null, but if it's non-null, then it must not be blank. Preconditions.checkArgument(!StringUtils.isBlank(config.getRelativeUrlPathForDeleteTaskQueue()), "Must be null (for the Default Queue) or a non-blank String!"); } Preconditions.checkNotNull(counterNameValidator); this.counterNameValidator = counterNameValidator; } // ///////////////////////////// // Retrieval Functions // ///////////////////////////// /** * The cache will expire after {@code defaultCounterCountExpiration} seconds, so the counter will be accurate after * a minute because it performs a load from the datastore. * * @param counterName * @return */ @Override public Optional<Counter> getCounter(final String counterName) { return this.getCounter(counterName, USE_CACHE); } @Override public Optional<CounterOperation> getCounterOperation(String counterName, int shardNumber, UUID counterOperationId) { // Load the counterOperationData final Key<CounterData> counterDataKey = CounterData.key(counterName); final Key<CounterShardData> counterShardDataKey = CounterShardData.key(counterDataKey, shardNumber); final Key<CounterShardOperationData> counterShardOperationDataKey = CounterShardOperationData .key(counterShardDataKey, counterOperationId); final CounterShardOperationData counterShardOperationData = ObjectifyService.ofy().load() .key(counterShardOperationDataKey).now(); if (counterShardOperationData == null) { return Optional.absent(); } else { return Optional.<CounterOperation>of(new CounterOperation.Impl(counterShardOperationData)); } } @Override public Counter createCounter(final String counterName) { final CounterData counterData = this.createCounterData(counterName); // No need to call call #getCounter here. The count will be zero, and so we don't need to perform any cache // warm-up (this will be performed on the first read) and performing #getCounter here will load all counter // shards, which is wasteful since they're likely 0. This will be extra wasteful for large-shard counters. return new CounterBuilder(counterData).withCount(BigInteger.ZERO).build(); } /** * The cache will expire after {@code defaultCounterCountExpiration} seconds, so the counter will be accurate after * a minute because it performs a load from the datastore. * * @param counterName * @param skipCache A boolean that allows a caller to skip memcache when retrieving a counter. Set to {@code true} * to load the counter and all of its shards directly from the Datastore. Set to {@code false} to attempt * to load the count from memcache, with fallback to the datastore. * @return */ @Override public Optional<Counter> getCounter(final String counterName, final boolean skipCache) { Preconditions.checkNotNull(counterName); // This method always load the CounterData from the Datastore (or its Objectify cache), but sometimes returns // the // cached count value. // ////////////// // ShortCircuit: If nothing is present in the datastore. // ////////////// final Optional<CounterData> optCounterData = this.getCounterData(counterName); if (!optCounterData.isPresent()) { logger.log(Level.FINEST, String.format("Counter '%s' was not found in hte Datastore!", counterName)); return Optional.absent(); } final CounterData counterData = optCounterData.get(); // ////////////// // ShortCircuit: If the counter is in an indeterminate state, then return its count as 0. // ////////////// if (this.counterStatusYieldsIndeterminateCount(counterData.getCounterStatus())) { logger.log(Level.FINEST, String.format("Counter '%s' was in an indeterminate state. Returning 0!", counterName)); return Optional.of(new CounterBuilder(counterData).withCount(BigInteger.ZERO).build()); } // ////////////// // ShortCircuit: If the counter was found in memcache. // ////////////// final String memCacheKey = this.assembleCounterKeyforMemcache(counterName); if (!skipCache) { final BigInteger cachedCounterCount = this.memcacheSafeGet(memCacheKey); if (cachedCounterCount != null) { // ///////////////////////////////////// // The count was found in memcache, so return it. // ///////////////////////////////////// logger.log(Level.FINEST, String.format("Cache Hit for Counter Named '%s': value=%s", counterName, cachedCounterCount)); return Optional.of(new CounterBuilder(counterData).withCount(cachedCounterCount).build()); } else { logger.log(Level.FINE, String.format( "Cache Miss for CounterData Named '%s': value='%s'. Checking Datastore instead!", counterName, cachedCounterCount)); } } // ///////////////////////////////////// // skipCache was true or the count was NOT found in memcache! // ///////////////////////////////////// // Note: No Need to clear the Objectify session cache here because it will be cleared automatically and // repopulated upon every request. logger.log(Level.FINE, String.format("Aggregating counts from '%s' CounterDataShards for CounterData named '%s'!", counterData.getNumShards(), counterData.getName())); // /////////////////// // Assemble a List of CounterShardData Keys to retrieve in parallel! final List<Key<CounterShardData>> keysToLoad = Lists.newArrayList(); for (int i = 0; i < counterData.getNumShards(); i++) { final Key<CounterShardData> counterShardKey = CounterShardData.key(counterData.getTypedKey(), i); keysToLoad.add(counterShardKey); } long sum = 0; // For added performance, we could spawn multiple threads to wait for each value to be returned from the // DataStore, and then aggregate that way. However, the simple summation below is not very expensive, so // creating multiple threads to get each value would probably be overkill. Just let objectify do this for // us. Even though we have to wait for all entities to return before summation begins, the summation is a quick // in-memory operation with a relatively small number of shards, so parallelizing it would likely not increase // performance. // No TX - get is Strongly consistent by default, and we will exceed the TX limit for high-shard-count // counters if we try to do this in a TX. final Map<Key<CounterShardData>, CounterShardData> counterShardDatasMap = ObjectifyService.ofy() .transactionless().load().keys(keysToLoad); final Collection<CounterShardData> counterShardDatas = counterShardDatasMap.values(); for (CounterShardData counterShardData : counterShardDatas) { if (counterShardData != null) { sum += counterShardData.getCount(); } } logger.log(Level.FINE, String.format( "The Datastore is reporting a count of %s for CounterData '%s' count. Resetting memcache " + "count to %s for this counter name.", sum, counterData.getName(), sum)); final BigInteger bdSum = BigInteger.valueOf(sum); try { // This method will only get here if there was nothing in Memcache, or if the caller requested to skip // reading the Counter count from memcache. In these cases, the value in memcache should always be replaced. memcacheService.put(memCacheKey, bdSum, config.getDefaultCounterCountExpiration(), SetPolicy.SET_ALWAYS); } catch (MemcacheServiceException mse) { // Do nothing. The method will still return even though memcache is not available. } return Optional.of(new CounterBuilder(counterData).withCount(bdSum).build()); } /** * NOTE: We don't allow the counter's "count" to be updated by this method. Instead, {@link #increment} and * {@link #decrement} should be used. * * @param incomingCounter */ @Override public void updateCounterDetails(final Counter incomingCounter) { Preconditions.checkNotNull(incomingCounter); // First, assert the counter is in a proper state. If done consistently (i.e., in a TX, then this will function // as an effective CounterData lock). // Second, Update the counter details. ObjectifyService.ofy().transact(new Work<Void>() { @Override public Void run() { // First, load the incomingCounter from the datastore via a transactional get to ensure it has the // proper state. final Optional<CounterData> optCounterData = getCounterData(incomingCounter.getName()); // ////////////// // ShortCircuit: Can't update a counter that doesn't exist! // ////////////// if (!optCounterData.isPresent()) { throw new NoCounterExistsException(incomingCounter.getName()); } final CounterData counterDataInDatastore = optCounterData.get(); // Make sure the stored counter is currently in an updatable state! assertCounterDetailsMutatable(incomingCounter.getName(), counterDataInDatastore.getCounterStatus()); // Make sure the incoming counter status is a validly settable status assertValidExternalCounterStatus(incomingCounter.getName(), incomingCounter.getCounterStatus()); // NOTE: READ_ONLY_COUNT status means the count can't be incremented/decremented. However, it's details // can still be mutated. // NOTE: The counterName/counterId may not change! // Update the Description counterDataInDatastore.setDescription(incomingCounter.getDescription()); // Update the numShards. Aside from setting this value, nothing explicitly needs to happen in the // datastore since shards will be created when a counter in incremented (if the shard doesn't already // exist). However, if the number of shards is being reduced, then throw an exception since this // requires counter shard reduction and some extra thinking. We can't allow the shard-count to go down // unless we collapse the entire counter's shards into a single shard or zero, and it's ambiguous if // this is even required. Note that if we allow this numShards value to decrease without capturing // the count from any of the shards that might no longer be used, then we might lose counts from the // shards that would no longer be factored into the #getCount method. if (incomingCounter.getNumShards() < counterDataInDatastore.getNumShards()) { throw new IllegalArgumentException( "Reducing the number of counter shards is not currently allowed! See https://github.com/instacount/appengine-counter/issues/4 for more details."); } counterDataInDatastore.setNumShards(incomingCounter.getNumShards()); // The Exception above disallows any invalid states. counterDataInDatastore.setCounterStatus(incomingCounter.getCounterStatus()); // Update the CounterDataIndexes counterDataInDatastore.setIndexes(incomingCounter.getIndexes() == null ? CounterIndexes.none() : incomingCounter.getIndexes()); // Update the counter in the datastore. ObjectifyService.ofy().save().entity(counterDataInDatastore).now(); // return null to satisfy Java... return null; } }); } /** * Helper method to determine the random shard number for the mutation operation. If {@code optShardNumber} is * present, then this value will be used so long as it is greater-than or equal to zero. Otherwise, a PRNG will be * used to select the next shard number based upon the current number of shards that are allowed for the current * counter as specified by {@code numShards}. * * @param optShardNumber An {@link Optional} instance of {@link Integer} that specifies the shard number to use, if * present. * @param numShards The number of shards that the mutating counter has available to it for increment/decrement * operations. * @return */ @Override public int determineRandomShardNumber(final Optional<Integer> optShardNumber, final int numShards) { Preconditions.checkArgument(numShards > 0); Preconditions.checkNotNull(optShardNumber); if (optShardNumber.isPresent()) { final int shardNumber = optShardNumber.get(); if (shardNumber >= 0 && shardNumber < numShards) { return optShardNumber.get(); } } // Otherwise... return generator.nextInt(numShards); } // ///////////////////////////// // Increment Functions // ///////////////////////////// @Override public CounterOperation increment(final String counterName, final long amount) { Preconditions.checkNotNull(counterName); Preconditions.checkArgument(!StringUtils.isBlank(counterName)); Preconditions.checkArgument(amount > 0, "Increment amounts must be positive!"); return this.mutateCounterShard(counterName, amount, Optional.<Integer>absent(), UUID.randomUUID()); } @Override public CounterOperation increment(final String counterName, final long amount, final int shardNumber, final UUID incrementOperationId) { Preconditions.checkNotNull(counterName); Preconditions.checkArgument(!StringUtils.isBlank(counterName)); Preconditions.checkArgument(amount > 0, "Increment amounts must be positive!"); Preconditions.checkArgument(shardNumber >= 0); Preconditions.checkNotNull(incrementOperationId); return this.mutateCounterShard(counterName, amount, Optional.of(shardNumber), incrementOperationId); } // ///////////////////////////// // Decrementing Functions // ///////////////////////////// @Override public CounterOperation decrement(final String counterName, final long amount) { Preconditions.checkNotNull(counterName); Preconditions.checkArgument(!StringUtils.isBlank(counterName)); Preconditions.checkArgument(amount > 0, "Decrement amounts must be positive!"); // Perform the Decrement final long decrementAmount = amount * -1L; return this.mutateCounterShard(counterName, decrementAmount, Optional.<Integer>absent(), UUID.randomUUID()); } @Override public CounterOperation decrement(final String counterName, final long amount, final int shardNumber, final UUID decrementOperationUuid) { Preconditions.checkNotNull(counterName); Preconditions.checkArgument(!StringUtils.isBlank(counterName)); Preconditions.checkArgument(amount > 0, "Decrement amounts must be positive!"); Preconditions.checkArgument(shardNumber >= 0); Preconditions.checkNotNull(decrementOperationUuid); final long decrementAmount = amount * -1L; return this.mutateCounterShard(counterName, decrementAmount, Optional.of(shardNumber), decrementOperationUuid); } // The following two methods exists for posterity to see one manner in which non-negative counter enforcement might // be accomplished (though imperfectly). These are not used because it's not possible to perfectly (nor easily) // guarantee non-negative semantics in a sharded-counter because all shards need to be retrieved in order to // accurately gauge the count to determine if it will be negative. Since multiple threads may be operating on a // given CounterShardData object, we would need to lock the entire counter to make this check, which would defeat // the purpose of sharding. After some thought, it's not clear that the "can't go negative" requirement exists for // a sharded counter. Additionally, with the #reset method, clients of appengine-counter could enforce this // requirement in their own ways. // @Override // public CounterOperation decrement(final String counterName, final long amount) // { // Preconditions.checkNotNull(counterName); // Preconditions.checkArgument(!StringUtils.isBlank(counterName)); // Preconditions.checkArgument(amount > 0, "Decrement amounts must be positive!"); // // // A Note About Count-Below-Zero Limiting // // -- // // The mutateCounterShard operation does not have the ability to determine the count of a counter while it's // // performing it's work. Additionally, since multiple shards might be operating upon the same counter at the // // same time, getting the count of a counter and passing it into the mutateCounterShard operation would not // // accurately stop the counter from going below zero. Instead, we perform a preemptive check here to see if // // the counter is currently at zero, and throw an exception if that's the case and this request is attempting // // to decrement below zero. However, this check may occur while other decrement threads have already made their // // way into the mutateCounterShard method (but not completed). Thus, we perform an additional "negative check" // // after the decrement is applied, and optionally reset the counter if it has become errantly negative. // // // Preemptive Negative Check // this.assertCounterCanDecrementBelowZero(counterName, amount); // // // Perform the Decrement // final long decrementAmount = amount * -1L; // final CounterOperation counterOperation = this.mutateCounterShard(counterName, decrementAmount, // Optional.<Integer> absent(), UUID.randomUUID()); // // // Post-Decrement Negative Check // this.resetCounterIfErrantlyNegative(counterName, amount); // // return counterOperation; // } // /** // * Helper method to assert that a counter can decrement below zero. // * // * @param counterName The name of a counter being decremented. // * @param amount The amount of the decrement that is about to be applied to a counter. // * @throws CounterNotMutableException If a counter cannot be decremented by the amount specified by {@code // amount}. // */ // @VisibleForTesting // protected void assertCounterCanDecrementBelowZero(final String counterName, final long amount) // { // Preconditions.checkNotNull(counterName); // // final Optional<Counter> optCounter = this.getCounter(counterName); // Preconditions.checkArgument(optCounter.isPresent(), "This counter should have existed but didn't!"); // final Counter counter = optCounter.get(); // if (!counter.isNegativeCountAllowed() // && (counter.getCount().subtract(BigInteger.valueOf(amount))).compareTo(BigInteger.ZERO) < 0) // { // throw new CounterNotMutableException(counterName, String.format("Cannot decrement counter below zero!")); // } // } // /** // * Helper method to reset a counter to zero if it has a negative count but is configured to not allow the counter // to // * be negative. // * // * @param counterName The name of a counter being decremented. // * @param amount The amount of the decrement that is about to be applied to a counter. // * @throws CounterNotMutableException If a counter was negative but was reset to zero. This exception is chosen // * because it is assumed that a counter became negative as a part of this request (or at least during // * this request). // */ // @VisibleForTesting // protected void resetCounterIfErrantlyNegative(final String counterName, final long amount) // { // Preconditions.checkNotNull(counterName); // // final Optional<Counter> optCounter = this.getCounter(counterName); // Preconditions.checkArgument(optCounter.isPresent(), "This counter should have existed but didn't!"); // final Counter counter = optCounter.get(); // if (!counter.isNegativeCountAllowed() && counter.getCount().compareTo(BigInteger.ZERO) < 0) // { // this.reset(counterName); // // This exception is chosen because it is assumed that a counter became negative as a part of this request // // (or at least during this request). The above call will set the counter to zero. // throw new CounterNotMutableException(counterName, String.format("Cannot decrement counter below zero!")); // } // } @Override public void reset(final String counterName) { Preconditions.checkNotNull(counterName); try { // Update the Counter's Status in a New TX to the RESETTING_STATE and apply the TX. // The "new" TX ensures that no other thread nor parent transaction is performing this operation at the // same time, since if that were the case the updateCounter would fail. This Work returns the number of // counter shards that existed for the counter. Capture the number of shards that exist in the counter // so that the datastore doesn't need to be hit again. final Integer numCounterShards = ObjectifyService.ofy().transactNew(new Work<Integer>() { @Override public Integer run() { // Use the cache here if possible, because the first two short-circuits don't really need accurate // counts to work. final Optional<CounterData> optCounterData = getCounterData(counterName); // ///////////// // ShortCircuit: Do nothing - there's nothing to return. // ///////////// if (!optCounterData.isPresent()) { // The counter was not found, so we can't reset it. throw new NoCounterExistsException(counterName); } else { final CounterData counterData = optCounterData.get(); // Make sure the counter can be put into the RESETTING state! assertCounterDetailsMutatable(counterName, counterData.getCounterStatus()); counterData.setCounterStatus(CounterStatus.RESETTING); ObjectifyService.ofy().save().entity(counterData); return counterData.getNumShards(); } } }); // TODO: Refactor the below code into a transactional enqueing mechansim that either encodes all shards, or // else creates a single task that re-enqueues itself with a decrementing number so that it will run on // every shard and then reset the status of the counter. In this way, "reset" can be async, and work for // shards larger than 25, but since the counter is in the RESETTING state, it won't be allowed to be // mutated. // For now, perform all reset operations in same TX to provide atomic rollback. Since Appengine now support // up to 25 entities in a transaction, this will work for all counters up to 25 shards. ObjectifyService.ofy().transactNew(new VoidWork() { @Override public void vrun() { // For each shard, reset it. Shard Index starts at 0. for (int shardNumber = 0; shardNumber < numCounterShards; shardNumber++) { final ResetCounterShardWork resetWork = new ResetCounterShardWork(counterName, shardNumber); ObjectifyService.ofy().transact(resetWork); } } }); } finally { // Clear the cache to ensure that future callers get the right count. this.memcacheSafeDelete(counterName); // If the reset-shards loop above completes properly, OR if an exception is thrown above, the counter status // needs to be reset to AVAILABLE. In the first case (an exception is thrown) then the reset TX will be // rolled-back, and the counter status needs to become AVAILABLE. In the second case (the counter was // successfully reset) the same counter status update to AVAILABLE needs to occur. ObjectifyService.ofy().transactNew(new VoidWork() { /** * Updates the {@link CounterData} status to be {@link CounterStatus#AVAILABLE}, but only if the current * status is {@link CounterStatus#RESETTING}. */ @Override public void vrun() { final Optional<CounterData> optCounterData = getCounterData(counterName); if (optCounterData.isPresent()) { final CounterData counterData = optCounterData.get(); if (counterData.getCounterStatus() == CounterStatus.RESETTING) { counterData.setCounterStatus(CounterStatus.AVAILABLE); ObjectifyService.ofy().save().entity(counterData).now(); } } } }); } } /** * Does the work of incrementing or decrementing the value of a single shard for the counter named * {@code counterName}. * * @param counterName * @param amount The amount to mutate a counter shard by. This value will be negative for a decrement, and positive * for an increment. * @param incrementOperationId * @param optShardNumber An optionally specified shard number to increment. If not specified, a random shard number * will be chosen. * @return An instance of {@link CounterOperation} with information about the increment/decrement. */ private CounterOperation mutateCounterShard(final String counterName, final long amount, final Optional<Integer> optShardNumber, final UUID incrementOperationId) { // /////////// // Precondition Checks are performed by calling methods. // This get is transactional and strongly consistent, but we perform it here so that the increment doesn't have // to be part of an XG transaction. Since the CounterData isn't expected to mutate that often, we want // increment/decrement operations to be as speedy as possibly. In this way, the IncrementWork can take place in // a non-XG transaction. final CounterDataGetCreateContainer counterDataGetCreateContainer = this .getOrCreateCounterData(counterName); final CounterData counterData = counterDataGetCreateContainer.getCounterData(); // Create the Work to be done for this increment, which will be done inside of a TX. See // "https://developers.google.com/appengine/docs/java/datastore/transactions#Java_Isolation_and_consistency" final Work<CounterOperation> atomicIncrementShardWork = new IncrementShardWork(counterData, incrementOperationId, optShardNumber, amount, counterDataGetCreateContainer.isNewCounterDataCreated()); // Note that this operation is idempotent from the perspective of a ConcurrentModificationException. In that // case, the increment operation will fail and will not have been applied. An Objectify retry of the // increment/decrement will occur, however and a successful increment/decrement will only ever happen once (if // the Appengine datastore is functioning properly). See the Javadoc in the API about DatastoreTimeoutException // or // DatastoreFailureException in cases where transactions have been committed and eventually will be applied // successfully." // We use the "counterShardOperationInTx" to force this thread to wait until the work inside of // "atomicIncrementShardWork" completes. This is because we don't want to increment memcache (below) until after // that operation's transaction successfully commits. final CounterOperation counterShardOperationInTx = ObjectifyService.ofy() .transact(atomicIncrementShardWork); // ///////////////// // Try to increment this counter in memcache atomically, but only if we're not inside of a parent caller's // transaction. If that's the case, then we can't know if the parent TX will fail upon commit, which would // happen after our call to memcache. // ///////////////// if (isParentTransactionActive()) { // If a parent-transaction is active, then don't update memcache. Instead, clear it out since we can't know // if the parent commit will actually stick. this.memcacheSafeDelete(counterName); } else { // Otherwise, try to increment memcache. If the memcache operation fails, it's ok because memcache is merely // a cache of the actual count data, and will eventually become accurate when the cache is reloaded via a // call to #getCount. long amountToMutateCache = counterShardOperationInTx.getAppliedAmount(); if (amount < 0) { amountToMutateCache *= -1L; } this.incrementMemcacheAtomic(counterName, amountToMutateCache); } return counterShardOperationInTx; } // ///////////////////////////// // Counter Deletion Functions // ///////////////////////////// @Override public void delete(final String counterName) { Preconditions.checkNotNull(counterName); Preconditions.checkArgument(!StringUtils.isBlank(counterName)); // Delete the main counter in a new TX... ObjectifyService.ofy().transactNew(new VoidWork() { @Override public void vrun() { // Load in a TX so that two threads don't mark the counter as deleted at the same time. final Key<CounterData> counterDataKey = CounterData.key(counterName); final CounterData counterData = ObjectifyService.ofy().load().key(counterDataKey).now(); if (counterData == null) { // We throw an exception here so that callers can be aware that the deletion failed. In the // task-queue, however, no exception is thrown since failing to delete something that's not there // can be treated the same as succeeding at deleting something that's there. throw new NoCounterExistsException(counterName); } Queue queue; if (config.getDeleteCounterShardQueueName() == null) { queue = QueueFactory.getDefaultQueue(); } else { queue = QueueFactory.getQueue(config.getDeleteCounterShardQueueName()); } // The TaskQueue will delete the counter once all shards are deleted. counterData.setCounterStatus(CounterData.CounterStatus.DELETING); // Call this Async so that the rest of the thread can // continue. Everything will block till commit is called. ObjectifyService.ofy().save().entity(counterData); // Transactionally enqueue this task to the path specified in the constructor (if this is null, then the // default queue will be used). TaskOptions taskOptions = TaskOptions.Builder.withParam(COUNTER_NAME, counterName); if (config.getRelativeUrlPathForDeleteTaskQueue() != null) { taskOptions = taskOptions.url(config.getRelativeUrlPathForDeleteTaskQueue()); } // Kick off a Task to delete the Shards for this CounterData and the CounterData itself, but only if the // overall TX commit succeeds queue.add(taskOptions); } }); } @Override public void onTaskQueueCounterDeletion(final String counterName) { Preconditions.checkNotNull(counterName); // Load in a TX so that two threads don't mark the counter as // deleted at the same time. final Key<CounterData> counterDataKey = CounterData.key(counterName); final CounterData counterData = ObjectifyService.ofy().load().key(counterDataKey).now(); if (counterData == null) { // Nothing to delete...perhaps another task already did the deletion? final String msg = String.format( "No CounterData was found in the Datastore while attempting to delete CounterData named '%s", counterName); getLogger().severe(msg); // Clear this counter from Memcache, just in case. this.memcacheSafeDelete(counterName); return; } else if (counterData.getCounterStatus() != CounterData.CounterStatus.DELETING) { final String msg = String.format( "Can't delete counter '%s' because it is currently not in the DELETING state!", counterName); throw new IllegalArgumentException(msg); } else { // Assemble a list of CounterShard keys, and delete them all in a batch! final Collection<Key<CounterShardData>> counterShardDataKeys = Lists.newArrayList(); for (int i = 0; i < counterData.getNumShards(); i++) { final Key<CounterShardData> counterShardDataKey = CounterShardData.key(counterDataKey, i); counterShardDataKeys.add(counterShardDataKey); // Delete all CounterShardOperationData entries for this Shard. If the taskqueue times out, this // operation will be retried, which will continue deleting the counter operations for this counter. ObjectifyService.ofy().transactionless().delete().type(CounterShardOperationData.class) .parent(counterShardDataKey); } // No TX allowed since there may be many shards. ObjectifyService.ofy().transactionless().delete().keys(counterShardDataKeys).now(); // Delete the CounterData itself...No TX needed. ObjectifyService.ofy().transactionless().delete().key(counterData.getTypedKey()).now(); // Clear this counter from Memcache. this.memcacheSafeDelete(counterName); } } // ////////////////////////////////// // Protected Helpers // ////////////////////////////////// /** * A private implementation of {@link Work} that increments the {@code incrementAmount} of a * {@link CounterShardData} by a specified non-negative {@code incrementAmount}. */ @VisibleForTesting final class IncrementShardWork implements Work<CounterOperation> { private final CounterData counterData; private final UUID counterShardOperationUuid; private final Optional<Integer> optShardNumber; private final long incrementAmount; private final boolean newCounterCreated; /** * Required-Args Constructor. * * @param counterData A {@link CounterData} that contains information about the counter being incremented. * @param counterShardOperationUuid The unique identifier of the job performing the increment. * @param optShardNumber An optionally supplied shard number that should an increment/decrement should be * applied to. If specified as {@link Optional#absent()} , then a random shard will be selected. * @param incrementAmount A long representing the amount of the increment to be applied. * @param newCounterCreated {@code true} if a new counter was created to perform this operation; {@code false} * otherwise. */ IncrementShardWork(final CounterData counterData, final UUID counterShardOperationUuid, final Optional<Integer> optShardNumber, final long incrementAmount, boolean newCounterCreated) { Preconditions.checkNotNull(counterData); this.counterData = counterData; Preconditions.checkNotNull(counterShardOperationUuid); this.counterShardOperationUuid = counterShardOperationUuid; Preconditions.checkArgument(incrementAmount != 0, "Counter increment amounts must be positive or negative numbers!"); this.incrementAmount = incrementAmount; Preconditions.checkNotNull(optShardNumber); this.optShardNumber = optShardNumber; this.newCounterCreated = newCounterCreated; } /** * NOTE: In order for this to work properly, the CounterShardData must be gotten, created, and updated all in * the same transaction in order to remain consistent (in other words, it must be atomic). * * @return An instance of {@link CounterOperation} containing the results of this increment. * @throws ArithmeticException if after adding {@code incrementAmount} to the current shard's amount, an * Overflow or Underflow would occur. */ @Override public CounterOperation run() { final String counterName = counterData.getName(); // Callers may specify a random shard number. If not specified, then choose the shard randomly from the // available shards. final int shardNumber = determineRandomShardNumber(optShardNumber, counterData.getNumShards()); // If this operation exists in the datastore, then don't apply it again. This is not expensive (i.e., not // XG) because this lookup will be for an entity in the same entity group as the ultimate shard increment // operation. In fact, they're the exact same entity. if (counterShardOperationAlreadyExists(counterData.getTypedKey(), shardNumber)) { final String msg = String.format( "CounterShardOperation '%s' for CounterShard %s on Counter '%s' already exists and will not be" + " applied again! CounterData was: %s", counterShardOperationUuid, shardNumber, counterName, counterData); throw new IllegalArgumentException(msg); } // Increments/Decrements can only occur on Counters with a counterStatus of AVAILABLE. assertCounterAmountMutatable(counterData.getName(), counterData.getCounterStatus()); final Key<CounterShardData> counterShardDataKey = CounterShardData.key(counterData.getTypedKey(), shardNumber); // Load the Shard from the DS. CounterShardData counterShardData = ObjectifyService.ofy().load().key(counterShardDataKey).now(); if (counterShardData == null) { // Create a new CounterShardData in memory. No need to preemptively save to the Datastore until the very // end. counterShardData = new CounterShardData(counterName, shardNumber); } // Increment the count by {incrementAmount}, but only if there will be no overflow/underflow, since if it // overflows, it goes back to the minimum value and continues from there. If it underflows, it goes back to // the maximum value and continues from there. final long newAmount = LongMath.checkedAdd(counterShardData.getCount(), incrementAmount); counterShardData.setCount(newAmount); counterShardData.setUpdatedDateTime(DateTime.now(DateTimeZone.UTC)); final String msg = String.format( "About to update CounterShardData (%s-->Shard-%s) with current incrementAmount %s and new " + "incrementAmount %s", counterName, shardNumber, counterShardData.getCount(), newAmount); getLogger().log(Level.FINE, msg); // Save Both CounterShardData and CounterShardOperationData together in the same TX. final CounterOperationType counterOperationType = incrementAmount < 0 ? CounterOperationType.DECREMENT : CounterOperationType.INCREMENT; final CounterShardOperationData counterShardOperationData = new CounterShardOperationData( counterShardData.getTypedKey(), this.counterShardOperationUuid, counterOperationType, Math.abs(incrementAmount), counterShardData.getUpdatedDateTime()); final CounterShardData savableCounterShardData = counterShardData; // We run these in a Transaction because they are in the same entity group and should be stored together. // Additionally, this is compatible with a parent-tx. try { ObjectifyService.ofy().transact(new VoidWork() { @Override public void vrun() { ObjectifyService.ofy().save().entities(savableCounterShardData); ObjectifyService.ofy().save().entities(counterShardOperationData); } }); } catch (DatastoreFailureException | DatastoreTimeoutException dse) { logger.log(Level.SEVERE, String.format( "Counter Shard Mutation '%s' for Shard Number %s on Counter '%s' may or may not have " + "completed! Error: %s", counterShardOperationUuid, shardNumber, counterData.getName(), dse.getMessage()), dse); // Throw this for now. In the future, we may implement retry logic. throw dse; } return new CounterOperation.Impl(counterShardOperationUuid, counterShardDataKey, counterOperationType, Math.abs(incrementAmount), counterShardData.getUpdatedDateTime(), newCounterCreated); } @VisibleForTesting boolean counterShardOperationAlreadyExists(final Key<CounterData> counterDataKey, final int shardNumber) { Preconditions.checkNotNull(counterDataKey); Preconditions.checkArgument(shardNumber >= 0); final Key<CounterShardData> counterShardDataKey = CounterShardData.key(counterDataKey, shardNumber); final Key<CounterShardOperationData> counterShardOperationDataKey = CounterShardOperationData .key(counterShardDataKey, counterShardOperationUuid); // See "https://groups.google.com/forum/#!searchin/objectify-appengine/exist/objectify-appengine/zFI2YWP5DTI // /BpwFNlVQo1UJ". This methodology will be less expensive, and strongly-consistent, but will be slightly // slower than a simple index query since it loads the entire entity. return ObjectifyService.ofy().load().key(counterShardOperationDataKey).now() != null; } } /** * A private implementation of {@link Work} that sets a CounterShard's amount to zero. */ @VisibleForTesting final static class ResetCounterShardWork extends VoidWork { private final String counterName; private final int shardNumber; /** * Required-Args Constructor. * * @param counterName A {@link String} that identifies the name of the counter being incremented. * @param shardNumber The index of the shard number to reset to zero. */ ResetCounterShardWork(final String counterName, int shardNumber) { Preconditions.checkNotNull(counterName); Preconditions.checkArgument(!StringUtils.isBlank(counterName)); this.counterName = counterName; Preconditions.checkArgument(shardNumber >= 0); this.shardNumber = shardNumber; } /** * Reset the indicated CounterShardData 'count' to zero. * * @return */ @Override public void vrun() { final Key<CounterData> counterDataKey = CounterData.key(counterName); final Key<CounterShardData> counterShardDataKey = CounterShardData.key(counterDataKey, shardNumber); final CounterShardData counterShardData = ObjectifyService.ofy().load().key(counterShardDataKey).now(); if (counterShardData == null) { return; } else { counterShardData.setCount(0L); counterShardData.setUpdatedDateTime(DateTime.now(DateTimeZone.UTC)); logger.log(Level.FINE, String.format("Resetting CounterShardData (%s-->ShardNum: %s) to zero!", counterShardDataKey, shardNumber, counterShardData.getCount())); ObjectifyService.ofy().save().entities(counterShardData).now(); } } } /** * Attempt to delete a counter from memcache but swallow any exceptions from memcache if it's down. * * @param counterName */ @VisibleForTesting void memcacheSafeDelete(final String counterName) { Preconditions.checkNotNull(counterName); try { memcacheService.delete(counterName); } catch (MemcacheServiceException mse) { // Do nothing. This merely indicates that memcache was unreachable, which is fine. If it's // unreachable, there's likely nothing in the cache anyway, but in any case there's nothing we can do here. } } /** * Attempt to delete a counter from memcache but swallow any exceptions from memcache if it's down. * * @param memcacheKey */ @VisibleForTesting BigInteger memcacheSafeGet(final String memcacheKey) { Preconditions.checkNotNull(memcacheKey); BigInteger cachedCounterCount; try { cachedCounterCount = (BigInteger) memcacheService.get(memcacheKey); } catch (MemcacheServiceException mse) { // Do nothing. This merely indicates that memcache was unreachable, which is fine. If it's // unreachable, there's likely nothing in the cache anyway, but in any case there's nothing we can do here. cachedCounterCount = null; } return cachedCounterCount; } /** * Helper method to get the {@link CounterData} associated with the supplied {@code counterName}. * * @param counterName * @return */ @VisibleForTesting protected Optional<CounterData> getCounterData(final String counterName) { Preconditions.checkNotNull(counterName); final Key<CounterData> counterKey = CounterData.key(counterName); final CounterData counterData = ObjectifyService.ofy().load().key(counterKey).now(); return Optional.fromNullable(counterData); } /** * Helper method to create the {@link CounterData} associated with the supplied counter information. * * @param counterName * @return * @throws IllegalArgumentException If the {@code counterName} is invalid. * @throws CounterExistsException If the counter with {@code counterName} already exists in the Datastore. */ @VisibleForTesting protected CounterData createCounterData(final String counterName) { this.counterNameValidator.validateCounterName(counterName); final Key<CounterData> counterKey = CounterData.key(counterName); // Perform a transactional GET to see if the counter exists. If it does, throw an exception. Otherwise, create // the counter in the same TX. return ObjectifyService.ofy().transact(new Work<CounterData>() { @Override public CounterData run() { final CounterData loadedCounterData = ObjectifyService.ofy().load().key(counterKey).now(); if (loadedCounterData == null) { final CounterData counterData = new CounterData(counterName, config.getNumInitialShards()); ObjectifyService.ofy().save().entity(counterData).now(); return counterData; } else { throw new CounterExistsException(counterName); } } }); } // Used to indicate that, for a given operation, a CounterData already existed, so no new counter was created. private static final boolean COUNTER_EXISTED = false; // Used to indicate that, for a given operation, a new CounterData was created. private static final boolean NEW_COUNTER_CREATED = true; /** * Helper method to get (or create and then get) a {@link CounterData} from the Datastore with a given name. The * result of this function is guaranteed to be non-null if no exception is thrown. * * @param counterName * @return * @throws NullPointerException if the {@code counterName} is null. * @throws InvalidCounterNameException If the counter name is too long or too short. */ @VisibleForTesting protected CounterDataGetCreateContainer getOrCreateCounterData(final String counterName) { // Validation failures will throw an exception. See JavaDoc for details. this.counterNameValidator.validateCounterName(counterName); final Key<CounterData> counterKey = CounterData.key(counterName); // This get/create is acceptable to perform outside of a transaction. In certain edge-cases, it's possible // that two threads might call this operation at the same time, and one or both threads might think (errantly) // that no CounterData exists in the Datastore when in reality one of the threads (or some other thread) may // have created the CoutnerData already. However, in this case, the two create operations will be the same using // default data, so even though the "winner" will be random, the result will be the same. In future, this // assumption may need to change. CounterData counterData = ObjectifyService.ofy().load().key(counterKey).now(); if (counterData == null) { counterData = new CounterData(counterName, config.getNumInitialShards()); // counterData.setNegativeCountAllowed(config.isNegativeCountAllowed()); ObjectifyService.ofy().save().entity(counterData).now(); return new CounterDataGetCreateContainer(counterData, NEW_COUNTER_CREATED); } else { return new CounterDataGetCreateContainer(counterData, COUNTER_EXISTED); } } /** * A helper object to indicate if a new counter was created after calling {@link #getOrCreateCounterData}. */ @Getter @ToString @EqualsAndHashCode @VisibleForTesting protected static final class CounterDataGetCreateContainer { private final CounterData counterData; private final boolean newCounterDataCreated; /** * Required-args Constructor. * * @param counterData * @param newCounterDataCreated */ private CounterDataGetCreateContainer(final CounterData counterData, final boolean newCounterDataCreated) { Preconditions.checkNotNull(counterData); this.counterData = counterData; this.newCounterDataCreated = newCounterDataCreated; } } private static final int NUM_RETRIES_LIMIT = 10; /** * Increment the memcache version of the named-counter by {@code amount} (positive or negative) in an atomic * fashion. Use memcache as a Semaphore/Mutex, and retry up to 10 times if other threads are attempting to update * memcache at the same time. If nothing is in Memcache when this function is called, then do nothing because only * #getCounter should "put" a value to memcache. Additionally, if this operation fails, the cache will be * re-populated after a configurable amount of time, so the count will eventually become correct. * * @param counterName * @param amount * @return The new count of this counter as reflected by memcache */ @VisibleForTesting protected Optional<Long> incrementMemcacheAtomic(final String counterName, final long amount) { Preconditions.checkNotNull(counterName); // Get the cache counter at a current point in time. String memCacheKey = this.assembleCounterKeyforMemcache(counterName); for (int currentRetry = 0; currentRetry < NUM_RETRIES_LIMIT; currentRetry++) { try { final IdentifiableValue identifiableCounter = memcacheService.getIdentifiable(memCacheKey); // See Javadoc about a null identifiableCounter. If it's null, then the named counter doesn't exist in // memcache. if (identifiableCounter == null || (identifiableCounter != null && identifiableCounter.getValue() == null)) { final String msg = "No identifiableCounter was found in Memcache. Unable to Atomically increment for CounterName '%s'. Memcache will be populated on the next called to getCounter()!"; logger.log(Level.FINEST, String.format(msg, counterName)); // This will return an absent value. Only #getCounter should "put" a value to memcache. break; } // If we get here, the count exists in memcache, so it can be atomically incremented/decremented. final BigInteger cachedCounterAmount = (BigInteger) identifiableCounter.getValue(); final long newMemcacheAmount = cachedCounterAmount.longValue() + amount; logger.log(Level.FINEST, String.format("Just before Atomic Increment of %s, Memcache has value: %s", amount, identifiableCounter.getValue())); if (memcacheService.putIfUntouched(counterName, identifiableCounter, BigInteger.valueOf(newMemcacheAmount), config.getDefaultCounterCountExpiration())) { logger.log(Level.FINEST, String.format("MemcacheService.putIfUntouched SUCCESS! with value: %s", newMemcacheAmount)); // If we get here, the put succeeded... return Optional.of(new Long(newMemcacheAmount)); } else { logger.log(Level.WARNING, String.format( "Unable to update memcache counter atomically. Retrying %s more times...", (NUM_RETRIES_LIMIT - currentRetry))); continue; } } catch (MemcacheServiceException mse) { // Check and post-decrement the numRetries counter in one step if ((currentRetry + 1) < NUM_RETRIES_LIMIT) { logger.log(Level.WARNING, String.format( "Unable to update memcache counter atomically. Retrying %s more times...", (NUM_RETRIES_LIMIT - currentRetry))); // Keep trying... continue; } else { // Evict the counter here, and let the next call to getCounter populate memcache final String logMessage = "Unable to update memcache counter atomically, with no more allowed retries. Evicting counter named '%s' from the cache!"; logger.log(Level.SEVERE, String.format(logMessage, (NUM_RETRIES_LIMIT - currentRetry)), mse); this.memcacheSafeDelete(memCacheKey); break; } } } // The increment did not work... return Optional.absent(); } /** * Assembles a CounterKey for Memcache * * @param counterName * @return */ @VisibleForTesting protected String assembleCounterKeyforMemcache(final String counterName) { Preconditions.checkNotNull(counterName); return counterName; } /** * @return */ protected Logger getLogger() { return logger; } /** * Helper method to determine if a counter's incrementAmount can be mutated (incremented or decremented). In order * for that to happen, the counter's status must be {@link CounterStatus#AVAILABLE}. * * @param counterName * @param counterStatus * @return */ @VisibleForTesting protected void assertCounterAmountMutatable(final String counterName, final CounterStatus counterStatus) { if (counterStatus != CounterStatus.AVAILABLE) { final String msg = String.format( "Can't mutate the amount of counter '%s' because it's currently in the %s state but must be in in " + "the %s state!", counterName, counterStatus.name(), CounterStatus.AVAILABLE); throw new CounterNotMutableException(counterName, msg); } } /** * Helper method to determine if a counter's incrementAmount can be mutated (incremented or decremented). In order * for that to happen, the counter's status must be {@link CounterStatus#AVAILABLE}. * * @param counterName The name of the counter. * @param counterStatus The {@link CounterStatus} of a counter that is currently stored in the Datastore. * @return */ @VisibleForTesting protected void assertCounterDetailsMutatable(final String counterName, final CounterStatus counterStatus) { Preconditions.checkNotNull(counterName); Preconditions.checkNotNull(counterStatus); if (counterStatus != CounterStatus.AVAILABLE && counterStatus != CounterStatus.READ_ONLY_COUNT) { final String msg = String.format( "Can't mutate with status %s. Counter must be in in the %s or %s state!", counterStatus, CounterStatus.AVAILABLE, CounterStatus.READ_ONLY_COUNT); throw new CounterNotMutableException(counterName, msg); } } /** * Helper method to determine if a counter can be put into the {@code incomingCounterStatus} by an external caller. * Currently, external callers may only put a Counter into the AVAILABLE or READ_ONLY status. * * @param counterName The name of the counter. * @param incomingCounterStatus The status of an incoming counter that is being updated by an external counter. * @return */ @VisibleForTesting protected void assertValidExternalCounterStatus(final String counterName, final CounterStatus incomingCounterStatus) { Preconditions.checkNotNull(counterName); Preconditions.checkNotNull(incomingCounterStatus); if (incomingCounterStatus != CounterStatus.AVAILABLE && incomingCounterStatus != CounterStatus.READ_ONLY_COUNT) { final String msg = String.format( "This Counter can only be put into the %s or %s status! %s is not allowed.", CounterStatus.AVAILABLE, CounterStatus.READ_ONLY_COUNT, incomingCounterStatus); throw new CounterNotMutableException(counterName, msg); } } @VisibleForTesting protected boolean isParentTransactionActive() { return ObjectifyService.ofy().getTransaction() == null ? false : ObjectifyService.ofy().getTransaction().isActive(); } /** * When a counter is in certain statuses, its count can not be determined for various reasons. For example, if the * number of shards is contracting from 5 to 4, then the total summation of all shards will not be possible to * determine since counts from shard 5 may have been applied to shards 5 and 4, whereas the actual "number of * shards" value in the CoutnerData may not have been updated to indicate this change. Thus, for certain * {@link CounterStatus} values, we cannot provide a valid count, and instead return {@link BigInteger#ZERO}. * * @param counterStatus * @return */ protected boolean counterStatusYieldsIndeterminateCount(final CounterStatus counterStatus) { Preconditions.checkNotNull(counterStatus); return counterStatus == CounterStatus.DELETING || counterStatus == CounterStatus.CONTRACTING_SHARDS || counterStatus == CounterStatus.RESETTING || counterStatus == CounterStatus.EXPANDING_SHARDS; } /** * An implementation of {@link CounterNameValidator} that validates counter name existence and length. */ private static class DefaultCounterNameValidator implements CounterNameValidator { /** * The main validation for counter names. Ensures that a counter name does not exceed 500 characters, but is not * blank, empty, or null. * * @param counterName The name of a counter being operated upon. * @throws NullPointerException if the {@code counterName} is null. * @throws InvalidCounterNameException If the counter name is too long or too short. */ @Override public void validateCounterName(final String counterName) { final String preconditionMessage = "Counter names must not exceed 500 characters, and may not be null, blank, nor empty!"; Preconditions.checkArgument(!StringUtils.isBlank(counterName), preconditionMessage); if (StringUtils.length(counterName) > 500) { throw new InvalidCounterNameException(preconditionMessage); } } } }