List of usage examples for com.amazonaws.services.kinesis.model ResourceNotFoundException ResourceNotFoundException
public ResourceNotFoundException(String message)
From source file:StreamUtils.java
License:Open Source License
/** * Split a shard by dividing the hash key space in half. * * @param streamName Name of the stream that contains the shard to split. * @param shardId The id of the shard to split. * * @throws IllegalArgumentException When either streamName or shardId are null or empty. * @throws LimitExceededException Shard limit for the account has been reached. * @throws ResourceNotFoundException The stream or shard cannot be found. * @throws InvalidArgumentException If the shard is closed and no eligible for splitting. * @throws AmazonClientException Error communicating with Amazon Kinesis. * *//*from w w w. j a va2s. com*/ public void splitShardEvenly(String streamName, String shardId) throws LimitExceededException, ResourceNotFoundException, AmazonClientException, InvalidArgumentException, IllegalArgumentException { if (streamName == null || streamName.isEmpty()) { throw new IllegalArgumentException("stream name is required"); } if (shardId == null || shardId.isEmpty()) { throw new IllegalArgumentException("shard id is required"); } DescribeStreamResult result = kinesis.describeStream(streamName); StreamDescription description = result.getStreamDescription(); // Find the shard we want to split Shard shardToSplit = null; for (Shard shard : description.getShards()) { if (shardId.equals(shard.getShardId())) { shardToSplit = shard; break; } } if (shardToSplit == null) { throw new ResourceNotFoundException( "Could not find shard with id '" + shardId + "' in stream '" + streamName + "'"); } // Check if the shard is still open. Open shards do not have an ending sequence number. if (shardToSplit.getSequenceNumberRange().getEndingSequenceNumber() != null) { throw new InvalidArgumentException("Shard is CLOSED and is not eligible for splitting"); } // Calculate the median hash key to use as the new starting hash key for the shard. BigInteger startingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getStartingHashKey()); BigInteger endingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getEndingHashKey()); BigInteger[] medianHashKey = startingHashKey.add(endingHashKey).divideAndRemainder(new BigInteger("2")); BigInteger newStartingHashKey = medianHashKey[0]; if (!BigInteger.ZERO.equals(medianHashKey[1])) { // In order to more evenly distributed the new hash key ranges across the new shards we will "round up" to // the next integer when our current hash key range is not evenly divisible by 2. newStartingHashKey = newStartingHashKey.add(BigInteger.ONE); } // Submit the split shard request kinesis.splitShard(streamName, shardId, newStartingHashKey.toString()); }
From source file:com.facebook.presto.kinesis.KinesisSplitManager.java
License:Apache License
@Override public ConnectorPartitionResult getPartitions(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) { KinesisTableHandle kinesisTableHandle = handleResolver.convertTableHandle(tableHandle); DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest(); describeStreamRequest.setStreamName(kinesisTableHandle.getStreamName()); String exclusiveStartShardId = null; describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest); String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus(); while ((streamStatus.equals("ACTIVE") == false) && (streamStatus.equals("UPDATING") == false)) { throw new ResourceNotFoundException("Stream not Active"); }//www . ja v a2 s .c o m List<Shard> shards = new ArrayList<>(); ImmutableList.Builder<ConnectorPartition> builder = ImmutableList.builder(); do { shards.addAll(describeStreamResult.getStreamDescription().getShards()); for (Shard shard : shards) { builder.add(new KinesisShard(kinesisTableHandle.getStreamName(), shard)); } if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { exclusiveStartShardId = null; } } while (exclusiveStartShardId != null); return new ConnectorPartitionResult(builder.build(), tupleDomain); }
From source file:com.kinesis.datavis.utils.StreamUtils.java
License:Open Source License
/** * Split a shard by dividing the hash key space in half. * * @param streamName Name of the stream that contains the shard to split. * @param shardId The id of the shard to split. * * @throws IllegalArgumentException When either streamName or shardId are null or empty. * @throws LimitExceededException Shard limit for the account has been reached. * @throws ResourceNotFoundException The stream or shard cannot be found. * @throws InvalidArgumentException If the shard is closed and no eligible for splitting. * @throws AmazonClientException Error communicating with Amazon Kinesis. * *///from w w w . j ava 2s . c o m public void splitShardEvenly(String streamName, String shardId) throws LimitExceededException, ResourceNotFoundException, AmazonClientException, InvalidArgumentException, IllegalArgumentException { if (streamName == null || streamName.isEmpty()) { throw new IllegalArgumentException("stream name is required"); } if (shardId == null || shardId.isEmpty()) { throw new IllegalArgumentException("shard id is required"); } DescribeStreamResult result = kinesis.describeStream(streamName); StreamDescription description = result.getStreamDescription(); // Find the shard we want to split Shard shardToSplit = null; for (Shard shard : description.getShards()) { if (shardId.equals(shard.getShardId())) { shardToSplit = shard; break; } } if (shardToSplit == null) { throw new ResourceNotFoundException( "Could not find shard with id '" + shardId + "' in stream '" + streamName + "'"); } // Check if the shard is still open. Open shards do not have an ending sequence number. if (shardToSplit.getSequenceNumberRange().getEndingSequenceNumber() != null) { throw new InvalidArgumentException("Shard is CLOSED and is not eligible for splitting"); } // Calculate the median hash key to use as the new starting hash key for the shard. BigInteger startingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getStartingHashKey()); BigInteger endingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getEndingHashKey()); BigInteger[] medianHashKey = startingHashKey.add(endingHashKey).divideAndRemainder(new BigInteger("2")); BigInteger newStartingHashKey = medianHashKey[0]; if (!BigInteger.ZERO.equals(medianHashKey[1])) { // In order to more evenly distributed the new hash key ranges across the new shards we will "round up" to // the next integer when our current hash key range is not evenly divisible by 2. newStartingHashKey = newStartingHashKey.add(BigInteger.ONE); } // Submit the split shard request kinesis.splitShard(streamName, shardId, newStartingHashKey.toString()); }
From source file:com.qubole.presto.kinesis.KinesisSplitManager.java
License:Apache License
/** * Internal method to retrieve the stream description and get the shards from AWS. * * Gets from the internal cache unless not yet created or too old. * * @param streamName// w w w.j a v a2s. co m * @return */ protected InternalStreamDescription getStreamDescription(String streamName) { InternalStreamDescription desc = this.streamMap.get(streamName); if (desc == null || System.currentTimeMillis() - desc.getCreateTimeStamp() >= MAX_CACHE_AGE_MILLIS) { desc = new InternalStreamDescription(streamName); DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest(); describeStreamRequest.setStreamName(streamName); // Collect shards from Kinesis String exclusiveStartShardId = null; List<Shard> shards = new ArrayList<>(); do { describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = clientManager.getClient() .describeStream(describeStreamRequest); String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus(); if (!streamStatus.equals("ACTIVE") && !streamStatus.equals("UPDATING")) { throw new ResourceNotFoundException("Stream not Active"); } desc.addAllShards(describeStreamResult.getStreamDescription().getShards()); if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { exclusiveStartShardId = null; } } while (exclusiveStartShardId != null); this.streamMap.put(streamName, desc); } return desc; }
From source file:org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner.java
License:Apache License
private List<Shard> createOrUpdate(String stream, int shards) { List<Shard> shardList = new ArrayList<>(); int describeStreamRetries = 0; String exclusiveStartShardId = null; DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(stream); while (true) { DescribeStreamResult describeStreamResult = null; try {/* w w w. j a v a 2s. c o m*/ describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId); describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest); StreamDescription streamDescription = describeStreamResult.getStreamDescription(); if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) { shardList.addAll(streamDescription.getShards()); if (streamDescription.getHasMoreShards()) { exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId(); } else { break; } } } catch (ResourceNotFoundException ex) { if (!this.configurationProperties.isAutoCreateStream()) { throw new ProvisioningException( "The stream [" + stream + "] was not found and auto creation is disabled.", ex); } if (logger.isInfoEnabled()) { logger.info("Stream '" + stream + "' not found. Create one..."); } this.amazonKinesis.createStream(stream, Math.max(this.configurationProperties.getMinShardCount(), shards)); continue; } catch (LimitExceededException ex) { logger.info( "Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for [" + this.configurationProperties.getDescribeStreamBackoff() + "] millis."); } if (describeStreamResult == null || !StreamStatus.ACTIVE.toString() .equals(describeStreamResult.getStreamDescription().getStreamStatus())) { if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) { ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException( "The stream [" + stream + "] isn't ACTIVE or doesn't exist."); resourceNotFoundException.setServiceName("Kinesis"); throw new ProvisioningException( "Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error", resourceNotFoundException); } try { Thread.sleep(this.configurationProperties.getDescribeStreamBackoff()); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new ProvisioningException( "The [describeStream] thread for the stream [" + stream + "] has been interrupted.", ex); } } } int effectiveShardCount = Math.max(this.configurationProperties.getMinShardCount(), shards); if ((shardList.size() < effectiveShardCount) && this.configurationProperties.isAutoAddShards()) { return updateShardCount(stream, shardList.size(), effectiveShardCount); } return shardList; }
From source file:org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner.java
License:Apache License
private List<Shard> updateShardCount(String streamName, int shardCount, int targetCount) { if (logger.isInfoEnabled()) { logger.info("Stream [" + streamName + "] has [" + shardCount + "] shards compared to a target configuration of [" + targetCount + "], creating shards..."); }//w w w . jav a 2s .c o m UpdateShardCountRequest updateShardCountRequest = new UpdateShardCountRequest().withStreamName(streamName) .withTargetShardCount(targetCount).withScalingType(ScalingType.UNIFORM_SCALING); this.amazonKinesis.updateShardCount(updateShardCountRequest); // Wait for stream to become active again after resharding List<Shard> shardList = new ArrayList<>(); int describeStreamRetries = 0; String exclusiveStartShardId = null; DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(streamName); while (true) { DescribeStreamResult describeStreamResult = null; try { describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId); describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest); StreamDescription streamDescription = describeStreamResult.getStreamDescription(); if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) { shardList.addAll(streamDescription.getShards()); if (streamDescription.getHasMoreShards()) { exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId(); } else { break; } } } catch (LimitExceededException ex) { logger.info("Got LimitExceededException when describing stream [" + streamName + "]. " + "Backing off for [" + this.configurationProperties.getDescribeStreamBackoff() + "] millis."); } if (describeStreamResult == null || !StreamStatus.ACTIVE.toString() .equals(describeStreamResult.getStreamDescription().getStreamStatus())) { if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) { ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException( "The stream [" + streamName + "] isn't ACTIVE or doesn't exist."); resourceNotFoundException.setServiceName("Kinesis"); throw new ProvisioningException( "Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error", resourceNotFoundException); } try { Thread.sleep(this.configurationProperties.getDescribeStreamBackoff()); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new ProvisioningException( "The [describeStream] thread for the stream [" + streamName + "] has been interrupted.", ex); } } } return shardList; }
From source file:org.springframework.integration.aws.inbound.kinesis.KinesisMessageDrivenChannelAdapter.java
License:Apache License
private void populateShardsForStream(final String stream, final CountDownLatch shardsGatherLatch) { this.dispatcherExecutor.execute(new Runnable() { @Override//from www . j a v a 2s .co m public void run() { try { int describeStreamRetries = 0; List<Shard> shardsToConsume = new ArrayList<>(); String exclusiveStartShardId = null; while (true) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest() .withStreamName(stream).withExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = null; // Call DescribeStream, with backoff and retries (if we get LimitExceededException). try { describeStreamResult = KinesisMessageDrivenChannelAdapter.this.amazonKinesis .describeStream(describeStreamRequest); } catch (LimitExceededException e) { logger.info("Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for [" + KinesisMessageDrivenChannelAdapter.this.describeStreamBackoff + "] millis."); } if (describeStreamResult == null || !StreamStatus.ACTIVE.toString() .equals(describeStreamResult.getStreamDescription().getStreamStatus())) { if (describeStreamRetries++ > KinesisMessageDrivenChannelAdapter.this.describeStreamRetries) { ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException( "The stream [" + stream + "] isn't ACTIVE or doesn't exist."); resourceNotFoundException.setServiceName("Kinesis"); throw resourceNotFoundException; } try { Thread.sleep(KinesisMessageDrivenChannelAdapter.this.describeStreamBackoff); continue; } catch (InterruptedException e) { Thread.interrupted(); throw new IllegalStateException("The [describeStream] thread for the stream [" + stream + "] has been interrupted.", e); } } List<Shard> shards = describeStreamResult.getStreamDescription().getShards(); for (Shard shard : shards) { String endingSequenceNumber = shard.getSequenceNumberRange().getEndingSequenceNumber(); if (endingSequenceNumber != null) { String key = buildCheckpointKeyForShard(stream, shard.getShardId()); String checkpoint = KinesisMessageDrivenChannelAdapter.this.checkpointStore .get(key); boolean skipClosedShard = checkpoint != null && new BigInteger(endingSequenceNumber) .compareTo(new BigInteger(checkpoint)) <= 0; if (logger.isTraceEnabled()) { logger.trace("The shard [" + shard + "] in stream [" + stream + "] is closed CLOSED with endingSequenceNumber [" + endingSequenceNumber + "].\nThe last processed checkpoint is [" + checkpoint + "]." + (skipClosedShard ? "\nThe shard will be skipped." : "")); } if (skipClosedShard) { // Skip CLOSED shard which has been read before according a checkpoint continue; } } shardsToConsume.add(shard); } if (describeStreamResult.getStreamDescription().getHasMoreShards()) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { break; } } for (Shard shard : shardsToConsume) { KinesisShardOffset shardOffset = new KinesisShardOffset( KinesisMessageDrivenChannelAdapter.this.streamInitialSequence); shardOffset.setShard(shard.getShardId()); shardOffset.setStream(stream); boolean addedOffset; synchronized (KinesisMessageDrivenChannelAdapter.this.shardOffsets) { addedOffset = KinesisMessageDrivenChannelAdapter.this.shardOffsets.add(shardOffset); } if (addedOffset && shardsGatherLatch == null && KinesisMessageDrivenChannelAdapter.this.active) { populateConsumer(shardOffset); } } } finally { if (shardsGatherLatch != null) { shardsGatherLatch.countDown(); } KinesisMessageDrivenChannelAdapter.this.inResharding.remove(stream); } } }); }