List of usage examples for com.amazonaws.services.kinesis.model Shard getSequenceNumberRange
public SequenceNumberRange getSequenceNumberRange()
The range of possible sequence numbers for the shard.
From source file:StreamUtils.java
License:Open Source License
/** * Split a shard by dividing the hash key space in half. * * @param streamName Name of the stream that contains the shard to split. * @param shardId The id of the shard to split. * * @throws IllegalArgumentException When either streamName or shardId are null or empty. * @throws LimitExceededException Shard limit for the account has been reached. * @throws ResourceNotFoundException The stream or shard cannot be found. * @throws InvalidArgumentException If the shard is closed and no eligible for splitting. * @throws AmazonClientException Error communicating with Amazon Kinesis. * *///from ww w . j av a2 s. co m public void splitShardEvenly(String streamName, String shardId) throws LimitExceededException, ResourceNotFoundException, AmazonClientException, InvalidArgumentException, IllegalArgumentException { if (streamName == null || streamName.isEmpty()) { throw new IllegalArgumentException("stream name is required"); } if (shardId == null || shardId.isEmpty()) { throw new IllegalArgumentException("shard id is required"); } DescribeStreamResult result = kinesis.describeStream(streamName); StreamDescription description = result.getStreamDescription(); // Find the shard we want to split Shard shardToSplit = null; for (Shard shard : description.getShards()) { if (shardId.equals(shard.getShardId())) { shardToSplit = shard; break; } } if (shardToSplit == null) { throw new ResourceNotFoundException( "Could not find shard with id '" + shardId + "' in stream '" + streamName + "'"); } // Check if the shard is still open. Open shards do not have an ending sequence number. if (shardToSplit.getSequenceNumberRange().getEndingSequenceNumber() != null) { throw new InvalidArgumentException("Shard is CLOSED and is not eligible for splitting"); } // Calculate the median hash key to use as the new starting hash key for the shard. BigInteger startingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getStartingHashKey()); BigInteger endingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getEndingHashKey()); BigInteger[] medianHashKey = startingHashKey.add(endingHashKey).divideAndRemainder(new BigInteger("2")); BigInteger newStartingHashKey = medianHashKey[0]; if (!BigInteger.ZERO.equals(medianHashKey[1])) { // In order to more evenly distributed the new hash key ranges across the new shards we will "round up" to // the next integer when our current hash key range is not evenly divisible by 2. newStartingHashKey = newStartingHashKey.add(BigInteger.ONE); } // Submit the split shard request kinesis.splitShard(streamName, shardId, newStartingHashKey.toString()); }
From source file:com.datatorrent.contrib.kinesis.KinesisConsumer.java
License:Open Source License
/** * This method is called in the activate method of the operator *//*w w w .j a v a2s . com*/ public void start() { isAlive = true; int realNumStream = simpleConsumerThreads.size(); if (realNumStream == 0) return; consumerThreadExecutor = Executors.newFixedThreadPool(realNumStream); for (final Shard shd : simpleConsumerThreads) { consumerThreadExecutor.submit(new Runnable() { @Override public void run() { logger.debug("Thread " + Thread.currentThread().getName() + " start consuming Records..."); while (isAlive) { Shard shard = shd; try { List<Record> records = KinesisUtil.getInstance().getRecords(streamName, recordsLimit, shard, getIteratorType(shard.getShardId()), shardPosition.get(shard.getShardId())); if (records == null || records.isEmpty()) { if (shard.getSequenceNumberRange().getEndingSequenceNumber() != null) { closedShards.add(shard); break; } try { Thread.sleep(recordsCheckInterval); } catch (Exception e) { throw new RuntimeException(e); } } else { String seqNo = ""; for (Record rc : records) { seqNo = rc.getSequenceNumber(); putRecord(shd.getShardId(), rc); } shardPosition.put(shard.getShardId(), new String(seqNo)); } } catch (Exception e) { throw new RuntimeException(e); } } logger.debug("Thread " + Thread.currentThread().getName() + " stop consuming Records..."); } }); } }
From source file:com.kinesis.datavis.utils.StreamUtils.java
License:Open Source License
/** * Split a shard by dividing the hash key space in half. * * @param streamName Name of the stream that contains the shard to split. * @param shardId The id of the shard to split. * * @throws IllegalArgumentException When either streamName or shardId are null or empty. * @throws LimitExceededException Shard limit for the account has been reached. * @throws ResourceNotFoundException The stream or shard cannot be found. * @throws InvalidArgumentException If the shard is closed and no eligible for splitting. * @throws AmazonClientException Error communicating with Amazon Kinesis. * *///from w ww. ja v a 2 s . co m public void splitShardEvenly(String streamName, String shardId) throws LimitExceededException, ResourceNotFoundException, AmazonClientException, InvalidArgumentException, IllegalArgumentException { if (streamName == null || streamName.isEmpty()) { throw new IllegalArgumentException("stream name is required"); } if (shardId == null || shardId.isEmpty()) { throw new IllegalArgumentException("shard id is required"); } DescribeStreamResult result = kinesis.describeStream(streamName); StreamDescription description = result.getStreamDescription(); // Find the shard we want to split Shard shardToSplit = null; for (Shard shard : description.getShards()) { if (shardId.equals(shard.getShardId())) { shardToSplit = shard; break; } } if (shardToSplit == null) { throw new ResourceNotFoundException( "Could not find shard with id '" + shardId + "' in stream '" + streamName + "'"); } // Check if the shard is still open. Open shards do not have an ending sequence number. if (shardToSplit.getSequenceNumberRange().getEndingSequenceNumber() != null) { throw new InvalidArgumentException("Shard is CLOSED and is not eligible for splitting"); } // Calculate the median hash key to use as the new starting hash key for the shard. BigInteger startingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getStartingHashKey()); BigInteger endingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getEndingHashKey()); BigInteger[] medianHashKey = startingHashKey.add(endingHashKey).divideAndRemainder(new BigInteger("2")); BigInteger newStartingHashKey = medianHashKey[0]; if (!BigInteger.ZERO.equals(medianHashKey[1])) { // In order to more evenly distributed the new hash key ranges across the new shards we will "round up" to // the next integer when our current hash key range is not evenly divisible by 2. newStartingHashKey = newStartingHashKey.add(BigInteger.ONE); } // Submit the split shard request kinesis.splitShard(streamName, shardId, newStartingHashKey.toString()); }
From source file:com.qubole.presto.kinesis.KinesisSplitManager.java
License:Apache License
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout) { KinesisTableLayoutHandle kinesislayout = handleResolver.convertLayout(layout); KinesisTableHandle kinesisTableHandle = kinesislayout.getTable(); InternalStreamDescription desc = this.getStreamDescription(kinesisTableHandle.getStreamName()); ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder(); for (Shard shard : desc.getShards()) { KinesisSplit split = new KinesisSplit(connectorId, kinesisTableHandle.getStreamName(), kinesisTableHandle.getMessageDataFormat(), shard.getShardId(), shard.getSequenceNumberRange().getStartingSequenceNumber(), shard.getSequenceNumberRange().getEndingSequenceNumber()); builder.add(split);//www . j av a 2 s. co m } return new FixedSplitSource(connectorId, builder.build()); }
From source file:com.streamsets.pipeline.stage.lib.kinesis.KinesisUtil.java
License:Apache License
public static long getShardCount(Regions region, AWSConfig awsConfig, String streamName) throws AmazonClientException { ClientConfiguration kinesisConfiguration = new ClientConfiguration(); AmazonKinesisClient kinesisClient = new AmazonKinesisClient(AWSUtil.getCredentialsProvider(awsConfig), kinesisConfiguration);//from w ww. ja v a 2 s. c o m kinesisClient.setRegion(Region.getRegion(region)); try { long numShards = 0; String lastShardId = null; StreamDescription description; do { if (lastShardId == null) { description = kinesisClient.describeStream(streamName).getStreamDescription(); } else { description = kinesisClient.describeStream(streamName, lastShardId).getStreamDescription(); } for (Shard shard : description.getShards()) { if (shard.getSequenceNumberRange().getEndingSequenceNumber() == null) { // Then this shard is open, so we should count it. Shards with an ending sequence number // are closed and cannot be written to, so we skip counting them. ++numShards; } } int pageSize = description.getShards().size(); lastShardId = description.getShards().get(pageSize - 1).getShardId(); } while (description.getHasMoreShards()); LOG.debug("Connected successfully to stream: '{}' with '{}' shards.", streamName, numShards); return numShards; } finally { kinesisClient.shutdown(); } }
From source file:org.apache.apex.malhar.contrib.kinesis.KinesisConsumer.java
License:Apache License
/** * This method is called in the activate method of the operator *//* w w w .ja va2 s. com*/ public void start() { isAlive = true; int realNumStream = simpleConsumerThreads.size(); if (realNumStream == 0) { return; } consumerThreadExecutor = Executors.newFixedThreadPool(realNumStream); for (final Shard shd : simpleConsumerThreads) { consumerThreadExecutor.submit(new Runnable() { @Override public void run() { logger.debug("Thread " + Thread.currentThread().getName() + " start consuming Records..."); while (isAlive) { Shard shard = shd; try { List<Record> records = KinesisUtil.getInstance().getRecords(streamName, recordsLimit, shard.getShardId(), getIteratorType(shard.getShardId()), shardPosition.get(shard.getShardId())); if (records == null || records.isEmpty()) { if (shard.getSequenceNumberRange().getEndingSequenceNumber() != null) { closedShards.add(shard); break; } try { Thread.sleep(recordsCheckInterval); } catch (Exception e) { throw new RuntimeException(e); } } else { String seqNo = ""; for (Record rc : records) { seqNo = rc.getSequenceNumber(); putRecord(shd.getShardId(), rc); } shardPosition.put(shard.getShardId(), seqNo); } } catch (Exception e) { throw new RuntimeException(e); } } logger.debug("Thread " + Thread.currentThread().getName() + " stop consuming Records..."); } }); } }
From source file:org.springframework.integration.aws.inbound.kinesis.KinesisMessageDrivenChannelAdapter.java
License:Apache License
private void populateShardsForStream(final String stream, final CountDownLatch shardsGatherLatch) { this.dispatcherExecutor.execute(new Runnable() { @Override//from w w w . j a v a2 s . c o m public void run() { try { int describeStreamRetries = 0; List<Shard> shardsToConsume = new ArrayList<>(); String exclusiveStartShardId = null; while (true) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest() .withStreamName(stream).withExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = null; // Call DescribeStream, with backoff and retries (if we get LimitExceededException). try { describeStreamResult = KinesisMessageDrivenChannelAdapter.this.amazonKinesis .describeStream(describeStreamRequest); } catch (LimitExceededException e) { logger.info("Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for [" + KinesisMessageDrivenChannelAdapter.this.describeStreamBackoff + "] millis."); } if (describeStreamResult == null || !StreamStatus.ACTIVE.toString() .equals(describeStreamResult.getStreamDescription().getStreamStatus())) { if (describeStreamRetries++ > KinesisMessageDrivenChannelAdapter.this.describeStreamRetries) { ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException( "The stream [" + stream + "] isn't ACTIVE or doesn't exist."); resourceNotFoundException.setServiceName("Kinesis"); throw resourceNotFoundException; } try { Thread.sleep(KinesisMessageDrivenChannelAdapter.this.describeStreamBackoff); continue; } catch (InterruptedException e) { Thread.interrupted(); throw new IllegalStateException("The [describeStream] thread for the stream [" + stream + "] has been interrupted.", e); } } List<Shard> shards = describeStreamResult.getStreamDescription().getShards(); for (Shard shard : shards) { String endingSequenceNumber = shard.getSequenceNumberRange().getEndingSequenceNumber(); if (endingSequenceNumber != null) { String key = buildCheckpointKeyForShard(stream, shard.getShardId()); String checkpoint = KinesisMessageDrivenChannelAdapter.this.checkpointStore .get(key); boolean skipClosedShard = checkpoint != null && new BigInteger(endingSequenceNumber) .compareTo(new BigInteger(checkpoint)) <= 0; if (logger.isTraceEnabled()) { logger.trace("The shard [" + shard + "] in stream [" + stream + "] is closed CLOSED with endingSequenceNumber [" + endingSequenceNumber + "].\nThe last processed checkpoint is [" + checkpoint + "]." + (skipClosedShard ? "\nThe shard will be skipped." : "")); } if (skipClosedShard) { // Skip CLOSED shard which has been read before according a checkpoint continue; } } shardsToConsume.add(shard); } if (describeStreamResult.getStreamDescription().getHasMoreShards()) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { break; } } for (Shard shard : shardsToConsume) { KinesisShardOffset shardOffset = new KinesisShardOffset( KinesisMessageDrivenChannelAdapter.this.streamInitialSequence); shardOffset.setShard(shard.getShardId()); shardOffset.setStream(stream); boolean addedOffset; synchronized (KinesisMessageDrivenChannelAdapter.this.shardOffsets) { addedOffset = KinesisMessageDrivenChannelAdapter.this.shardOffsets.add(shardOffset); } if (addedOffset && shardsGatherLatch == null && KinesisMessageDrivenChannelAdapter.this.active) { populateConsumer(shardOffset); } } } finally { if (shardsGatherLatch != null) { shardsGatherLatch.countDown(); } KinesisMessageDrivenChannelAdapter.this.inResharding.remove(stream); } } }); }