List of usage examples for com.amazonaws.services.kinesis.model DescribeStreamRequest setExclusiveStartShardId
public void setExclusiveStartShardId(String exclusiveStartShardId)
The shard ID of the shard to start with.
From source file:AmazonKinesisGet.java
License:Open Source License
public static void main(String[] args) throws Exception { init();// ww w. j a v a 2s.c o m final String myStreamName = "philsteststream"; final Integer myStreamSize = 1; // list all of my streams ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesisClient.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } LOG.info("Printing my list of streams : "); // print all of my streams. if (!streamNames.isEmpty()) { System.out.println("List of my streams: "); } for (int i = 0; i < streamNames.size(); i++) { System.out.println(streamNames.get(i)); } //System.out.println(streamNames.get(0)); String myownstream = streamNames.get(0); // Retrieve the Shards from a Stream DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(myownstream); DescribeStreamResult describeStreamResult; List<Shard> shards = new ArrayList<>(); String lastShardId = null; do { describeStreamRequest.setExclusiveStartShardId(lastShardId); describeStreamResult = kinesisClient.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (shards.size() > 0) { lastShardId = shards.get(shards.size() - 1).getShardId(); } } while (describeStreamResult.getStreamDescription().getHasMoreShards()); // Get Data from the Shards in a Stream // Hard-coded to use only 1 shard String shardIterator; GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(myownstream); //get(0) shows hardcoded to 1 stream getShardIteratorRequest.setShardId(shards.get(0).getShardId()); // using TRIM_HORIZON but could use alternatives getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON"); GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest); shardIterator = getShardIteratorResult.getShardIterator(); // Continuously read data records from shard. List<Record> records; while (true) { // Create new GetRecordsRequest with existing shardIterator. // Set maximum records to return to 1000. GetRecordsRequest getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(1000); GetRecordsResult result = kinesisClient.getRecords(getRecordsRequest); // Put result into record list. Result may be empty. records = result.getRecords(); // Print records for (Record record : records) { ByteBuffer byteBuffer = record.getData(); System.out.println(String.format("Seq No: %s - %s", record.getSequenceNumber(), new String(byteBuffer.array()))); } try { Thread.sleep(1000); } catch (InterruptedException exception) { throw new RuntimeException(exception); } shardIterator = result.getNextShardIterator(); } }
From source file:com.facebook.presto.kinesis.KinesisSplitManager.java
License:Apache License
@Override public ConnectorPartitionResult getPartitions(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) { KinesisTableHandle kinesisTableHandle = handleResolver.convertTableHandle(tableHandle); DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest(); describeStreamRequest.setStreamName(kinesisTableHandle.getStreamName()); String exclusiveStartShardId = null; describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest); String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus(); while ((streamStatus.equals("ACTIVE") == false) && (streamStatus.equals("UPDATING") == false)) { throw new ResourceNotFoundException("Stream not Active"); }// ww w .j a v a2 s. c o m List<Shard> shards = new ArrayList<>(); ImmutableList.Builder<ConnectorPartition> builder = ImmutableList.builder(); do { shards.addAll(describeStreamResult.getStreamDescription().getShards()); for (Shard shard : shards) { builder.add(new KinesisShard(kinesisTableHandle.getStreamName(), shard)); } if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { exclusiveStartShardId = null; } } while (exclusiveStartShardId != null); return new ConnectorPartitionResult(builder.build(), tupleDomain); }
From source file:com.qubole.presto.kinesis.KinesisSplitManager.java
License:Apache License
/** * Internal method to retrieve the stream description and get the shards from AWS. * * Gets from the internal cache unless not yet created or too old. * * @param streamName//from ww w .j av a 2s . co m * @return */ protected InternalStreamDescription getStreamDescription(String streamName) { InternalStreamDescription desc = this.streamMap.get(streamName); if (desc == null || System.currentTimeMillis() - desc.getCreateTimeStamp() >= MAX_CACHE_AGE_MILLIS) { desc = new InternalStreamDescription(streamName); DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest(); describeStreamRequest.setStreamName(streamName); // Collect shards from Kinesis String exclusiveStartShardId = null; List<Shard> shards = new ArrayList<>(); do { describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = clientManager.getClient() .describeStream(describeStreamRequest); String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus(); if (!streamStatus.equals("ACTIVE") && !streamStatus.equals("UPDATING")) { throw new ResourceNotFoundException("Stream not Active"); } desc.addAllShards(describeStreamResult.getStreamDescription().getShards()); if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { exclusiveStartShardId = null; } } while (exclusiveStartShardId != null); this.streamMap.put(streamName, desc); } return desc; }
From source file:com.srotya.flume.kinesis.source.KinesisSource.java
License:Apache License
@Override protected void doStart() throws FlumeException { client = new AmazonKinesisClient(awsCredentials, clientConfig); client.setEndpoint(endpoint);/*from ww w . j a va 2 s.co m*/ DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(streamName); shards = new ArrayList<>(); String exclusiveStartShardId = null; do { describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = client.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { exclusiveStartShardId = null; } } while (exclusiveStartShardId != null); getShardIterator(); }
From source file:com.trulia.stail.Stail.java
License:Apache License
private static List<Shard> getShards(AmazonKinesis client, String stream) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(stream); List<Shard> shards = new ArrayList<>(); String exclusiveStartShardId = null; do {/*from w w w . jav a 2 s . c o m*/ describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = client.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { exclusiveStartShardId = null; } } while (exclusiveStartShardId != null); return shards; }
From source file:org.apache.druid.indexing.kinesis.KinesisRecordSupplier.java
License:Apache License
@Override public Set<String> getPartitionIds(String stream) { return wrapExceptions(() -> { final Set<String> retVal = new HashSet<>(); DescribeStreamRequest request = new DescribeStreamRequest(); request.setStreamName(stream);//from ww w.j a v a2 s. c o m while (request != null) { final DescribeStreamResult result = kinesis.describeStream(request); final StreamDescription streamDescription = result.getStreamDescription(); final List<Shard> shards = streamDescription.getShards(); for (Shard shard : shards) { retVal.add(shard.getShardId()); } if (streamDescription.isHasMoreShards()) { request.setExclusiveStartShardId(Iterables.getLast(shards).getShardId()); } else { request = null; } } return retVal; }); }
From source file:org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxy.java
License:Apache License
/** * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis stream possess. * * This method is using a "full jitter" approach described in AWS's article, * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and Jitter"</a>. * This is necessary because concurrent calls will be made by all parallel subtask's fetcher. This * jitter backoff approach will help distribute calls across the fetchers over time. * * @param streamName the stream to describe * @param startShardId which shard to start with for this describe operation (earlier shard's infos will not appear in result) * @return the result of the describe stream operation *//* w w w . j a v a2 s .co m*/ private DescribeStreamResult describeStream(String streamName, @Nullable String startShardId) throws InterruptedException { final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(streamName); describeStreamRequest.setExclusiveStartShardId(startShardId); DescribeStreamResult describeStreamResult = null; // Call DescribeStream, with full-jitter backoff (if we get LimitExceededException). int attemptCount = 0; while (describeStreamResult == null) { // retry until we get a result try { describeStreamResult = kinesisClient.describeStream(describeStreamRequest); } catch (LimitExceededException le) { long backoffMillis = fullJitterBackoff(describeStreamBaseBackoffMillis, describeStreamMaxBackoffMillis, describeStreamExpConstant, attemptCount++); LOG.warn("Got LimitExceededException when describing stream " + streamName + ". Backing off for " + backoffMillis + " millis."); Thread.sleep(backoffMillis); } catch (ResourceNotFoundException re) { throw new RuntimeException("Error while getting stream details", re); } } String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus(); if (!(streamStatus.equals(StreamStatus.ACTIVE.toString()) || streamStatus.equals(StreamStatus.UPDATING.toString()))) { if (LOG.isWarnEnabled()) { LOG.warn("The status of stream " + streamName + " is " + streamStatus + "; result of the current " + "describeStream operation will not contain any shard information."); } } // Kinesalite (mock implementation of Kinesis) does not correctly exclude shards before the exclusive // start shard id in the returned shards list; check if we need to remove these erroneously returned shards if (startShardId != null) { List<Shard> shards = describeStreamResult.getStreamDescription().getShards(); Iterator<Shard> shardItr = shards.iterator(); while (shardItr.hasNext()) { if (KinesisStreamShard.compareShardIds(shardItr.next().getShardId(), startShardId) <= 0) { shardItr.remove(); } } } return describeStreamResult; }
From source file:org.apache.storm.kinesis.spout.KinesisConnection.java
License:Apache License
List<Shard> getShardsForStream(String stream) {
DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
describeStreamRequest.setStreamName(stream);
List<Shard> shards = new ArrayList<>();
String exclusiveStartShardId = null;
do {/*from www.ja v a 2 s .c om*/
describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
DescribeStreamResult describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
shards.addAll(describeStreamResult.getStreamDescription().getShards());
if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) {
exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
} else {
exclusiveStartShardId = null;
}
} while (exclusiveStartShardId != null);
LOG.info("Number of shards for stream " + stream + " are " + shards.size());
return shards;
}