List of usage examples for com.amazonaws.services.kinesis.model GetShardIteratorRequest setShardId
public void setShardId(String shardId)
The shard ID of the Kinesis Data Streams shard to get the iterator for.
From source file:AmazonKinesisGet.java
License:Open Source License
public static void main(String[] args) throws Exception { init();//from w w w . j a v a2 s . c om final String myStreamName = "philsteststream"; final Integer myStreamSize = 1; // list all of my streams ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesisClient.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } LOG.info("Printing my list of streams : "); // print all of my streams. if (!streamNames.isEmpty()) { System.out.println("List of my streams: "); } for (int i = 0; i < streamNames.size(); i++) { System.out.println(streamNames.get(i)); } //System.out.println(streamNames.get(0)); String myownstream = streamNames.get(0); // Retrieve the Shards from a Stream DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(myownstream); DescribeStreamResult describeStreamResult; List<Shard> shards = new ArrayList<>(); String lastShardId = null; do { describeStreamRequest.setExclusiveStartShardId(lastShardId); describeStreamResult = kinesisClient.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (shards.size() > 0) { lastShardId = shards.get(shards.size() - 1).getShardId(); } } while (describeStreamResult.getStreamDescription().getHasMoreShards()); // Get Data from the Shards in a Stream // Hard-coded to use only 1 shard String shardIterator; GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(myownstream); //get(0) shows hardcoded to 1 stream getShardIteratorRequest.setShardId(shards.get(0).getShardId()); // using TRIM_HORIZON but could use alternatives getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON"); GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest); shardIterator = getShardIteratorResult.getShardIterator(); // Continuously read data records from shard. List<Record> records; while (true) { // Create new GetRecordsRequest with existing shardIterator. // Set maximum records to return to 1000. GetRecordsRequest getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(1000); GetRecordsResult result = kinesisClient.getRecords(getRecordsRequest); // Put result into record list. Result may be empty. records = result.getRecords(); // Print records for (Record record : records) { ByteBuffer byteBuffer = record.getData(); System.out.println(String.format("Seq No: %s - %s", record.getSequenceNumber(), new String(byteBuffer.array()))); } try { Thread.sleep(1000); } catch (InterruptedException exception) { throw new RuntimeException(exception); } shardIterator = result.getNextShardIterator(); } }
From source file:com.datatorrent.contrib.kinesis.KinesisTestConsumer.java
License:Open Source License
@Override public void run() { DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.setStreamName(streamName); DescribeStreamResult describeResponse = client.describeStream(describeRequest); final List<Shard> shards = describeResponse.getStreamDescription().getShards(); logger.debug("Inside consumer::run receiveCount= {}", receiveCount); while (isAlive) { Shard shId = shards.get(0);/*from w w w.j a v a2 s . c o m*/ GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId.getShardId()); iteratorRequest.setShardIteratorType("TRIM_HORIZON"); GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); String iterator = iteratorResponse.getShardIterator(); GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(1000); getRequest.setShardIterator(iterator); //call "get" operation and get everything in this shard range GetRecordsResult getResponse = client.getRecords(getRequest); //get reference to next iterator for this shard //retrieve records List<Record> records = getResponse.getRecords(); if (records == null || records.isEmpty()) { try { Thread.sleep(1000); } catch (InterruptedException e) { throw new RuntimeException(e); } } else { String seqNo = ""; for (Record rc : records) { if (latch != null) { latch.countDown(); } seqNo = rc.getSequenceNumber(); if (getData(rc).equals(KinesisOperatorTestBase.END_TUPLE)) break; holdingBuffer.add(rc); receiveCount++; logger.debug("Consuming {}, receiveCount= {}", getData(rc), receiveCount); } } } logger.debug("DONE consuming"); }
From source file:com.datatorrent.contrib.kinesis.KinesisUtil.java
License:Open Source License
/** * Get the records from the particular shard * @param streamName Name of the stream from where the records to be accessed * @param recordsLimit Number of records to return from shard * @param shId Shard Id of the shard//from w w w . j av a2 s. c om * @param iteratorType Shard iterator type * @param seqNo Record sequence number * @return the list of records from the given shard * @throws AmazonClientException */ public List<Record> getRecords(String streamName, Integer recordsLimit, Shard shId, ShardIteratorType iteratorType, String seqNo) throws AmazonClientException { assert client != null : "Illegal client"; try { // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType to it GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId.getShardId()); iteratorRequest.setShardIteratorType(iteratorType); // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType)) iteratorRequest.setStartingSequenceNumber(seqNo); // Get the Response from the getShardIterator service method & get the shardIterator from that response GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); // getShardIterator() specifies the position in the shard String iterator = iteratorResponse.getShardIterator(); // Create the GetRecordsRequest instance and set the recordsLimit and iterator GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(recordsLimit); getRequest.setShardIterator(iterator); // Get the Response from the getRecords service method and get the data records from that response. GetRecordsResult getResponse = client.getRecords(getRequest); return getResponse.getRecords(); } catch (AmazonClientException e) { throw new RuntimeException(e); } }
From source file:com.srotya.flume.kinesis.source.KinesisSource.java
License:Apache License
protected void getShardIterator() { System.out.println("Listing shards:"); shards.forEach(shard -> System.out.println("Shard id:" + shard.getShardId())); GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(streamName); getShardIteratorRequest.setShardId(shards.get(shardId).getShardId()); getShardIteratorRequest.setShardIteratorType(shardIteratorType); GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest); shardIterator = getShardIteratorResult.getShardIterator(); }
From source file:com.trulia.stail.Stail.java
License:Apache License
private static String getShardIterator(AmazonKinesis client, String stream, Shard shard, String start) { GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(stream); getShardIteratorRequest.setShardId(shard.getShardId()); if (!Strings.isNullOrEmpty(start)) { getShardIteratorRequest.setShardIteratorType(ShardIteratorType.AT_TIMESTAMP); getShardIteratorRequest//from www . j av a 2 s . c o m .setTimestamp(new Date(System.currentTimeMillis() - Duration.parse(start).toMillis())); } else { getShardIteratorRequest.setShardIteratorType(ShardIteratorType.LATEST); } GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest); return getShardIteratorResult.getShardIterator(); }
From source file:com.trulia.stail.Stail.java
License:Apache License
private static String getShardIteratorAtSequenceNumber(AmazonKinesis client, String stream, Shard shard, String sequenceNumber) {/* w ww .ja v a 2s . c o m*/ GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(stream); getShardIteratorRequest.setShardId(shard.getShardId()); getShardIteratorRequest.setShardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER); getShardIteratorRequest.setStartingSequenceNumber(sequenceNumber); GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest); return getShardIteratorResult.getShardIterator(); }
From source file:com.trulia.stail.Stail.java
License:Apache License
private static String getOldestShardIterator(AmazonKinesis client, String stream, Shard shard) { GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(stream); getShardIteratorRequest.setShardId(shard.getShardId()); getShardIteratorRequest.setShardIteratorType(ShardIteratorType.TRIM_HORIZON); GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest); return getShardIteratorResult.getShardIterator(); }
From source file:dbtucker.connect.kinesis.KinesisSourceTask.java
License:Apache License
private GetShardIteratorRequest getShardIteratorRequest(String shardId, String streamName, String seqNum) { final GetShardIteratorRequest req = new GetShardIteratorRequest(); req.setShardId(shardId); req.setStreamName(streamName);//from w w w . j ava 2 s . c o m if (seqNum == null) { req.setShardIteratorType(ShardIteratorType.TRIM_HORIZON); } else { req.setShardIteratorType(ShardIteratorType.AFTER_SEQUENCE_NUMBER); req.setStartingSequenceNumber(seqNum); } return req; }
From source file:org.apache.apex.malhar.contrib.kinesis.KinesisTestConsumer.java
License:Apache License
public String prepareIterator() { DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.setStreamName(streamName); List<Shard> shards = null; for (int i = 0; i < MAX_TRY_TIMES; ++i) { try {// w ww. j a va2 s. c o m DescribeStreamResult describeResponse = client.describeStream(describeRequest); shards = describeResponse.getStreamDescription().getShards(); if (shards.isEmpty()) { logger.warn("shards is empty"); } else { break; } } catch (Exception e) { logger.error("get Stream description exception: ", e); throw new RuntimeException(e); } try { Thread.sleep(1000); } catch (Exception e) { // } } Shard shId = shards.get(0); GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId.getShardId()); iteratorRequest.setShardIteratorType("TRIM_HORIZON"); GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); return iteratorResponse.getShardIterator(); }
From source file:org.apache.apex.malhar.contrib.kinesis.KinesisUtil.java
License:Apache License
/** * Get the records from the particular shard * @param streamName Name of the stream from where the records to be accessed * @param recordsLimit Number of records to return from shard * @param shId Shard Id of the shard/* ww w .j a v a2s .c o m*/ * @param iteratorType Shard iterator type * @param seqNo Record sequence number * @return the list of records from the given shard * @throws AmazonClientException */ public List<Record> getRecords(String streamName, Integer recordsLimit, String shId, ShardIteratorType iteratorType, String seqNo) throws AmazonClientException { assert client != null : "Illegal client"; try { // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType to it GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId); iteratorRequest.setShardIteratorType(iteratorType); // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType) || ShardIteratorType.AT_SEQUENCE_NUMBER.equals(iteratorType)) { iteratorRequest.setStartingSequenceNumber(seqNo); } // Get the Response from the getShardIterator service method & get the shardIterator from that response GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); // getShardIterator() specifies the position in the shard String iterator = iteratorResponse.getShardIterator(); // Create the GetRecordsRequest instance and set the recordsLimit and iterator GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(recordsLimit); getRequest.setShardIterator(iterator); // Get the Response from the getRecords service method and get the data records from that response. GetRecordsResult getResponse = client.getRecords(getRequest); return getResponse.getRecords(); } catch (AmazonClientException e) { throw new RuntimeException(e); } }