List of usage examples for com.amazonaws.services.kinesis.model GetRecordsRequest setShardIterator
public void setShardIterator(String shardIterator)
The position in the shard from which you want to start sequentially reading data records.
From source file:AmazonKinesisGet.java
License:Open Source License
public static void main(String[] args) throws Exception { init();//from w w w. j a va 2 s. c o m final String myStreamName = "philsteststream"; final Integer myStreamSize = 1; // list all of my streams ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesisClient.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } LOG.info("Printing my list of streams : "); // print all of my streams. if (!streamNames.isEmpty()) { System.out.println("List of my streams: "); } for (int i = 0; i < streamNames.size(); i++) { System.out.println(streamNames.get(i)); } //System.out.println(streamNames.get(0)); String myownstream = streamNames.get(0); // Retrieve the Shards from a Stream DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(myownstream); DescribeStreamResult describeStreamResult; List<Shard> shards = new ArrayList<>(); String lastShardId = null; do { describeStreamRequest.setExclusiveStartShardId(lastShardId); describeStreamResult = kinesisClient.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (shards.size() > 0) { lastShardId = shards.get(shards.size() - 1).getShardId(); } } while (describeStreamResult.getStreamDescription().getHasMoreShards()); // Get Data from the Shards in a Stream // Hard-coded to use only 1 shard String shardIterator; GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(myownstream); //get(0) shows hardcoded to 1 stream getShardIteratorRequest.setShardId(shards.get(0).getShardId()); // using TRIM_HORIZON but could use alternatives getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON"); GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest); shardIterator = getShardIteratorResult.getShardIterator(); // Continuously read data records from shard. List<Record> records; while (true) { // Create new GetRecordsRequest with existing shardIterator. // Set maximum records to return to 1000. GetRecordsRequest getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(1000); GetRecordsResult result = kinesisClient.getRecords(getRecordsRequest); // Put result into record list. Result may be empty. records = result.getRecords(); // Print records for (Record record : records) { ByteBuffer byteBuffer = record.getData(); System.out.println(String.format("Seq No: %s - %s", record.getSequenceNumber(), new String(byteBuffer.array()))); } try { Thread.sleep(1000); } catch (InterruptedException exception) { throw new RuntimeException(exception); } shardIterator = result.getNextShardIterator(); } }
From source file:com.datatorrent.contrib.kinesis.KinesisTestConsumer.java
License:Open Source License
@Override public void run() { DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.setStreamName(streamName); DescribeStreamResult describeResponse = client.describeStream(describeRequest); final List<Shard> shards = describeResponse.getStreamDescription().getShards(); logger.debug("Inside consumer::run receiveCount= {}", receiveCount); while (isAlive) { Shard shId = shards.get(0);//w ww . j a va 2 s . c o m GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId.getShardId()); iteratorRequest.setShardIteratorType("TRIM_HORIZON"); GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); String iterator = iteratorResponse.getShardIterator(); GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(1000); getRequest.setShardIterator(iterator); //call "get" operation and get everything in this shard range GetRecordsResult getResponse = client.getRecords(getRequest); //get reference to next iterator for this shard //retrieve records List<Record> records = getResponse.getRecords(); if (records == null || records.isEmpty()) { try { Thread.sleep(1000); } catch (InterruptedException e) { throw new RuntimeException(e); } } else { String seqNo = ""; for (Record rc : records) { if (latch != null) { latch.countDown(); } seqNo = rc.getSequenceNumber(); if (getData(rc).equals(KinesisOperatorTestBase.END_TUPLE)) break; holdingBuffer.add(rc); receiveCount++; logger.debug("Consuming {}, receiveCount= {}", getData(rc), receiveCount); } } } logger.debug("DONE consuming"); }
From source file:com.datatorrent.contrib.kinesis.KinesisUtil.java
License:Open Source License
/** * Get the records from the particular shard * @param streamName Name of the stream from where the records to be accessed * @param recordsLimit Number of records to return from shard * @param shId Shard Id of the shard//from w ww . jav a2 s . c om * @param iteratorType Shard iterator type * @param seqNo Record sequence number * @return the list of records from the given shard * @throws AmazonClientException */ public List<Record> getRecords(String streamName, Integer recordsLimit, Shard shId, ShardIteratorType iteratorType, String seqNo) throws AmazonClientException { assert client != null : "Illegal client"; try { // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType to it GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId.getShardId()); iteratorRequest.setShardIteratorType(iteratorType); // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType)) iteratorRequest.setStartingSequenceNumber(seqNo); // Get the Response from the getShardIterator service method & get the shardIterator from that response GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); // getShardIterator() specifies the position in the shard String iterator = iteratorResponse.getShardIterator(); // Create the GetRecordsRequest instance and set the recordsLimit and iterator GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(recordsLimit); getRequest.setShardIterator(iterator); // Get the Response from the getRecords service method and get the data records from that response. GetRecordsResult getResponse = client.getRecords(getRequest); return getResponse.getRecords(); } catch (AmazonClientException e) { throw new RuntimeException(e); } }
From source file:com.srotya.flume.kinesis.source.KinesisSource.java
License:Apache License
@Override protected Status doProcess() throws EventDeliveryException { Status status = Status.READY;/* ww w .j a v a2 s .c o m*/ GetRecordsRequest recordRequest = new GetRecordsRequest(); recordRequest.setShardIterator(shardIterator); recordRequest.setLimit(putSize); GetRecordsResult records = client.getRecords(recordRequest); for (Record record : records.getRecords()) { try { getChannelProcessor().processEvent(serializer.deserialize(record.getData())); } catch (Exception e) { logger.error("Failed to deserialize event:" + new String(record.getData().array()), e); } } shardIterator = records.getNextShardIterator(); if (shardIterator == null) { getShardIterator(); } return status; }
From source file:com.trulia.stail.Stail.java
License:Apache License
public static void main(String[] args) { final Stail stail = new Stail(); JCommander jct = new JCommander(stail); jct.setProgramName("stail"); try {/* w w w .ja va 2s .c o m*/ jct.parse(args); AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); if (stail.profile != null) { credentialsProvider = new ProfileCredentialsProvider(stail.profile); } if (stail.role != null) { credentialsProvider = new STSAssumeRoleSessionCredentialsProvider.Builder(stail.role, "stail") .withStsClient(AWSSecurityTokenServiceClientBuilder.standard() .withCredentials(credentialsProvider).build()) .build(); } AmazonKinesis client = AmazonKinesisClientBuilder.standard().withRegion(stail.region) .withCredentials(credentialsProvider).build(); // prepare the initial shard iterators at the LATEST position Map<Shard, String> shardIterators = getShardIterators(client, stail.stream, stail.start); IRecordProcessor processor = stail.json ? new JSONRecordProcessor() : new RawRecordProcessor(); Map<Shard, RateLimiter> rateLimiters = new HashMap<>(); shardIterators.keySet() .forEach(shard -> rateLimiters.put(shard, RateLimiter.create(MAX_SHARD_THROUGHPUT))); long end = Strings.isNullOrEmpty(stail.duration) ? Long.MAX_VALUE : System.currentTimeMillis() + Duration.parse(stail.duration).toMillis(); Set<String> reshardedShards = new HashSet<>(); Map<Shard, String> sequenceNumbers = new HashMap<>(); while (System.currentTimeMillis() < end) { if (!reshardedShards.isEmpty()) { // get the new list of shards List<Shard> shards = getShards(client, stail.stream); for (Shard shard : shards) { if (!Strings.isNullOrEmpty(shard.getParentShardId()) && reshardedShards.contains(shard.getParentShardId())) { // the old shard was split, so we need to consume this new shard from the beginning shardIterators.put(shard, getOldestShardIterator(client, stail.stream, shard)); } else if (!Strings.isNullOrEmpty(shard.getAdjacentParentShardId()) && reshardedShards.contains(shard.getAdjacentParentShardId())) { // the old shards were merged into a new shard shardIterators.put(shard, getOldestShardIterator(client, stail.stream, shard)); } } reshardedShards.clear(); } for (Shard shard : Lists.newArrayList(shardIterators.keySet())) { String shardIterator = shardIterators.remove(shard); GetRecordsRequest getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(BATCH_SIZE); try { GetRecordsResult getRecordsResult = client.getRecords(getRecordsRequest); List<Record> records = getRecordsResult.getRecords(); processor.processRecords(records, null); shardIterator = getRecordsResult.getNextShardIterator(); if (records.size() <= 0) { // nothing on the stream yet, so lets wait a bit to see if something appears TimeUnit.SECONDS.sleep(1); } else { int bytesRead = records.stream().map(record -> record.getData().position()) .reduce((_1, _2) -> _1 + _2).get(); sequenceNumbers.put(shard, records.get(records.size() - 1).getSequenceNumber()); // optionally sleep if we have hit the limit for this shard rateLimiters.get(shard).acquire(bytesRead); } if (!Strings.isNullOrEmpty(shardIterator)) { shardIterators.put(shard, shardIterator); } else { reshardedShards.add(shard.getShardId()); } } catch (ProvisionedThroughputExceededException e) { logger.warn("tripped the max throughput. Backing off: {}", e.getMessage()); TimeUnit.SECONDS.sleep(6); // we tripped the max throughput. Back off // add the original iterator back into the map so we can try it again shardIterators.put(shard, shardIterator); } catch (ExpiredIteratorException e) { logger.debug("Iterator expired", e); String sequenceNumber = sequenceNumbers.get(shard); if (sequenceNumber == null) { logger.warn("No previously known sequence number for {}. Moving to LATEST", shard.getShardId()); shardIterators.put(shard, getShardIterator(client, stail.stream, shard, null)); } else { shardIterators.put(shard, getShardIteratorAtSequenceNumber(client, stail.stream, shard, sequenceNumber)); } } } } } catch (ParameterException e) { jct.usage(); System.exit(1); } catch (InterruptedException e) { Thread.currentThread().interrupt(); System.exit(2); } }
From source file:dbtucker.connect.kinesis.KinesisSourceTask.java
License:Apache License
@Override public List<SourceRecord> poll() throws InterruptedException { if (assignedShards.isEmpty()) { throw new ConnectException("No source shards remaining for this task"); }//from w ww . j a v a2s .com // Kinesis best practice is to sleep 1 second between calls to getRecords // We'll do that only after we've cycled through all the shards we're polling if (currentShardIdx == 0) { try { Thread.sleep(1000); } catch (InterruptedException exception) { throw exception; } } final String shardUid = assignedShards.get(currentShardIdx); final GetRecordsRequest req = new GetRecordsRequest(); req.setShardIterator(toShardIterator(shardUid)); req.setLimit(config.getRecPerReq()); final GetRecordsResult rsp = client.getRecords(req); log.info("client.getRecords retrieve {} records", rsp.getRecords().size()); log.debug("client.getRecords returns {}", rsp.toString()); if (rsp.getNextShardIterator() == null) { log.info("Shard ID `{}` for stream `{}` has been closed, it will no longer be polled", shardUid.split("/")[1], shardUid.split("/")[0]); shardIterators.remove(shardUid); assignedShards.remove(shardUid); } else { shardIterators.put(shardUid, rsp.getNextShardIterator()); } currentShardIdx = (currentShardIdx + 1) % assignedShards.size(); final String streamName = shardUid.split("/")[0]; final String topic = config.getTopicFormat().replace("${stream}", streamName); final Map<String, String> sourcePartition = toSourcePartition(shardUid); return rsp.getRecords().stream().map(kinesisRecord -> toSourceRecord(sourcePartition, topic, kinesisRecord)) .collect(Collectors.toList()); }
From source file:org.apache.apex.malhar.contrib.kinesis.KinesisTestConsumer.java
License:Apache License
public String processNextIterator(String iterator) { GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(1000);//from w w w . j a va 2 s . co m getRequest.setShardIterator(iterator); // call "get" operation and get everything in this shard range GetRecordsResult getResponse = client.getRecords(getRequest); iterator = getResponse.getNextShardIterator(); List<Record> records = getResponse.getRecords(); processResponseRecords(records); return iterator; }
From source file:org.apache.apex.malhar.contrib.kinesis.KinesisUtil.java
License:Apache License
/** * Get the records from the particular shard * @param streamName Name of the stream from where the records to be accessed * @param recordsLimit Number of records to return from shard * @param shId Shard Id of the shard//from w w w .j ava 2s . c o m * @param iteratorType Shard iterator type * @param seqNo Record sequence number * @return the list of records from the given shard * @throws AmazonClientException */ public List<Record> getRecords(String streamName, Integer recordsLimit, String shId, ShardIteratorType iteratorType, String seqNo) throws AmazonClientException { assert client != null : "Illegal client"; try { // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType to it GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId); iteratorRequest.setShardIteratorType(iteratorType); // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType) || ShardIteratorType.AT_SEQUENCE_NUMBER.equals(iteratorType)) { iteratorRequest.setStartingSequenceNumber(seqNo); } // Get the Response from the getShardIterator service method & get the shardIterator from that response GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); // getShardIterator() specifies the position in the shard String iterator = iteratorResponse.getShardIterator(); // Create the GetRecordsRequest instance and set the recordsLimit and iterator GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(recordsLimit); getRequest.setShardIterator(iterator); // Get the Response from the getRecords service method and get the data records from that response. GetRecordsResult getResponse = client.getRecords(getRequest); return getResponse.getRecords(); } catch (AmazonClientException e) { throw new RuntimeException(e); } }
From source file:org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxy.java
License:Apache License
/** * {@inheritDoc}//from w w w.j av a2 s . c o m */ @Override public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException { final GetRecordsRequest getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(maxRecordsToGet); GetRecordsResult getRecordsResult = null; int attempt = 0; while (attempt <= getRecordsMaxAttempts && getRecordsResult == null) { try { getRecordsResult = kinesisClient.getRecords(getRecordsRequest); } catch (AmazonServiceException ex) { if (isRecoverableException(ex)) { long backoffMillis = fullJitterBackoff(getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, attempt++); LOG.warn("Got recoverable AmazonServiceException. Backing off for " + backoffMillis + " millis (" + ex.getErrorMessage() + ")"); Thread.sleep(backoffMillis); } else { throw ex; } } } if (getRecordsResult == null) { throw new RuntimeException("Rate Exceeded for getRecords operation - all " + getRecordsMaxAttempts + " retry attempts returned ProvisionedThroughputExceededException."); } return getRecordsResult; }
From source file:org.apache.storm.kinesis.spout.KinesisConnection.java
License:Apache License
GetRecordsResult fetchRecords(String shardIterator) {
GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
getRecordsRequest.setShardIterator(shardIterator);
getRecordsRequest.setLimit(kinesisConnectionInfo.getRecordsLimit());
GetRecordsResult getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
return getRecordsResult;
}