List of usage examples for com.amazonaws.services.kinesis.model GetRecordsResult getNextShardIterator
public String getNextShardIterator()
The next position in the shard from which to start sequentially reading data records.
From source file:AmazonKinesisGet.java
License:Open Source License
public static void main(String[] args) throws Exception { init();// w w w . j ava 2 s . co m final String myStreamName = "philsteststream"; final Integer myStreamSize = 1; // list all of my streams ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesisClient.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } LOG.info("Printing my list of streams : "); // print all of my streams. if (!streamNames.isEmpty()) { System.out.println("List of my streams: "); } for (int i = 0; i < streamNames.size(); i++) { System.out.println(streamNames.get(i)); } //System.out.println(streamNames.get(0)); String myownstream = streamNames.get(0); // Retrieve the Shards from a Stream DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(myownstream); DescribeStreamResult describeStreamResult; List<Shard> shards = new ArrayList<>(); String lastShardId = null; do { describeStreamRequest.setExclusiveStartShardId(lastShardId); describeStreamResult = kinesisClient.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (shards.size() > 0) { lastShardId = shards.get(shards.size() - 1).getShardId(); } } while (describeStreamResult.getStreamDescription().getHasMoreShards()); // Get Data from the Shards in a Stream // Hard-coded to use only 1 shard String shardIterator; GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(myownstream); //get(0) shows hardcoded to 1 stream getShardIteratorRequest.setShardId(shards.get(0).getShardId()); // using TRIM_HORIZON but could use alternatives getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON"); GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest); shardIterator = getShardIteratorResult.getShardIterator(); // Continuously read data records from shard. List<Record> records; while (true) { // Create new GetRecordsRequest with existing shardIterator. // Set maximum records to return to 1000. GetRecordsRequest getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(1000); GetRecordsResult result = kinesisClient.getRecords(getRecordsRequest); // Put result into record list. Result may be empty. records = result.getRecords(); // Print records for (Record record : records) { ByteBuffer byteBuffer = record.getData(); System.out.println(String.format("Seq No: %s - %s", record.getSequenceNumber(), new String(byteBuffer.array()))); } try { Thread.sleep(1000); } catch (InterruptedException exception) { throw new RuntimeException(exception); } shardIterator = result.getNextShardIterator(); } }
From source file:com.netflix.spectator.tdigest.KinesisTDigestReader.java
License:Apache License
@Override public List<TDigestMeasurement> read() throws IOException { init();/* w ww . ja va 2s .c o m*/ if (recRequest.getShardIterator() == null) { return Collections.emptyList(); } else { List<TDigestMeasurement> ms = new ArrayList<>(); GetRecordsResult result = client.getRecords(recRequest); recRequest.setShardIterator(result.getNextShardIterator()); for (Record r : result.getRecords()) { recordsProcessed.increment(); ByteBuffer data = r.getData(); try { ms.addAll(json.decode(data.array())); } catch (Exception e) { recordsSkipped.increment(); LOGGER.warn("failed to decode record, skipping (" + iterRequest + ")", e); } } return ms; } }
From source file:com.srotya.flume.kinesis.source.KinesisSource.java
License:Apache License
@Override protected Status doProcess() throws EventDeliveryException { Status status = Status.READY;//from ww w . ja v a 2 s . c o m GetRecordsRequest recordRequest = new GetRecordsRequest(); recordRequest.setShardIterator(shardIterator); recordRequest.setLimit(putSize); GetRecordsResult records = client.getRecords(recordRequest); for (Record record : records.getRecords()) { try { getChannelProcessor().processEvent(serializer.deserialize(record.getData())); } catch (Exception e) { logger.error("Failed to deserialize event:" + new String(record.getData().array()), e); } } shardIterator = records.getNextShardIterator(); if (shardIterator == null) { getShardIterator(); } return status; }
From source file:com.trulia.stail.Stail.java
License:Apache License
public static void main(String[] args) { final Stail stail = new Stail(); JCommander jct = new JCommander(stail); jct.setProgramName("stail"); try {/*w ww .j a v a 2s . c o m*/ jct.parse(args); AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); if (stail.profile != null) { credentialsProvider = new ProfileCredentialsProvider(stail.profile); } if (stail.role != null) { credentialsProvider = new STSAssumeRoleSessionCredentialsProvider.Builder(stail.role, "stail") .withStsClient(AWSSecurityTokenServiceClientBuilder.standard() .withCredentials(credentialsProvider).build()) .build(); } AmazonKinesis client = AmazonKinesisClientBuilder.standard().withRegion(stail.region) .withCredentials(credentialsProvider).build(); // prepare the initial shard iterators at the LATEST position Map<Shard, String> shardIterators = getShardIterators(client, stail.stream, stail.start); IRecordProcessor processor = stail.json ? new JSONRecordProcessor() : new RawRecordProcessor(); Map<Shard, RateLimiter> rateLimiters = new HashMap<>(); shardIterators.keySet() .forEach(shard -> rateLimiters.put(shard, RateLimiter.create(MAX_SHARD_THROUGHPUT))); long end = Strings.isNullOrEmpty(stail.duration) ? Long.MAX_VALUE : System.currentTimeMillis() + Duration.parse(stail.duration).toMillis(); Set<String> reshardedShards = new HashSet<>(); Map<Shard, String> sequenceNumbers = new HashMap<>(); while (System.currentTimeMillis() < end) { if (!reshardedShards.isEmpty()) { // get the new list of shards List<Shard> shards = getShards(client, stail.stream); for (Shard shard : shards) { if (!Strings.isNullOrEmpty(shard.getParentShardId()) && reshardedShards.contains(shard.getParentShardId())) { // the old shard was split, so we need to consume this new shard from the beginning shardIterators.put(shard, getOldestShardIterator(client, stail.stream, shard)); } else if (!Strings.isNullOrEmpty(shard.getAdjacentParentShardId()) && reshardedShards.contains(shard.getAdjacentParentShardId())) { // the old shards were merged into a new shard shardIterators.put(shard, getOldestShardIterator(client, stail.stream, shard)); } } reshardedShards.clear(); } for (Shard shard : Lists.newArrayList(shardIterators.keySet())) { String shardIterator = shardIterators.remove(shard); GetRecordsRequest getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(BATCH_SIZE); try { GetRecordsResult getRecordsResult = client.getRecords(getRecordsRequest); List<Record> records = getRecordsResult.getRecords(); processor.processRecords(records, null); shardIterator = getRecordsResult.getNextShardIterator(); if (records.size() <= 0) { // nothing on the stream yet, so lets wait a bit to see if something appears TimeUnit.SECONDS.sleep(1); } else { int bytesRead = records.stream().map(record -> record.getData().position()) .reduce((_1, _2) -> _1 + _2).get(); sequenceNumbers.put(shard, records.get(records.size() - 1).getSequenceNumber()); // optionally sleep if we have hit the limit for this shard rateLimiters.get(shard).acquire(bytesRead); } if (!Strings.isNullOrEmpty(shardIterator)) { shardIterators.put(shard, shardIterator); } else { reshardedShards.add(shard.getShardId()); } } catch (ProvisionedThroughputExceededException e) { logger.warn("tripped the max throughput. Backing off: {}", e.getMessage()); TimeUnit.SECONDS.sleep(6); // we tripped the max throughput. Back off // add the original iterator back into the map so we can try it again shardIterators.put(shard, shardIterator); } catch (ExpiredIteratorException e) { logger.debug("Iterator expired", e); String sequenceNumber = sequenceNumbers.get(shard); if (sequenceNumber == null) { logger.warn("No previously known sequence number for {}. Moving to LATEST", shard.getShardId()); shardIterators.put(shard, getShardIterator(client, stail.stream, shard, null)); } else { shardIterators.put(shard, getShardIteratorAtSequenceNumber(client, stail.stream, shard, sequenceNumber)); } } } } } catch (ParameterException e) { jct.usage(); System.exit(1); } catch (InterruptedException e) { Thread.currentThread().interrupt(); System.exit(2); } }
From source file:dbtucker.connect.kinesis.KinesisSourceTask.java
License:Apache License
@Override public List<SourceRecord> poll() throws InterruptedException { if (assignedShards.isEmpty()) { throw new ConnectException("No source shards remaining for this task"); }//from www .j a va2 s. c o m // Kinesis best practice is to sleep 1 second between calls to getRecords // We'll do that only after we've cycled through all the shards we're polling if (currentShardIdx == 0) { try { Thread.sleep(1000); } catch (InterruptedException exception) { throw exception; } } final String shardUid = assignedShards.get(currentShardIdx); final GetRecordsRequest req = new GetRecordsRequest(); req.setShardIterator(toShardIterator(shardUid)); req.setLimit(config.getRecPerReq()); final GetRecordsResult rsp = client.getRecords(req); log.info("client.getRecords retrieve {} records", rsp.getRecords().size()); log.debug("client.getRecords returns {}", rsp.toString()); if (rsp.getNextShardIterator() == null) { log.info("Shard ID `{}` for stream `{}` has been closed, it will no longer be polled", shardUid.split("/")[1], shardUid.split("/")[0]); shardIterators.remove(shardUid); assignedShards.remove(shardUid); } else { shardIterators.put(shardUid, rsp.getNextShardIterator()); } currentShardIdx = (currentShardIdx + 1) % assignedShards.size(); final String streamName = shardUid.split("/")[0]; final String topic = config.getTopicFormat().replace("${stream}", streamName); final Map<String, String> sourcePartition = toSourcePartition(shardUid); return rsp.getRecords().stream().map(kinesisRecord -> toSourceRecord(sourcePartition, topic, kinesisRecord)) .collect(Collectors.toList()); }
From source file:org.apache.apex.malhar.contrib.kinesis.KinesisTestConsumer.java
License:Apache License
public String processNextIterator(String iterator) { GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(1000);// w w w . ja v a 2 s . c o m getRequest.setShardIterator(iterator); // call "get" operation and get everything in this shard range GetRecordsResult getResponse = client.getRecords(getRequest); iterator = getResponse.getNextShardIterator(); List<Record> records = getResponse.getRecords(); processResponseRecords(records); return iterator; }
From source file:org.apache.beam.sdk.io.kinesis.client.SimplifiedKinesisClient.java
License:Apache License
/*** * Gets records from Kinesis and deaggregates them if needed. * * @return list of deaggregated records/*w w w .jav a2 s .c o m*/ * @throws IOException - in case of recoverable situation */ public GetKinesisRecordsResult getRecords(final String shardIterator, final String streamName, final String shardId, final Integer limit) throws TransientKinesisException { return wrapExceptions(new Callable<GetKinesisRecordsResult>() { @Override public GetKinesisRecordsResult call() throws Exception { GetRecordsResult response = kinesis .getRecords(new GetRecordsRequest().withShardIterator(shardIterator).withLimit(limit)); return new GetKinesisRecordsResult(UserRecord.deaggregate(response.getRecords()), response.getNextShardIterator(), streamName, shardId); } }); }
From source file:org.apache.beam.sdk.io.kinesis.SimplifiedKinesisClient.java
License:Apache License
/** * Gets records from Kinesis and deaggregates them if needed. * * @return list of deaggregated records/*w w w . j a va 2s .co m*/ * @throws TransientKinesisException - in case of recoverable situation */ public GetKinesisRecordsResult getRecords(final String shardIterator, final String streamName, final String shardId, final Integer limit) throws TransientKinesisException { return wrapExceptions(() -> { GetRecordsResult response = kinesis .getRecords(new GetRecordsRequest().withShardIterator(shardIterator).withLimit(limit)); return new GetKinesisRecordsResult(UserRecord.deaggregate(response.getRecords()), response.getNextShardIterator(), response.getMillisBehindLatest(), streamName, shardId); }); }
From source file:org.apache.druid.indexing.kinesis.KinesisRecordSupplier.java
License:Apache License
@Nullable private String getSequenceNumberInternal(StreamPartition<String> partition, String shardIterator) { long timeoutMillis = System.currentTimeMillis() + fetchSequenceNumberTimeout; while (shardIterator != null && System.currentTimeMillis() < timeoutMillis) { if (closed) { log.info("KinesisRecordSupplier closed while fetching sequenceNumber"); return null; }//from w w w . ja va2s . c o m GetRecordsResult recordsResult; try { // we call getRecords with limit 1000 to make sure that we can find the first (earliest) record in the shard. // In the case where the shard is constantly removing records that are past their retention period, it is possible // that we never find the first record in the shard if we use a limit of 1. recordsResult = kinesis .getRecords(new GetRecordsRequest().withShardIterator(shardIterator).withLimit(1000)); } catch (ProvisionedThroughputExceededException e) { log.warn(e, "encountered ProvisionedThroughputExceededException while fetching records, this means " + "that the request rate for the stream is too high, or the requested data is too large for " + "the available throughput. Reduce the frequency or size of your requests. Consider increasing " + "the number of shards to increase throughput."); try { Thread.sleep(PROVISIONED_THROUGHPUT_EXCEEDED_BACKOFF_MS); continue; } catch (InterruptedException e1) { log.warn(e1, "Thread interrupted!"); Thread.currentThread().interrupt(); break; } } List<Record> records = recordsResult.getRecords(); if (!records.isEmpty()) { return records.get(0).getSequenceNumber(); } shardIterator = recordsResult.getNextShardIterator(); } if (shardIterator == null) { log.info("Partition[%s] returned a null shard iterator, is the shard closed?", partition.getPartitionId()); return KinesisSequenceNumber.END_OF_SHARD_MARKER; } // if we reach here, it usually means either the shard has no more records, or records have not been // added to this shard log.warn("timed out while trying to fetch position for shard[%s], likely no more records in shard", partition.getPartitionId()); return null; }
From source file:org.apache.flink.streaming.connectors.kinesis.internals.ShardConsumer.java
License:Apache License
@SuppressWarnings("unchecked") @Override//from w w w . ja va 2s. com public void run() { String nextShardItr; try { // before infinitely looping, we set the initial nextShardItr appropriately if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_LATEST_SEQUENCE_NUM.get())) { // if the shard is already closed, there will be no latest next record to get for this shard if (subscribedShard.isClosed()) { nextShardItr = null; } else { nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.LATEST.toString(), null); } } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get())) { nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.TRIM_HORIZON.toString(), null); } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get())) { nextShardItr = null; } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_AT_TIMESTAMP_SEQUENCE_NUM.get())) { nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AT_TIMESTAMP.toString(), initTimestamp); } else { // we will be starting from an actual sequence number (due to restore from failure). // if the last sequence number refers to an aggregated record, we need to clean up any dangling sub-records // from the last aggregated record; otherwise, we can simply start iterating from the record right after. if (lastSequenceNum.isAggregated()) { String itrForLastAggregatedRecord = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AT_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber()); // get only the last aggregated record GetRecordsResult getRecordsResult = getRecords(itrForLastAggregatedRecord, 1); List<UserRecord> fetchedRecords = deaggregateRecords(getRecordsResult.getRecords(), subscribedShard.getShard().getHashKeyRange().getStartingHashKey(), subscribedShard.getShard().getHashKeyRange().getEndingHashKey()); long lastSubSequenceNum = lastSequenceNum.getSubSequenceNumber(); for (UserRecord record : fetchedRecords) { // we have found a dangling sub-record if it has a larger subsequence number // than our last sequence number; if so, collect the record and update state if (record.getSubSequenceNumber() > lastSubSequenceNum) { deserializeRecordForCollectionAndUpdateState(record); } } // set the nextShardItr so we can continue iterating in the next while loop nextShardItr = getRecordsResult.getNextShardIterator(); } else { // the last record was non-aggregated, so we can simply start from the next record nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber()); } } while (isRunning()) { if (nextShardItr == null) { fetcherRef.updateState(subscribedShardStateIndex, SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get()); // we can close this consumer thread once we've reached the end of the subscribed shard break; } else { if (fetchIntervalMillis != 0) { Thread.sleep(fetchIntervalMillis); } GetRecordsResult getRecordsResult = getRecords(nextShardItr, maxNumberOfRecordsPerFetch); // each of the Kinesis records may be aggregated, so we must deaggregate them before proceeding List<UserRecord> fetchedRecords = deaggregateRecords(getRecordsResult.getRecords(), subscribedShard.getShard().getHashKeyRange().getStartingHashKey(), subscribedShard.getShard().getHashKeyRange().getEndingHashKey()); for (UserRecord record : fetchedRecords) { deserializeRecordForCollectionAndUpdateState(record); } nextShardItr = getRecordsResult.getNextShardIterator(); } } } catch (Throwable t) { fetcherRef.stopWithError(t); } }