List of usage examples for com.amazonaws.services.kinesis.model ShardIteratorType AT_TIMESTAMP
ShardIteratorType AT_TIMESTAMP
To view the source code for com.amazonaws.services.kinesis.model ShardIteratorType AT_TIMESTAMP.
Click Source Link
From source file:com.trulia.stail.Stail.java
License:Apache License
private static String getShardIterator(AmazonKinesis client, String stream, Shard shard, String start) { GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(stream); getShardIteratorRequest.setShardId(shard.getShardId()); if (!Strings.isNullOrEmpty(start)) { getShardIteratorRequest.setShardIteratorType(ShardIteratorType.AT_TIMESTAMP); getShardIteratorRequest// w w w . j a v a 2 s. c om .setTimestamp(new Date(System.currentTimeMillis() - Duration.parse(start).toMillis())); } else { getShardIteratorRequest.setShardIteratorType(ShardIteratorType.LATEST); } GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest); return getShardIteratorResult.getShardIterator(); }
From source file:org.apache.beam.sdk.io.kinesis.source.checkpoint.StartingPoint.java
License:Apache License
public String getPositionName() { return position != null ? position.name() : ShardIteratorType.AT_TIMESTAMP.name(); }
From source file:org.apache.flink.streaming.connectors.kinesis.internals.ShardConsumer.java
License:Apache License
@SuppressWarnings("unchecked") @Override//from w ww . j av a 2s . c om public void run() { String nextShardItr; try { // before infinitely looping, we set the initial nextShardItr appropriately if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_LATEST_SEQUENCE_NUM.get())) { // if the shard is already closed, there will be no latest next record to get for this shard if (subscribedShard.isClosed()) { nextShardItr = null; } else { nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.LATEST.toString(), null); } } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get())) { nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.TRIM_HORIZON.toString(), null); } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get())) { nextShardItr = null; } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_AT_TIMESTAMP_SEQUENCE_NUM.get())) { nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AT_TIMESTAMP.toString(), initTimestamp); } else { // we will be starting from an actual sequence number (due to restore from failure). // if the last sequence number refers to an aggregated record, we need to clean up any dangling sub-records // from the last aggregated record; otherwise, we can simply start iterating from the record right after. if (lastSequenceNum.isAggregated()) { String itrForLastAggregatedRecord = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AT_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber()); // get only the last aggregated record GetRecordsResult getRecordsResult = getRecords(itrForLastAggregatedRecord, 1); List<UserRecord> fetchedRecords = deaggregateRecords(getRecordsResult.getRecords(), subscribedShard.getShard().getHashKeyRange().getStartingHashKey(), subscribedShard.getShard().getHashKeyRange().getEndingHashKey()); long lastSubSequenceNum = lastSequenceNum.getSubSequenceNumber(); for (UserRecord record : fetchedRecords) { // we have found a dangling sub-record if it has a larger subsequence number // than our last sequence number; if so, collect the record and update state if (record.getSubSequenceNumber() > lastSubSequenceNum) { deserializeRecordForCollectionAndUpdateState(record); } } // set the nextShardItr so we can continue iterating in the next while loop nextShardItr = getRecordsResult.getNextShardIterator(); } else { // the last record was non-aggregated, so we can simply start from the next record nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber()); } } while (isRunning()) { if (nextShardItr == null) { fetcherRef.updateState(subscribedShardStateIndex, SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get()); // we can close this consumer thread once we've reached the end of the subscribed shard break; } else { if (fetchIntervalMillis != 0) { Thread.sleep(fetchIntervalMillis); } GetRecordsResult getRecordsResult = getRecords(nextShardItr, maxNumberOfRecordsPerFetch); // each of the Kinesis records may be aggregated, so we must deaggregate them before proceeding List<UserRecord> fetchedRecords = deaggregateRecords(getRecordsResult.getRecords(), subscribedShard.getShard().getHashKeyRange().getStartingHashKey(), subscribedShard.getShard().getHashKeyRange().getEndingHashKey()); for (UserRecord record : fetchedRecords) { deserializeRecordForCollectionAndUpdateState(record); } nextShardItr = getRecordsResult.getNextShardIterator(); } } } catch (Throwable t) { fetcherRef.stopWithError(t); } }
From source file:org.apache.storm.kinesis.spout.Config.java
License:Apache License
private void validate() { if (streamName == null || streamName.length() < 1) { throw new IllegalArgumentException("streamName is required and cannot be of length 0."); }// www.j a v a 2s. com if (shardIteratorType == null || shardIteratorType.equals(ShardIteratorType.AFTER_SEQUENCE_NUMBER) || shardIteratorType.equals(ShardIteratorType.AT_SEQUENCE_NUMBER)) { throw new IllegalArgumentException( "shardIteratorType has to be one of the " + ShardIteratorType.AT_TIMESTAMP + "," + ShardIteratorType.LATEST + "," + ShardIteratorType.TRIM_HORIZON); } if (shardIteratorType.equals(ShardIteratorType.AT_TIMESTAMP) && timestamp == null) { throw new IllegalArgumentException( "timestamp must be provided if shardIteratorType is " + ShardIteratorType.AT_TIMESTAMP); } if (recordToTupleMapper == null) { throw new IllegalArgumentException("recordToTupleMapper cannot be null"); } if (failedMessageRetryHandler == null) { throw new IllegalArgumentException("failedMessageRetryHandler cannot be null"); } if (zkInfo == null) { throw new IllegalArgumentException("zkInfo cannot be null"); } if (kinesisConnectionInfo == null) { throw new IllegalArgumentException("kinesisConnectionInfo cannot be null"); } if (maxUncommittedRecords == null || maxUncommittedRecords < 1) { throw new IllegalArgumentException("maxUncommittedRecords has to be a positive integer"); } }
From source file:org.apache.storm.kinesis.spout.KinesisConnection.java
License:Apache License
String getShardIterator(String stream, String shardId, ShardIteratorType shardIteratorType,
String sequenceNumber, Date timestamp) {
String shardIterator = "";
try {/*from w w w . ja v a2 s . com*/
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
getShardIteratorRequest.setStreamName(stream);
getShardIteratorRequest.setShardId(shardId);
getShardIteratorRequest.setShardIteratorType(shardIteratorType);
if (shardIteratorType.equals(ShardIteratorType.AFTER_SEQUENCE_NUMBER)
|| shardIteratorType.equals(ShardIteratorType.AT_SEQUENCE_NUMBER)) {
getShardIteratorRequest.setStartingSequenceNumber(sequenceNumber);
} else if (shardIteratorType.equals(ShardIteratorType.AT_TIMESTAMP)) {
getShardIteratorRequest.setTimestamp(timestamp);
}
GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
if (getShardIteratorResult != null) {
shardIterator = getShardIteratorResult.getShardIterator();
}
} catch (Exception e) {
LOG.warn(
"Exception occured while getting shardIterator for shard " + shardId + " shardIteratorType "
+ shardIteratorType + " sequence number " + sequenceNumber + " timestamp " + timestamp,
e);
}
LOG.warn("Returning shardIterator " + shardIterator + " for shardId " + shardId + " shardIteratorType "
+ shardIteratorType + " sequenceNumber " + sequenceNumber + " timestamp" + timestamp);
return shardIterator;
}
From source file:org.springframework.cloud.stream.binder.kinesis.KinesisMessageChannelBinder.java
License:Apache License
private MessageProducer createKinesisConsumerEndpoint(ConsumerDestination destination, String group, ExtendedConsumerProperties<KinesisConsumerProperties> properties) { KinesisConsumerProperties kinesisConsumerProperties = properties.getExtension(); Set<KinesisShardOffset> shardOffsets = null; String shardIteratorType = kinesisConsumerProperties.getShardIteratorType(); KinesisShardOffset kinesisShardOffset = KinesisShardOffset.latest(); if (StringUtils.hasText(shardIteratorType)) { String[] typeValue = shardIteratorType.split(":", 2); ShardIteratorType iteratorType = ShardIteratorType.valueOf(typeValue[0]); kinesisShardOffset = new KinesisShardOffset(iteratorType); if (typeValue.length > 1) { if (ShardIteratorType.AT_TIMESTAMP.equals(iteratorType)) { kinesisShardOffset.setTimestamp(new Date(Long.parseLong(typeValue[1]))); } else { kinesisShardOffset.setSequenceNumber(typeValue[1]); }/*from w w w. j av a 2 s .co m*/ } } if (properties.getInstanceCount() > 1) { shardOffsets = new HashSet<>(); KinesisConsumerDestination kinesisConsumerDestination = (KinesisConsumerDestination) destination; List<Shard> shards = kinesisConsumerDestination.getShards(); for (int i = 0; i < shards.size(); i++) { // divide shards across instances if ((i % properties.getInstanceCount()) == properties.getInstanceIndex()) { KinesisShardOffset shardOffset = new KinesisShardOffset(kinesisShardOffset); shardOffset.setStream(destination.getName()); shardOffset.setShard(shards.get(i).getShardId()); shardOffsets.add(shardOffset); } } } KinesisMessageDrivenChannelAdapter adapter; if (CollectionUtils.isEmpty(shardOffsets)) { adapter = new KinesisMessageDrivenChannelAdapter(this.amazonKinesis, destination.getName()); } else { adapter = new KinesisMessageDrivenChannelAdapter(this.amazonKinesis, shardOffsets.toArray(new KinesisShardOffset[0])); } boolean anonymous = !StringUtils.hasText(group); String consumerGroup = anonymous ? "anonymous." + UUID.randomUUID().toString() : group; adapter.setConsumerGroup(consumerGroup); adapter.setStreamInitialSequence(anonymous || StringUtils.hasText(shardIteratorType) ? kinesisShardOffset : KinesisShardOffset.trimHorizon()); adapter.setListenerMode(kinesisConsumerProperties.getListenerMode()); if (properties.isUseNativeDecoding()) { adapter.setConverter(null); } else { // Defer byte[] conversion to the InboundContentTypeConvertingInterceptor adapter.setConverter((bytes) -> bytes); } adapter.setCheckpointMode(kinesisConsumerProperties.getCheckpointMode()); adapter.setRecordsLimit(kinesisConsumerProperties.getRecordsLimit()); adapter.setIdleBetweenPolls(kinesisConsumerProperties.getIdleBetweenPolls()); adapter.setConsumerBackoff(kinesisConsumerProperties.getConsumerBackoff()); if (this.configurationProperties.getCheckpoint().getInterval() != null) { adapter.setCheckpointsInterval(this.configurationProperties.getCheckpoint().getInterval()); } if (this.checkpointStore != null) { adapter.setCheckpointStore(this.checkpointStore); } adapter.setLockRegistry(this.lockRegistry); adapter.setConcurrency(properties.getConcurrency()); adapter.setStartTimeout(kinesisConsumerProperties.getStartTimeout()); adapter.setDescribeStreamBackoff(this.configurationProperties.getDescribeStreamBackoff()); adapter.setDescribeStreamRetries(this.configurationProperties.getDescribeStreamRetries()); ErrorInfrastructure errorInfrastructure = registerErrorInfrastructure(destination, consumerGroup, properties); adapter.setErrorMessageStrategy(ERROR_MESSAGE_STRATEGY); adapter.setErrorChannel(errorInfrastructure.getErrorChannel()); return adapter; }
From source file:org.springframework.integration.aws.inbound.kinesis.KinesisShardOffset.java
License:Apache License
public static KinesisShardOffset atTimestamp(String stream, String shard, Date timestamp) { KinesisShardOffset kinesisShardOffset = new KinesisShardOffset(ShardIteratorType.AT_TIMESTAMP); kinesisShardOffset.stream = stream;/*from ww w . jav a 2s .c o m*/ kinesisShardOffset.shard = shard; kinesisShardOffset.timestamp = timestamp; return kinesisShardOffset; }