List of usage examples for com.amazonaws.services.kinesis.model ShardIteratorType AFTER_SEQUENCE_NUMBER
ShardIteratorType AFTER_SEQUENCE_NUMBER
To view the source code for com.amazonaws.services.kinesis.model ShardIteratorType AFTER_SEQUENCE_NUMBER.
Click Source Link
From source file:com.datatorrent.contrib.kinesis.KinesisConsumer.java
License:Open Source License
/** * This method returns the iterator type of the given shard *//*from w w w.j a va 2 s. com*/ public ShardIteratorType getIteratorType(String shardId) { if (shardPosition.containsKey(shardId)) { return ShardIteratorType.AFTER_SEQUENCE_NUMBER; } return initialOffset.equalsIgnoreCase("earliest") ? ShardIteratorType.TRIM_HORIZON : ShardIteratorType.LATEST; }
From source file:com.datatorrent.contrib.kinesis.KinesisUtil.java
License:Open Source License
/** * Get the records from the particular shard * @param streamName Name of the stream from where the records to be accessed * @param recordsLimit Number of records to return from shard * @param shId Shard Id of the shard/*from ww w . j a v a 2s .c om*/ * @param iteratorType Shard iterator type * @param seqNo Record sequence number * @return the list of records from the given shard * @throws AmazonClientException */ public List<Record> getRecords(String streamName, Integer recordsLimit, Shard shId, ShardIteratorType iteratorType, String seqNo) throws AmazonClientException { assert client != null : "Illegal client"; try { // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType to it GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId.getShardId()); iteratorRequest.setShardIteratorType(iteratorType); // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType)) iteratorRequest.setStartingSequenceNumber(seqNo); // Get the Response from the getShardIterator service method & get the shardIterator from that response GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); // getShardIterator() specifies the position in the shard String iterator = iteratorResponse.getShardIterator(); // Create the GetRecordsRequest instance and set the recordsLimit and iterator GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(recordsLimit); getRequest.setShardIterator(iterator); // Get the Response from the getRecords service method and get the data records from that response. GetRecordsResult getResponse = client.getRecords(getRequest); return getResponse.getRecords(); } catch (AmazonClientException e) { throw new RuntimeException(e); } }
From source file:dbtucker.connect.kinesis.KinesisSourceTask.java
License:Apache License
private GetShardIteratorRequest getShardIteratorRequest(String shardId, String streamName, String seqNum) { final GetShardIteratorRequest req = new GetShardIteratorRequest(); req.setShardId(shardId);//w w w. j a va 2 s .com req.setStreamName(streamName); if (seqNum == null) { req.setShardIteratorType(ShardIteratorType.TRIM_HORIZON); } else { req.setShardIteratorType(ShardIteratorType.AFTER_SEQUENCE_NUMBER); req.setStartingSequenceNumber(seqNum); } return req; }
From source file:org.apache.apex.malhar.contrib.kinesis.KinesisUtil.java
License:Apache License
/** * Get the records from the particular shard * @param streamName Name of the stream from where the records to be accessed * @param recordsLimit Number of records to return from shard * @param shId Shard Id of the shard/*from www . j a v a 2 s.com*/ * @param iteratorType Shard iterator type * @param seqNo Record sequence number * @return the list of records from the given shard * @throws AmazonClientException */ public List<Record> getRecords(String streamName, Integer recordsLimit, String shId, ShardIteratorType iteratorType, String seqNo) throws AmazonClientException { assert client != null : "Illegal client"; try { // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType to it GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId); iteratorRequest.setShardIteratorType(iteratorType); // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType) || ShardIteratorType.AT_SEQUENCE_NUMBER.equals(iteratorType)) { iteratorRequest.setStartingSequenceNumber(seqNo); } // Get the Response from the getShardIterator service method & get the shardIterator from that response GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); // getShardIterator() specifies the position in the shard String iterator = iteratorResponse.getShardIterator(); // Create the GetRecordsRequest instance and set the recordsLimit and iterator GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(recordsLimit); getRequest.setShardIterator(iterator); // Get the Response from the getRecords service method and get the data records from that response. GetRecordsResult getResponse = client.getRecords(getRequest); return getResponse.getRecords(); } catch (AmazonClientException e) { throw new RuntimeException(e); } }
From source file:org.apache.flink.streaming.connectors.kinesis.internals.ShardConsumer.java
License:Apache License
@SuppressWarnings("unchecked") @Override/*from www. j ava 2 s . c o m*/ public void run() { String nextShardItr; try { // before infinitely looping, we set the initial nextShardItr appropriately if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_LATEST_SEQUENCE_NUM.get())) { // if the shard is already closed, there will be no latest next record to get for this shard if (subscribedShard.isClosed()) { nextShardItr = null; } else { nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.LATEST.toString(), null); } } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get())) { nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.TRIM_HORIZON.toString(), null); } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get())) { nextShardItr = null; } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_AT_TIMESTAMP_SEQUENCE_NUM.get())) { nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AT_TIMESTAMP.toString(), initTimestamp); } else { // we will be starting from an actual sequence number (due to restore from failure). // if the last sequence number refers to an aggregated record, we need to clean up any dangling sub-records // from the last aggregated record; otherwise, we can simply start iterating from the record right after. if (lastSequenceNum.isAggregated()) { String itrForLastAggregatedRecord = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AT_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber()); // get only the last aggregated record GetRecordsResult getRecordsResult = getRecords(itrForLastAggregatedRecord, 1); List<UserRecord> fetchedRecords = deaggregateRecords(getRecordsResult.getRecords(), subscribedShard.getShard().getHashKeyRange().getStartingHashKey(), subscribedShard.getShard().getHashKeyRange().getEndingHashKey()); long lastSubSequenceNum = lastSequenceNum.getSubSequenceNumber(); for (UserRecord record : fetchedRecords) { // we have found a dangling sub-record if it has a larger subsequence number // than our last sequence number; if so, collect the record and update state if (record.getSubSequenceNumber() > lastSubSequenceNum) { deserializeRecordForCollectionAndUpdateState(record); } } // set the nextShardItr so we can continue iterating in the next while loop nextShardItr = getRecordsResult.getNextShardIterator(); } else { // the last record was non-aggregated, so we can simply start from the next record nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber()); } } while (isRunning()) { if (nextShardItr == null) { fetcherRef.updateState(subscribedShardStateIndex, SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get()); // we can close this consumer thread once we've reached the end of the subscribed shard break; } else { if (fetchIntervalMillis != 0) { Thread.sleep(fetchIntervalMillis); } GetRecordsResult getRecordsResult = getRecords(nextShardItr, maxNumberOfRecordsPerFetch); // each of the Kinesis records may be aggregated, so we must deaggregate them before proceeding List<UserRecord> fetchedRecords = deaggregateRecords(getRecordsResult.getRecords(), subscribedShard.getShard().getHashKeyRange().getStartingHashKey(), subscribedShard.getShard().getHashKeyRange().getEndingHashKey()); for (UserRecord record : fetchedRecords) { deserializeRecordForCollectionAndUpdateState(record); } nextShardItr = getRecordsResult.getNextShardIterator(); } } } catch (Throwable t) { fetcherRef.stopWithError(t); } }
From source file:org.apache.flink.streaming.connectors.kinesis.internals.ShardConsumer.java
License:Apache License
/** * Calls {@link KinesisProxyInterface#getRecords(String, int)}, while also handling unexpected * AWS {@link ExpiredIteratorException}s to assure that we get results and don't just fail on * such occasions. The returned shard iterator within the successful {@link GetRecordsResult} should * be used for the next call to this method. * * Note: it is important that this method is not called again before all the records from the last result have been * fully collected with {@link ShardConsumer#deserializeRecordForCollectionAndUpdateState(UserRecord)}, otherwise * {@link ShardConsumer#lastSequenceNum} may refer to a sub-record in the middle of an aggregated record, leading to * incorrect shard iteration if the iterator had to be refreshed. * * @param shardItr shard iterator to use * @param maxNumberOfRecords the maximum number of records to fetch for this getRecords attempt * @return get records result// w w w . j a v a 2 s. co m * @throws InterruptedException */ private GetRecordsResult getRecords(String shardItr, int maxNumberOfRecords) throws InterruptedException { GetRecordsResult getRecordsResult = null; while (getRecordsResult == null) { try { getRecordsResult = kinesis.getRecords(shardItr, maxNumberOfRecords); } catch (ExpiredIteratorException eiEx) { LOG.warn("Encountered an unexpected expired iterator {} for shard {};" + " refreshing the iterator ...", shardItr, subscribedShard); shardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber()); // sleep for the fetch interval before the next getRecords attempt with the refreshed iterator if (fetchIntervalMillis != 0) { Thread.sleep(fetchIntervalMillis); } } } return getRecordsResult; }
From source file:org.apache.flink.streaming.connectors.kinesis.internals.ShardConsumerThread.java
License:Apache License
@SuppressWarnings("unchecked") @Override//from www. ja va 2 s . c o m public void run() { String nextShardItr; try { // before infinitely looping, we set the initial nextShardItr appropriately if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_LATEST_SEQUENCE_NUM.get())) { // if the shard is already closed, there will be no latest next record to get for this shard if (assignedShard.isClosed()) { nextShardItr = null; } else { nextShardItr = kinesisProxy.getShardIterator(assignedShard, ShardIteratorType.LATEST.toString(), null); } } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get())) { nextShardItr = kinesisProxy.getShardIterator(assignedShard, ShardIteratorType.TRIM_HORIZON.toString(), null); } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get())) { nextShardItr = null; } else { // we will be starting from an actual sequence number (due to restore from failure). // if the last sequence number refers to an aggregated record, we need to clean up any dangling sub-records // from the last aggregated record; otherwise, we can simply start iterating from the record right after. if (lastSequenceNum.isAggregated()) { String itrForLastAggregatedRecord = kinesisProxy.getShardIterator(assignedShard, ShardIteratorType.AT_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber()); // get only the last aggregated record GetRecordsResult getRecordsResult = kinesisProxy.getRecords(itrForLastAggregatedRecord, 1); List<UserRecord> fetchedRecords = deaggregateRecords(getRecordsResult.getRecords(), assignedShard.getStartingHashKey(), assignedShard.getEndingHashKey()); long lastSubSequenceNum = lastSequenceNum.getSubSequenceNumber(); for (UserRecord record : fetchedRecords) { // we have found a dangling sub-record if it has a larger subsequence number // than our last sequence number; if so, collect the record and update state if (record.getSubSequenceNumber() > lastSubSequenceNum) { collectRecordAndUpdateState(record); } } // set the nextShardItr so we can continue iterating in the next while loop nextShardItr = getRecordsResult.getNextShardIterator(); } else { // the last record was non-aggregated, so we can simply start from the next record nextShardItr = kinesisProxy.getShardIterator(assignedShard, ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber()); } } while (running) { if (nextShardItr == null) { synchronized (sourceContext.getCheckpointLock()) { seqNoState.put(assignedShard, SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get()); } break; } else { GetRecordsResult getRecordsResult = kinesisProxy.getRecords(nextShardItr, maxNumberOfRecordsPerFetch); // each of the Kinesis records may be aggregated, so we must deaggregate them before proceeding List<UserRecord> fetchedRecords = deaggregateRecords(getRecordsResult.getRecords(), assignedShard.getStartingHashKey(), assignedShard.getEndingHashKey()); for (UserRecord record : fetchedRecords) { collectRecordAndUpdateState(record); } nextShardItr = getRecordsResult.getNextShardIterator(); } } } catch (Throwable t) { ownerRef.stopWithError(t); } }
From source file:org.apache.storm.kinesis.spout.Config.java
License:Apache License
private void validate() { if (streamName == null || streamName.length() < 1) { throw new IllegalArgumentException("streamName is required and cannot be of length 0."); }/*from w w w. ja v a 2s.c om*/ if (shardIteratorType == null || shardIteratorType.equals(ShardIteratorType.AFTER_SEQUENCE_NUMBER) || shardIteratorType.equals(ShardIteratorType.AT_SEQUENCE_NUMBER)) { throw new IllegalArgumentException( "shardIteratorType has to be one of the " + ShardIteratorType.AT_TIMESTAMP + "," + ShardIteratorType.LATEST + "," + ShardIteratorType.TRIM_HORIZON); } if (shardIteratorType.equals(ShardIteratorType.AT_TIMESTAMP) && timestamp == null) { throw new IllegalArgumentException( "timestamp must be provided if shardIteratorType is " + ShardIteratorType.AT_TIMESTAMP); } if (recordToTupleMapper == null) { throw new IllegalArgumentException("recordToTupleMapper cannot be null"); } if (failedMessageRetryHandler == null) { throw new IllegalArgumentException("failedMessageRetryHandler cannot be null"); } if (zkInfo == null) { throw new IllegalArgumentException("zkInfo cannot be null"); } if (kinesisConnectionInfo == null) { throw new IllegalArgumentException("kinesisConnectionInfo cannot be null"); } if (maxUncommittedRecords == null || maxUncommittedRecords < 1) { throw new IllegalArgumentException("maxUncommittedRecords has to be a positive integer"); } }
From source file:org.apache.storm.kinesis.spout.KinesisConnection.java
License:Apache License
String getShardIterator(String stream, String shardId, ShardIteratorType shardIteratorType,
String sequenceNumber, Date timestamp) {
String shardIterator = "";
try {//from ww w. ja v a 2 s.c o m
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
getShardIteratorRequest.setStreamName(stream);
getShardIteratorRequest.setShardId(shardId);
getShardIteratorRequest.setShardIteratorType(shardIteratorType);
if (shardIteratorType.equals(ShardIteratorType.AFTER_SEQUENCE_NUMBER)
|| shardIteratorType.equals(ShardIteratorType.AT_SEQUENCE_NUMBER)) {
getShardIteratorRequest.setStartingSequenceNumber(sequenceNumber);
} else if (shardIteratorType.equals(ShardIteratorType.AT_TIMESTAMP)) {
getShardIteratorRequest.setTimestamp(timestamp);
}
GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
if (getShardIteratorResult != null) {
shardIterator = getShardIteratorResult.getShardIterator();
}
} catch (Exception e) {
LOG.warn(
"Exception occured while getting shardIterator for shard " + shardId + " shardIteratorType "
+ shardIteratorType + " sequence number " + sequenceNumber + " timestamp " + timestamp,
e);
}
LOG.warn("Returning shardIterator " + shardIterator + " for shardId " + shardId + " shardIteratorType "
+ shardIteratorType + " sequenceNumber " + sequenceNumber + " timestamp" + timestamp);
return shardIterator;
}
From source file:org.apache.storm.kinesis.spout.KinesisRecordsManager.java
License:Apache License
private void refreshShardIteratorForNewRecords(String shardId) { String shardIterator = null;/*from ww w. j a v a 2 s . c o m*/ String lastFetchedSequenceNumber = fetchedSequenceNumberPerShard.get(shardId); ShardIteratorType shardIteratorType = (lastFetchedSequenceNumber == null ? kinesisConfig.getShardIteratorType() : ShardIteratorType.AFTER_SEQUENCE_NUMBER); // Set the shard iterator for last fetched sequence number to start from correct position in shard shardIterator = kinesisConnection.getShardIterator(kinesisConfig.getStreamName(), shardId, shardIteratorType, lastFetchedSequenceNumber, kinesisConfig.getTimestamp()); if (shardIterator != null && !shardIterator.isEmpty()) { LOG.warn("Refreshing shard iterator for new records for shardId " + shardId + " with shardIterator " + shardIterator); shardIteratorPerShard.put(shardId, shardIterator); } }