List of usage examples for com.amazonaws.services.dynamodbv2.model GetRecordsRequest GetRecordsRequest
GetRecordsRequest
From source file:dynamok.source.DynamoDbSourceTask.java
License:Apache License
@Override public List<SourceRecord> poll() throws InterruptedException { // TODO rate limiting? if (assignedShards.isEmpty()) { throw new ConnectException("No remaining source shards"); }/*ww w. j a va 2 s. c o m*/ final String shardId = assignedShards.get(currentShardIdx); final GetRecordsRequest req = new GetRecordsRequest(); req.setShardIterator(shardIterator(shardId)); req.setLimit(100); // TODO configurable final GetRecordsResult rsp = streamsClient.getRecords(req); if (rsp.getNextShardIterator() == null) { log.info("Shard ID `{}` for table `{}` has been closed, it will no longer be polled", shardId, config.tableForShard(shardId)); shardIterators.remove(shardId); assignedShards.remove(shardId); } else { log.debug("Retrieved {} records from shard ID `{}`", rsp.getRecords().size(), shardId); shardIterators.put(shardId, rsp.getNextShardIterator()); } currentShardIdx = (currentShardIdx + 1) % assignedShards.size(); final String tableName = config.tableForShard(shardId); final String topic = config.topicFormat.replace("${table}", tableName); final Map<String, String> sourcePartition = sourcePartition(shardId); return rsp.getRecords().stream() .map(dynamoRecord -> toSourceRecord(sourcePartition, topic, dynamoRecord.getDynamodb())) .collect(Collectors.toList()); }
From source file:reactive.dynamo.StreamClient.java
License:Open Source License
public static void main(String args[]) throws InterruptedException { streamsClient.setEndpoint("http://localhost:8000"); // streamsClient.setEndpoint("streams.dynamodb.eu-west-1.amazonaws.com"); String myStreamArn = "arn:aws:dynamodb:ddblocal:000000000000:table/TestTableForStreams/stream/2016-11-05T12:23:25.246"; // Get the shards in the stream DescribeStreamResult describeStreamResult = streamsClient .describeStream(new DescribeStreamRequest().withStreamArn(myStreamArn)); String streamArn = describeStreamResult.getStreamDescription().getStreamArn(); List<Shard> shards = describeStreamResult.getStreamDescription().getShards(); System.out.println("found shards " + shards.size()); // Process each shard int numChanges = 2; for (Shard shard : shards) { String shardId = shard.getShardId(); System.out.println("Processing " + shardId + " from stream " + streamArn + " with parent " + shard.getParentShardId()); // Get an iterator for the current shard GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest() .withStreamArn(myStreamArn).withShardId(shardId) .withShardIteratorType(ShardIteratorType.TRIM_HORIZON); GetShardIteratorResult getShardIteratorResult = streamsClient.getShardIterator(getShardIteratorRequest); String nextItr = getShardIteratorResult.getShardIterator(); while (nextItr != null) { // while (nextItr != null && numChanges > 0) { // Use the iterator to read the data records from the shard GetRecordsResult getRecordsResult = streamsClient .getRecords(new GetRecordsRequest().withShardIterator(nextItr)); List<Record> records = getRecordsResult.getRecords(); System.out.println("Getting records..."); if (records.isEmpty()) { Thread.sleep(2000); }//from w ww . j a v a2s .c o m for (Record record : records) { System.out.println(record); numChanges--; } nextItr = getRecordsResult.getNextShardIterator(); } System.out.println("Demo complete"); } }