Example usage for com.amazonaws.services.dynamodbv2.model GetRecordsResult getRecords

Introduction

In this page you can find the example usage for com.amazonaws.services.dynamodbv2.model GetRecordsResult getRecords.

Prototype

public java.util.List<Record> getRecords() 

Source Link

Document

The stream records from the shard, which were retrieved using the shard iterator.

Usage

From source file:dynamok.source.DynamoDbSourceTask.java

License:Apache License

@Override
public List<SourceRecord> poll() throws InterruptedException {
    // TODO rate limiting?

    if (assignedShards.isEmpty()) {
        throw new ConnectException("No remaining source shards");
    }//from  w w  w  .  j a  va 2s  .  co  m

    final String shardId = assignedShards.get(currentShardIdx);

    final GetRecordsRequest req = new GetRecordsRequest();
    req.setShardIterator(shardIterator(shardId));
    req.setLimit(100); // TODO configurable

    final GetRecordsResult rsp = streamsClient.getRecords(req);
    if (rsp.getNextShardIterator() == null) {
        log.info("Shard ID `{}` for table `{}` has been closed, it will no longer be polled", shardId,
                config.tableForShard(shardId));
        shardIterators.remove(shardId);
        assignedShards.remove(shardId);
    } else {
        log.debug("Retrieved {} records from shard ID `{}`", rsp.getRecords().size(), shardId);
        shardIterators.put(shardId, rsp.getNextShardIterator());
    }

    currentShardIdx = (currentShardIdx + 1) % assignedShards.size();

    final String tableName = config.tableForShard(shardId);
    final String topic = config.topicFormat.replace("${table}", tableName);
    final Map<String, String> sourcePartition = sourcePartition(shardId);

    return rsp.getRecords().stream()
            .map(dynamoRecord -> toSourceRecord(sourcePartition, topic, dynamoRecord.getDynamodb()))
            .collect(Collectors.toList());
}

---

From source file:reactive.dynamo.StreamClient.java

License:Open Source License

public static void main(String args[]) throws InterruptedException {

    streamsClient.setEndpoint("http://localhost:8000");
    //      streamsClient.setEndpoint("streams.dynamodb.eu-west-1.amazonaws.com");

    String myStreamArn = "arn:aws:dynamodb:ddblocal:000000000000:table/TestTableForStreams/stream/2016-11-05T12:23:25.246";

    // Get the shards in the stream

    DescribeStreamResult describeStreamResult = streamsClient
            .describeStream(new DescribeStreamRequest().withStreamArn(myStreamArn));
    String streamArn = describeStreamResult.getStreamDescription().getStreamArn();
    List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
    System.out.println("found shards " + shards.size());
    // Process each shard
    int numChanges = 2;
    for (Shard shard : shards) {
        String shardId = shard.getShardId();
        System.out.println("Processing " + shardId + " from stream " + streamArn + " with parent "
                + shard.getParentShardId());

        // Get an iterator for the current shard

        GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
                .withStreamArn(myStreamArn).withShardId(shardId)
                .withShardIteratorType(ShardIteratorType.TRIM_HORIZON);
        GetShardIteratorResult getShardIteratorResult = streamsClient.getShardIterator(getShardIteratorRequest);
        String nextItr = getShardIteratorResult.getShardIterator();

        while (nextItr != null) {
            //         while (nextItr != null && numChanges > 0) {

            // Use the iterator to read the data records from the shard

            GetRecordsResult getRecordsResult = streamsClient
                    .getRecords(new GetRecordsRequest().withShardIterator(nextItr));
            List<Record> records = getRecordsResult.getRecords();
            System.out.println("Getting records...");
            if (records.isEmpty()) {
                Thread.sleep(2000);
            }/*from   ww  w  .  j a v a2 s  .  c o  m*/
            for (Record record : records) {
                System.out.println(record);
                numChanges--;
            }
            nextItr = getRecordsResult.getNextShardIterator();
        }

        System.out.println("Demo complete");
    }
}

---