Example usage for com.amazonaws.services.dynamodbv2.model GetRecordsRequest setLimit

Introduction

In this page you can find the example usage for com.amazonaws.services.dynamodbv2.model GetRecordsRequest setLimit.

Prototype

public void setLimit(Integer limit) 

Source Link

Document

The maximum number of records to return from the shard.

Usage

From source file:dynamok.source.DynamoDbSourceTask.java

License:Apache License

@Override
public List<SourceRecord> poll() throws InterruptedException {
    // TODO rate limiting?

    if (assignedShards.isEmpty()) {
        throw new ConnectException("No remaining source shards");
    }/*from w ww  .  j  ava 2  s .c o  m*/

    final String shardId = assignedShards.get(currentShardIdx);

    final GetRecordsRequest req = new GetRecordsRequest();
    req.setShardIterator(shardIterator(shardId));
    req.setLimit(100); // TODO configurable

    final GetRecordsResult rsp = streamsClient.getRecords(req);
    if (rsp.getNextShardIterator() == null) {
        log.info("Shard ID `{}` for table `{}` has been closed, it will no longer be polled", shardId,
                config.tableForShard(shardId));
        shardIterators.remove(shardId);
        assignedShards.remove(shardId);
    } else {
        log.debug("Retrieved {} records from shard ID `{}`", rsp.getRecords().size(), shardId);
        shardIterators.put(shardId, rsp.getNextShardIterator());
    }

    currentShardIdx = (currentShardIdx + 1) % assignedShards.size();

    final String tableName = config.tableForShard(shardId);
    final String topic = config.topicFormat.replace("${table}", tableName);
    final Map<String, String> sourcePartition = sourcePartition(shardId);

    return rsp.getRecords().stream()
            .map(dynamoRecord -> toSourceRecord(sourcePartition, topic, dynamoRecord.getDynamodb()))
            .collect(Collectors.toList());
}

---