Example usage for com.amazonaws.services.dynamodbv2.model DescribeStreamRequest DescribeStreamRequest

Introduction

In this page you can find the example usage for com.amazonaws.services.dynamodbv2.model DescribeStreamRequest DescribeStreamRequest.

Prototype

DescribeStreamRequest

Source Link

Usage

From source file:dynamok.source.DynamoDbSourceConnector.java

License:Apache License

@Override
public void start(Map<String, String> props) {
    config = new ConnectorConfig(props);
    streamShards = new HashMap<>();

    final AmazonDynamoDBClient client;
    final AmazonDynamoDBStreamsClient streamsClient;

    if (config.accessKeyId.value().isEmpty() || config.secretKeyId.value().isEmpty()) {
        client = new AmazonDynamoDBClient();
        streamsClient = new AmazonDynamoDBStreamsClient();
        log.debug("AmazonDynamoDB clients created with default credentials");
    } else {/*from  www  . j a  v a 2 s. c o  m*/
        BasicAWSCredentials awsCreds = new BasicAWSCredentials(config.accessKeyId.value(),
                config.secretKeyId.value());
        client = new AmazonDynamoDBClient(awsCreds);
        streamsClient = new AmazonDynamoDBStreamsClient(awsCreds);
        log.debug("AmazonDynamoDB clients created with AWS credentials from connector configuration");
    }

    client.configureRegion(config.region);
    streamsClient.configureRegion(config.region);

    final Set<String> ignoredTables = new HashSet<>();
    final Set<String> consumeTables = new HashSet<>();

    String lastEvaluatedTableName = null;
    do {
        final ListTablesResult listResult = client.listTables(lastEvaluatedTableName);

        for (String tableName : listResult.getTableNames()) {
            if (!acceptTable(tableName)) {
                ignoredTables.add(tableName);
                continue;
            }

            final TableDescription tableDesc = client.describeTable(tableName).getTable();

            final StreamSpecification streamSpec = tableDesc.getStreamSpecification();

            if (streamSpec == null || !streamSpec.isStreamEnabled()) {
                throw new ConnectException(
                        String.format("DynamoDB table `%s` does not have streams enabled", tableName));
            }

            final String streamViewType = streamSpec.getStreamViewType();
            if (!streamViewType.equals(StreamViewType.NEW_IMAGE.name())
                    && !streamViewType.equals(StreamViewType.NEW_AND_OLD_IMAGES.name())) {
                throw new ConnectException(String.format("DynamoDB stream view type for table `%s` is %s",
                        tableName, streamViewType));
            }

            final DescribeStreamResult describeStreamResult = streamsClient
                    .describeStream(new DescribeStreamRequest().withStreamArn(tableDesc.getLatestStreamArn()));

            for (Shard shard : describeStreamResult.getStreamDescription().getShards()) {
                streamShards.put(shard, tableDesc);
            }

            consumeTables.add(tableName);
        }

        lastEvaluatedTableName = listResult.getLastEvaluatedTableName();
    } while (lastEvaluatedTableName != null);

    log.info("Tables to ignore: {}", ignoredTables);
    log.info("Tables to ingest: {}", consumeTables);

    client.shutdown();
    streamsClient.shutdown();
}

---

From source file:reactive.dynamo.StreamClient.java

License:Open Source License

public static void main(String args[]) throws InterruptedException {

    streamsClient.setEndpoint("http://localhost:8000");
    //      streamsClient.setEndpoint("streams.dynamodb.eu-west-1.amazonaws.com");

    String myStreamArn = "arn:aws:dynamodb:ddblocal:000000000000:table/TestTableForStreams/stream/2016-11-05T12:23:25.246";

    // Get the shards in the stream

    DescribeStreamResult describeStreamResult = streamsClient
            .describeStream(new DescribeStreamRequest().withStreamArn(myStreamArn));
    String streamArn = describeStreamResult.getStreamDescription().getStreamArn();
    List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
    System.out.println("found shards " + shards.size());
    // Process each shard
    int numChanges = 2;
    for (Shard shard : shards) {
        String shardId = shard.getShardId();
        System.out.println("Processing " + shardId + " from stream " + streamArn + " with parent "
                + shard.getParentShardId());

        // Get an iterator for the current shard

        GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
                .withStreamArn(myStreamArn).withShardId(shardId)
                .withShardIteratorType(ShardIteratorType.TRIM_HORIZON);
        GetShardIteratorResult getShardIteratorResult = streamsClient.getShardIterator(getShardIteratorRequest);
        String nextItr = getShardIteratorResult.getShardIterator();

        while (nextItr != null) {
            //         while (nextItr != null && numChanges > 0) {

            // Use the iterator to read the data records from the shard

            GetRecordsResult getRecordsResult = streamsClient
                    .getRecords(new GetRecordsRequest().withShardIterator(nextItr));
            List<Record> records = getRecordsResult.getRecords();
            System.out.println("Getting records...");
            if (records.isEmpty()) {
                Thread.sleep(2000);
            }//w  w  w.j  a  v a2 s .co m
            for (Record record : records) {
                System.out.println(record);
                numChanges--;
            }
            nextItr = getRecordsResult.getNextShardIterator();
        }

        System.out.println("Demo complete");
    }
}

---