Example usage for com.amazonaws.services.dynamodbv2.model DescribeStreamResult getStreamDescription

Introduction

In this page you can find the example usage for com.amazonaws.services.dynamodbv2.model DescribeStreamResult getStreamDescription.

Prototype

public StreamDescription getStreamDescription() 

Source Link

Document

A complete description of the stream, including its creation date and time, the DynamoDB table associated with the stream, the shard IDs within the stream, and the beginning and ending sequence numbers of stream records within the shards.

Usage

From source file:dynamok.source.DynamoDbSourceConnector.java

License:Apache License

@Override
public void start(Map<String, String> props) {
    config = new ConnectorConfig(props);
    streamShards = new HashMap<>();

    final AmazonDynamoDBClient client;
    final AmazonDynamoDBStreamsClient streamsClient;

    if (config.accessKeyId.value().isEmpty() || config.secretKeyId.value().isEmpty()) {
        client = new AmazonDynamoDBClient();
        streamsClient = new AmazonDynamoDBStreamsClient();
        log.debug("AmazonDynamoDB clients created with default credentials");
    } else {/*from  www. j a va 2 s .c o  m*/
        BasicAWSCredentials awsCreds = new BasicAWSCredentials(config.accessKeyId.value(),
                config.secretKeyId.value());
        client = new AmazonDynamoDBClient(awsCreds);
        streamsClient = new AmazonDynamoDBStreamsClient(awsCreds);
        log.debug("AmazonDynamoDB clients created with AWS credentials from connector configuration");
    }

    client.configureRegion(config.region);
    streamsClient.configureRegion(config.region);

    final Set<String> ignoredTables = new HashSet<>();
    final Set<String> consumeTables = new HashSet<>();

    String lastEvaluatedTableName = null;
    do {
        final ListTablesResult listResult = client.listTables(lastEvaluatedTableName);

        for (String tableName : listResult.getTableNames()) {
            if (!acceptTable(tableName)) {
                ignoredTables.add(tableName);
                continue;
            }

            final TableDescription tableDesc = client.describeTable(tableName).getTable();

            final StreamSpecification streamSpec = tableDesc.getStreamSpecification();

            if (streamSpec == null || !streamSpec.isStreamEnabled()) {
                throw new ConnectException(
                        String.format("DynamoDB table `%s` does not have streams enabled", tableName));
            }

            final String streamViewType = streamSpec.getStreamViewType();
            if (!streamViewType.equals(StreamViewType.NEW_IMAGE.name())
                    && !streamViewType.equals(StreamViewType.NEW_AND_OLD_IMAGES.name())) {
                throw new ConnectException(String.format("DynamoDB stream view type for table `%s` is %s",
                        tableName, streamViewType));
            }

            final DescribeStreamResult describeStreamResult = streamsClient
                    .describeStream(new DescribeStreamRequest().withStreamArn(tableDesc.getLatestStreamArn()));

            for (Shard shard : describeStreamResult.getStreamDescription().getShards()) {
                streamShards.put(shard, tableDesc);
            }

            consumeTables.add(tableName);
        }

        lastEvaluatedTableName = listResult.getLastEvaluatedTableName();
    } while (lastEvaluatedTableName != null);

    log.info("Tables to ignore: {}", ignoredTables);
    log.info("Tables to ingest: {}", consumeTables);

    client.shutdown();
    streamsClient.shutdown();
}

---

From source file:org.lendingclub.mercator.aws.KinesisScanner.java

License:Apache License

private void scanStream(String name) {
    com.amazonaws.services.kinesis.model.DescribeStreamResult result = getClient().describeStream(name);
    StreamDescription description = result.getStreamDescription();

    project(description);//from ww  w.  j  av  a 2 s.  c  om

}

---

From source file:reactive.dynamo.StreamClient.java

License:Open Source License

public static void main(String args[]) throws InterruptedException {

    streamsClient.setEndpoint("http://localhost:8000");
    //      streamsClient.setEndpoint("streams.dynamodb.eu-west-1.amazonaws.com");

    String myStreamArn = "arn:aws:dynamodb:ddblocal:000000000000:table/TestTableForStreams/stream/2016-11-05T12:23:25.246";

    // Get the shards in the stream

    DescribeStreamResult describeStreamResult = streamsClient
            .describeStream(new DescribeStreamRequest().withStreamArn(myStreamArn));
    String streamArn = describeStreamResult.getStreamDescription().getStreamArn();
    List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
    System.out.println("found shards " + shards.size());
    // Process each shard
    int numChanges = 2;
    for (Shard shard : shards) {
        String shardId = shard.getShardId();
        System.out.println("Processing " + shardId + " from stream " + streamArn + " with parent "
                + shard.getParentShardId());

        // Get an iterator for the current shard

        GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
                .withStreamArn(myStreamArn).withShardId(shardId)
                .withShardIteratorType(ShardIteratorType.TRIM_HORIZON);
        GetShardIteratorResult getShardIteratorResult = streamsClient.getShardIterator(getShardIteratorRequest);
        String nextItr = getShardIteratorResult.getShardIterator();

        while (nextItr != null) {
            //         while (nextItr != null && numChanges > 0) {

            // Use the iterator to read the data records from the shard

            GetRecordsResult getRecordsResult = streamsClient
                    .getRecords(new GetRecordsRequest().withShardIterator(nextItr));
            List<Record> records = getRecordsResult.getRecords();
            System.out.println("Getting records...");
            if (records.isEmpty()) {
                Thread.sleep(2000);
            }/*from  ww  w . j a  v a2 s  . c o  m*/
            for (Record record : records) {
                System.out.println(record);
                numChanges--;
            }
            nextItr = getRecordsResult.getNextShardIterator();
        }

        System.out.println("Demo complete");
    }
}

---