Example usage for com.amazonaws.services.kinesis.model StreamStatus DELETING

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model StreamStatus DELETING.

Prototype

StreamStatus DELETING

To view the source code for com.amazonaws.services.kinesis.model StreamStatus DELETING.

Click Source Link

Usage

From source file:dbtucker.connect.kinesis.KinesisSourceConnector.java

License:Apache License

@Override
public void start(Map<String, String> map) {
    config = new KinesisSourceConnectorConfig(map);
    streamShards = new HashMap<>();

    List<String> streamNames;
    final Set<String> ignoredStreams = new HashSet<>();
    final Set<String> consumedStreams = new HashSet<>();

    final AmazonKinesisClient client = new AmazonKinesisClient();
    client.configureRegion(config.getRegionId());

    ListStreamsResult listResult;/*from ww w  .  j  ava 2  s . c o  m*/
    ListStreamsRequest lsr = new ListStreamsRequest();
    lsr.setLimit(32);

    String lastEvaluatedStreamName = null;
    do {
        lsr.setExclusiveStartStreamName(lastEvaluatedStreamName);
        listResult = client.listStreams(lsr);

        streamNames = listResult.getStreamNames();
        for (String streamName : streamNames) {
            if (config.getStreamsPrefix() == null) {
                if ((config.getStreamsBlacklist() == null || config.getStreamsBlacklist().contains(streamName))
                        && (config.getStreamsWhitelist() == null
                                || !config.getStreamsWhitelist().contains(streamName))) {
                    ignoredStreams.add(streamName);
                    continue;
                }
            } else {
                if (streamName.startsWith(config.getStreamsPrefix())) {
                    if (config.getStreamsBlacklist() != null
                            && config.getStreamsBlacklist().contains(streamName)) {
                        ignoredStreams.add(streamName);
                        continue;
                    }
                } else {
                    ignoredStreams.add(streamName);
                    continue;
                }
            }

            final DescribeStreamResult streamDesc = client.describeStream(streamName);

            if (streamDesc.getStreamDescription().getStreamStatus().equals(StreamStatus.DELETING.toString())) {
                log.warn("Stream '{}' is being deleted and cannot be consumed", streamName);
                ignoredStreams.add(streamName);
                continue;
            }

            for (Shard shard : streamDesc.getStreamDescription().getShards()) {
                streamShards.put(shard, streamDesc);
            }

            consumedStreams.add(streamName);
        }

        if (streamNames.size() > 0) {
            lastEvaluatedStreamName = streamNames.get(streamNames.size() - 1);
        }

    } while (listResult.getHasMoreStreams());

    log.info("Streams to ingest: {}", consumedStreams);
    log.info("Streams to ignore: {}", ignoredStreams);

    client.shutdown();

    if (consumedStreams.isEmpty()) {
        throw new ConnectException("No matching Kinesis Streams found.  Exiting connector");
    }
}

---