Example usage for com.amazonaws.services.kinesis.model ListStreamsResult getHasMoreStreams

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model ListStreamsResult getHasMoreStreams.

Prototype

public Boolean getHasMoreStreams() 

Source Link

Document

If set to true, there are more streams available to list.

Usage

From source file:dbtucker.connect.kinesis.KinesisSourceConnector.java

License:Apache License

@Override
public void start(Map<String, String> map) {
    config = new KinesisSourceConnectorConfig(map);
    streamShards = new HashMap<>();

    List<String> streamNames;
    final Set<String> ignoredStreams = new HashSet<>();
    final Set<String> consumedStreams = new HashSet<>();

    final AmazonKinesisClient client = new AmazonKinesisClient();
    client.configureRegion(config.getRegionId());

    ListStreamsResult listResult;
    ListStreamsRequest lsr = new ListStreamsRequest();
    lsr.setLimit(32);/*w  w w.  j  a va  2 s .co  m*/

    String lastEvaluatedStreamName = null;
    do {
        lsr.setExclusiveStartStreamName(lastEvaluatedStreamName);
        listResult = client.listStreams(lsr);

        streamNames = listResult.getStreamNames();
        for (String streamName : streamNames) {
            if (config.getStreamsPrefix() == null) {
                if ((config.getStreamsBlacklist() == null || config.getStreamsBlacklist().contains(streamName))
                        && (config.getStreamsWhitelist() == null
                                || !config.getStreamsWhitelist().contains(streamName))) {
                    ignoredStreams.add(streamName);
                    continue;
                }
            } else {
                if (streamName.startsWith(config.getStreamsPrefix())) {
                    if (config.getStreamsBlacklist() != null
                            && config.getStreamsBlacklist().contains(streamName)) {
                        ignoredStreams.add(streamName);
                        continue;
                    }
                } else {
                    ignoredStreams.add(streamName);
                    continue;
                }
            }

            final DescribeStreamResult streamDesc = client.describeStream(streamName);

            if (streamDesc.getStreamDescription().getStreamStatus().equals(StreamStatus.DELETING.toString())) {
                log.warn("Stream '{}' is being deleted and cannot be consumed", streamName);
                ignoredStreams.add(streamName);
                continue;
            }

            for (Shard shard : streamDesc.getStreamDescription().getShards()) {
                streamShards.put(shard, streamDesc);
            }

            consumedStreams.add(streamName);
        }

        if (streamNames.size() > 0) {
            lastEvaluatedStreamName = streamNames.get(streamNames.size() - 1);
        }

    } while (listResult.getHasMoreStreams());

    log.info("Streams to ingest: {}", consumedStreams);
    log.info("Streams to ignore: {}", ignoredStreams);

    client.shutdown();

    if (consumedStreams.isEmpty()) {
        throw new ConnectException("No matching Kinesis Streams found.  Exiting connector");
    }
}

---

From source file:org.springframework.cloud.stream.binder.kinesis.KinesisTestBinder.java

License:Apache License

@Override
public void cleanup() {
    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    ListStreamsResult listStreamsResult = this.amazonKinesis.listStreams(listStreamsRequest);

    List<String> streamNames = listStreamsResult.getStreamNames();

    while (listStreamsResult.getHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }/*w  w  w . ja v  a2s .  c  o  m*/
        listStreamsResult = this.amazonKinesis.listStreams(listStreamsRequest);
        streamNames.addAll(listStreamsResult.getStreamNames());
    }

    for (String stream : streamNames) {
        this.amazonKinesis.deleteStream(stream);
        while (true) {
            try {
                this.amazonKinesis.describeStream(stream);
                try {
                    Thread.sleep(100);
                } catch (InterruptedException ex) {
                    Thread.currentThread().interrupt();
                    throw new IllegalStateException(ex);
                }
            } catch (ResourceNotFoundException ex) {
                break;
            }
        }
    }
}

---

From source file:org.springframework.cloud.stream.binder.kinesis.LocalKinesisResource.java

License:Apache License

@Override
protected void cleanupResource() {
    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    ListStreamsResult listStreamsResult = this.resource.listStreams(listStreamsRequest);

    List<String> streamNames = listStreamsResult.getStreamNames();

    while (listStreamsResult.getHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }/*from  w  ww  .j  a  v  a2s .c  o m*/
        listStreamsResult = this.resource.listStreams(listStreamsRequest);
        streamNames.addAll(listStreamsResult.getStreamNames());
    }

    for (String stream : streamNames) {
        this.resource.deleteStream(stream);
        while (true) {
            try {
                this.resource.describeStream(stream);
                try {
                    Thread.sleep(100);
                } catch (InterruptedException ex) {
                    Thread.currentThread().interrupt();
                    throw new IllegalStateException(ex);
                }
            } catch (ResourceNotFoundException ex) {
                break;
            }
        }
    }

    System.clearProperty(SDKGlobalConfiguration.AWS_CBOR_DISABLE_SYSTEM_PROPERTY);
    this.resource.shutdown();
}

---