Example usage for com.amazonaws.services.kinesis.model StreamDescription getHasMoreShards

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model StreamDescription getHasMoreShards.

Prototype

public Boolean getHasMoreShards() 

Source Link

Document

If set to true, more shards in the stream are available to describe.

Usage

From source file:com.haskins.cloudtrailviewer.dialog.resourcedetail.detailpanels.KinesisStreamDetail.java

License:Open Source License

private void buildUI(DescribeStreamResult detail) {

    this.add(primaryScrollPane, BorderLayout.CENTER);

    if (detail.getStreamDescription() != null) {

        StreamDescription stream = detail.getStreamDescription();

        if (stream.getHasMoreShards() != null) {
            primaryTableModel.addRow(new Object[] { "Has More Shards", stream.getHasMoreShards() });
        }/*ww w  .  ja  v a  2 s. c om*/
        if (stream.getStreamARN() != null) {
            primaryTableModel.addRow(new Object[] { "Stream Arn", stream.getStreamARN() });
        }
        if (stream.getStreamName() != null) {
            primaryTableModel.addRow(new Object[] { "Stream Name", stream.getStreamName() });
        }
        if (stream.getStreamStatus() != null) {
            primaryTableModel.addRow(new Object[] { "Stram Status", stream.getStreamStatus() });
        }
    }

}

---

From source file:com.streamsets.pipeline.stage.lib.kinesis.KinesisUtil.java

License:Apache License

public static long getShardCount(Regions region, AWSConfig awsConfig, String streamName)
        throws AmazonClientException {
    ClientConfiguration kinesisConfiguration = new ClientConfiguration();
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(AWSUtil.getCredentialsProvider(awsConfig),
            kinesisConfiguration);//w  w w .j a  v a 2s. c  o m
    kinesisClient.setRegion(Region.getRegion(region));

    try {
        long numShards = 0;
        String lastShardId = null;
        StreamDescription description;
        do {
            if (lastShardId == null) {
                description = kinesisClient.describeStream(streamName).getStreamDescription();
            } else {
                description = kinesisClient.describeStream(streamName, lastShardId).getStreamDescription();
            }

            for (Shard shard : description.getShards()) {
                if (shard.getSequenceNumberRange().getEndingSequenceNumber() == null) {
                    // Then this shard is open, so we should count it. Shards with an ending sequence number
                    // are closed and cannot be written to, so we skip counting them.
                    ++numShards;
                }
            }

            int pageSize = description.getShards().size();
            lastShardId = description.getShards().get(pageSize - 1).getShardId();

        } while (description.getHasMoreShards());

        LOG.debug("Connected successfully to stream: '{}' with '{}' shards.", streamName, numShards);

        return numShards;
    } finally {
        kinesisClient.shutdown();
    }
}

---

From source file:org.apache.beam.sdk.io.kinesis.client.SimplifiedKinesisClient.java

License:Apache License

public List<Shard> listShards(final String streamName) throws TransientKinesisException {
    return wrapExceptions(new Callable<List<Shard>>() {
        @Override/*from w w  w  .j  ava2  s  .c o  m*/
        public List<Shard> call() throws Exception {
            List<Shard> shards = Lists.newArrayList();
            String lastShardId = null;

            StreamDescription description;
            do {
                description = kinesis.describeStream(streamName, lastShardId).getStreamDescription();

                shards.addAll(description.getShards());
                lastShardId = shards.get(shards.size() - 1).getShardId();
            } while (description.getHasMoreShards());

            return shards;
        }
    });
}

---

From source file:org.apache.beam.sdk.io.kinesis.SimplifiedKinesisClient.java

License:Apache License

public List<Shard> listShards(final String streamName) throws TransientKinesisException {
    return wrapExceptions(() -> {
        List<Shard> shards = Lists.newArrayList();
        String lastShardId = null;

        StreamDescription description;
        do {// ww w .  j a  va  2 s .  co m
            description = kinesis.describeStream(streamName, lastShardId).getStreamDescription();

            shards.addAll(description.getShards());
            lastShardId = shards.get(shards.size() - 1).getShardId();
        } while (description.getHasMoreShards());

        return shards;
    });
}

---

From source file:org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner.java

License:Apache License

private List<Shard> createOrUpdate(String stream, int shards) {
    List<Shard> shardList = new ArrayList<>();

    int describeStreamRetries = 0;

    String exclusiveStartShardId = null;

    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(stream);

    while (true) {
        DescribeStreamResult describeStreamResult = null;

        try {/*  w  ww.  ja v a 2s  .com*/
            describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
            describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest);
            StreamDescription streamDescription = describeStreamResult.getStreamDescription();
            if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) {
                shardList.addAll(streamDescription.getShards());

                if (streamDescription.getHasMoreShards()) {
                    exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId();
                } else {
                    break;
                }
            }
        } catch (ResourceNotFoundException ex) {
            if (!this.configurationProperties.isAutoCreateStream()) {
                throw new ProvisioningException(
                        "The stream [" + stream + "] was not found and auto creation is disabled.", ex);
            }
            if (logger.isInfoEnabled()) {
                logger.info("Stream '" + stream + "' not found. Create one...");
            }

            this.amazonKinesis.createStream(stream,
                    Math.max(this.configurationProperties.getMinShardCount(), shards));
            continue;
        } catch (LimitExceededException ex) {
            logger.info(
                    "Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for ["
                            + this.configurationProperties.getDescribeStreamBackoff() + "] millis.");
        }

        if (describeStreamResult == null || !StreamStatus.ACTIVE.toString()
                .equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
            if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) {
                ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
                        "The stream [" + stream + "] isn't ACTIVE or doesn't exist.");
                resourceNotFoundException.setServiceName("Kinesis");
                throw new ProvisioningException(
                        "Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
                        resourceNotFoundException);
            }
            try {
                Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
            } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
                throw new ProvisioningException(
                        "The [describeStream] thread for the stream [" + stream + "] has been interrupted.",
                        ex);
            }
        }
    }

    int effectiveShardCount = Math.max(this.configurationProperties.getMinShardCount(), shards);

    if ((shardList.size() < effectiveShardCount) && this.configurationProperties.isAutoAddShards()) {
        return updateShardCount(stream, shardList.size(), effectiveShardCount);
    }

    return shardList;
}

---

From source file:org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner.java

License:Apache License

private List<Shard> updateShardCount(String streamName, int shardCount, int targetCount) {
    if (logger.isInfoEnabled()) {
        logger.info("Stream [" + streamName + "] has [" + shardCount
                + "] shards compared to a target configuration of [" + targetCount + "], creating shards...");
    }/* w  ww.  j  a  va 2  s.  c om*/

    UpdateShardCountRequest updateShardCountRequest = new UpdateShardCountRequest().withStreamName(streamName)
            .withTargetShardCount(targetCount).withScalingType(ScalingType.UNIFORM_SCALING);

    this.amazonKinesis.updateShardCount(updateShardCountRequest);

    // Wait for stream to become active again after resharding
    List<Shard> shardList = new ArrayList<>();

    int describeStreamRetries = 0;

    String exclusiveStartShardId = null;

    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(streamName);

    while (true) {
        DescribeStreamResult describeStreamResult = null;

        try {
            describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
            describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest);
            StreamDescription streamDescription = describeStreamResult.getStreamDescription();
            if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) {
                shardList.addAll(streamDescription.getShards());

                if (streamDescription.getHasMoreShards()) {
                    exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId();
                } else {
                    break;
                }
            }
        } catch (LimitExceededException ex) {
            logger.info("Got LimitExceededException when describing stream [" + streamName + "]. "
                    + "Backing off for [" + this.configurationProperties.getDescribeStreamBackoff()
                    + "] millis.");
        }

        if (describeStreamResult == null || !StreamStatus.ACTIVE.toString()
                .equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
            if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) {
                ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
                        "The stream [" + streamName + "] isn't ACTIVE or doesn't exist.");
                resourceNotFoundException.setServiceName("Kinesis");
                throw new ProvisioningException(
                        "Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
                        resourceNotFoundException);
            }
            try {
                Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
            } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
                throw new ProvisioningException(
                        "The [describeStream] thread for the stream [" + streamName + "] has been interrupted.",
                        ex);
            }
        }
    }
    return shardList;
}

---