Example usage for com.amazonaws.services.kinesis.model StreamDescription getShards

List of usage examples for com.amazonaws.services.kinesis.model StreamDescription getShards

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.kinesis.model.*
  5. StreamDescription
  6. getShards

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model StreamDescription getShards.

Prototype


public java.util.List<Shard> getShards()

Source Link

Document

The shards that comprise the stream.

Usage

From source file:StreamUtils.java

License:Open Source License

/**
 * Split a shard by dividing the hash key space in half.
 *
 * @param streamName Name of the stream that contains the shard to split.
 * @param shardId The id of the shard to split.
 *
 * @throws IllegalArgumentException When either streamName or shardId are null or empty.
 * @throws LimitExceededException Shard limit for the account has been reached.
 * @throws ResourceNotFoundException The stream or shard cannot be found.
 * @throws InvalidArgumentException If the shard is closed and no eligible for splitting.
 * @throws AmazonClientException Error communicating with Amazon Kinesis.
 *
 *//* w w w.j  a  va  2  s  .c  om*/
public void splitShardEvenly(String streamName, String shardId) throws LimitExceededException,
        ResourceNotFoundException, AmazonClientException, InvalidArgumentException, IllegalArgumentException {
    if (streamName == null || streamName.isEmpty()) {
        throw new IllegalArgumentException("stream name is required");
    }
    if (shardId == null || shardId.isEmpty()) {
        throw new IllegalArgumentException("shard id is required");
    }

    DescribeStreamResult result = kinesis.describeStream(streamName);
    StreamDescription description = result.getStreamDescription();

    // Find the shard we want to split
    Shard shardToSplit = null;
    for (Shard shard : description.getShards()) {
        if (shardId.equals(shard.getShardId())) {
            shardToSplit = shard;
            break;
        }
    }

    if (shardToSplit == null) {
        throw new ResourceNotFoundException(
                "Could not find shard with id '" + shardId + "' in stream '" + streamName + "'");
    }

    // Check if the shard is still open. Open shards do not have an ending sequence number.
    if (shardToSplit.getSequenceNumberRange().getEndingSequenceNumber() != null) {
        throw new InvalidArgumentException("Shard is CLOSED and is not eligible for splitting");
    }

    // Calculate the median hash key to use as the new starting hash key for the shard.
    BigInteger startingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getStartingHashKey());
    BigInteger endingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getEndingHashKey());
    BigInteger[] medianHashKey = startingHashKey.add(endingHashKey).divideAndRemainder(new BigInteger("2"));
    BigInteger newStartingHashKey = medianHashKey[0];
    if (!BigInteger.ZERO.equals(medianHashKey[1])) {
        // In order to more evenly distributed the new hash key ranges across the new shards we will "round up" to
        // the next integer when our current hash key range is not evenly divisible by 2.
        newStartingHashKey = newStartingHashKey.add(BigInteger.ONE);
    }

    // Submit the split shard request
    kinesis.splitShard(streamName, shardId, newStartingHashKey.toString());
}

From source file:com.kinesis.datavis.utils.StreamUtils.java

License:Open Source License

/**
 * Split a shard by dividing the hash key space in half.
 *
 * @param streamName Name of the stream that contains the shard to split.
 * @param shardId The id of the shard to split.
 *
 * @throws IllegalArgumentException When either streamName or shardId are null or empty.
 * @throws LimitExceededException Shard limit for the account has been reached.
 * @throws ResourceNotFoundException The stream or shard cannot be found.
 * @throws InvalidArgumentException If the shard is closed and no eligible for splitting.
 * @throws AmazonClientException Error communicating with Amazon Kinesis.
 *
 *//* w  ww . ja  va2s .  com*/
public void splitShardEvenly(String streamName, String shardId) throws LimitExceededException,
        ResourceNotFoundException, AmazonClientException, InvalidArgumentException, IllegalArgumentException {
    if (streamName == null || streamName.isEmpty()) {
        throw new IllegalArgumentException("stream name is required");
    }
    if (shardId == null || shardId.isEmpty()) {
        throw new IllegalArgumentException("shard id is required");
    }

    DescribeStreamResult result = kinesis.describeStream(streamName);
    StreamDescription description = result.getStreamDescription();

    // Find the shard we want to split
    Shard shardToSplit = null;
    for (Shard shard : description.getShards()) {
        if (shardId.equals(shard.getShardId())) {
            shardToSplit = shard;
            break;
        }
    }

    if (shardToSplit == null) {
        throw new ResourceNotFoundException(
                "Could not find shard with id '" + shardId + "' in stream '" + streamName + "'");
    }

    // Check if the shard is still open. Open shards do not have an ending sequence number.
    if (shardToSplit.getSequenceNumberRange().getEndingSequenceNumber() != null) {
        throw new InvalidArgumentException("Shard is CLOSED and is not eligible for splitting");
    }

    // Calculate the median hash key to use as the new starting hash key for the shard.
    BigInteger startingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getStartingHashKey());
    BigInteger endingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getEndingHashKey());
    BigInteger[] medianHashKey = startingHashKey.add(endingHashKey).divideAndRemainder(new BigInteger("2"));
    BigInteger newStartingHashKey = medianHashKey[0];

    if (!BigInteger.ZERO.equals(medianHashKey[1])) {
        // In order to more evenly distributed the new hash key ranges across the new shards we will "round up" to
        // the next integer when our current hash key range is not evenly divisible by 2.
        newStartingHashKey = newStartingHashKey.add(BigInteger.ONE);
    }

    // Submit the split shard request
    kinesis.splitShard(streamName, shardId, newStartingHashKey.toString());
}

From source file:com.streamsets.pipeline.stage.lib.kinesis.KinesisUtil.java

License:Apache License

public static long getShardCount(Regions region, AWSConfig awsConfig, String streamName)
        throws AmazonClientException {
    ClientConfiguration kinesisConfiguration = new ClientConfiguration();
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(AWSUtil.getCredentialsProvider(awsConfig),
            kinesisConfiguration);/*ww w . ja  v  a2s . co m*/
    kinesisClient.setRegion(Region.getRegion(region));

    try {
        long numShards = 0;
        String lastShardId = null;
        StreamDescription description;
        do {
            if (lastShardId == null) {
                description = kinesisClient.describeStream(streamName).getStreamDescription();
            } else {
                description = kinesisClient.describeStream(streamName, lastShardId).getStreamDescription();
            }

            for (Shard shard : description.getShards()) {
                if (shard.getSequenceNumberRange().getEndingSequenceNumber() == null) {
                    // Then this shard is open, so we should count it. Shards with an ending sequence number
                    // are closed and cannot be written to, so we skip counting them.
                    ++numShards;
                }
            }

            int pageSize = description.getShards().size();
            lastShardId = description.getShards().get(pageSize - 1).getShardId();

        } while (description.getHasMoreShards());

        LOG.debug("Connected successfully to stream: '{}' with '{}' shards.", streamName, numShards);

        return numShards;
    } finally {
        kinesisClient.shutdown();
    }
}

From source file:org.apache.beam.sdk.io.kinesis.client.SimplifiedKinesisClient.java

License:Apache License

public List<Shard> listShards(final String streamName) throws TransientKinesisException {
    return wrapExceptions(new Callable<List<Shard>>() {
        @Override//from w w  w  .  ja  v  a  2  s.c o  m
        public List<Shard> call() throws Exception {
            List<Shard> shards = Lists.newArrayList();
            String lastShardId = null;

            StreamDescription description;
            do {
                description = kinesis.describeStream(streamName, lastShardId).getStreamDescription();

                shards.addAll(description.getShards());
                lastShardId = shards.get(shards.size() - 1).getShardId();
            } while (description.getHasMoreShards());

            return shards;
        }
    });
}

From source file:org.apache.beam.sdk.io.kinesis.SimplifiedKinesisClient.java

License:Apache License

public List<Shard> listShards(final String streamName) throws TransientKinesisException {
    return wrapExceptions(() -> {
        List<Shard> shards = Lists.newArrayList();
        String lastShardId = null;

        StreamDescription description;
        do {//  w ww  . j  a v  a  2 s  .co  m
            description = kinesis.describeStream(streamName, lastShardId).getStreamDescription();

            shards.addAll(description.getShards());
            lastShardId = shards.get(shards.size() - 1).getShardId();
        } while (description.getHasMoreShards());

        return shards;
    });
}

From source file:org.apache.druid.indexing.kinesis.KinesisRecordSupplier.java

License:Apache License

@Override
public Set<String> getPartitionIds(String stream) {
    return wrapExceptions(() -> {
        final Set<String> retVal = new HashSet<>();
        DescribeStreamRequest request = new DescribeStreamRequest();
        request.setStreamName(stream);/*from   w w  w  . ja v a 2 s. c o  m*/

        while (request != null) {
            final DescribeStreamResult result = kinesis.describeStream(request);
            final StreamDescription streamDescription = result.getStreamDescription();
            final List<Shard> shards = streamDescription.getShards();

            for (Shard shard : shards) {
                retVal.add(shard.getShardId());
            }

            if (streamDescription.isHasMoreShards()) {
                request.setExclusiveStartShardId(Iterables.getLast(shards).getShardId());
            } else {
                request = null;
            }
        }

        return retVal;
    });
}

From source file:org.apache.samza.system.kinesis.KinesisSystemAdmin.java

License:Apache License

private SystemStreamMetadata createSystemStreamMetadata(String stream) {
    LOG.info("create stream metadata for stream {} based on aws stream", stream);
    Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> metadata = new HashMap<>();
    AmazonKinesisClient client = null;/*from w w  w.j a  va2  s. c  om*/

    try {
        ClientConfiguration clientConfig = kConfig.getAWSClientConfig(system);
        AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
                .withCredentials(kConfig.credentialsProviderForStream(system, stream))
                .withClientConfiguration(clientConfig);
        builder.setRegion(kConfig.getRegion(system, stream).getName());
        client = (AmazonKinesisClient) builder.build();
        StreamDescription desc = client.describeStream(stream).getStreamDescription();
        IntStream.range(0, desc.getShards().size())
                .forEach(i -> metadata.put(new Partition(i), SYSTEM_STREAM_PARTITION_METADATA));
    } catch (Exception e) {
        String errMsg = "couldn't load metadata for stream " + stream;
        LOG.error(errMsg, e);
        throw new SamzaException(errMsg, e);
    } finally {
        if (client != null) {
            client.shutdown();
        }
    }

    return new SystemStreamMetadata(stream, metadata);
}

From source file:org.lendingclub.mercator.aws.KinesisScanner.java

License:Apache License

private void project(StreamDescription description) {

    ObjectNode n = mapper.createObjectNode();
    n.put("aws_account", getAccountId());
    n.put("aws_region", getRegion().getName());
    n.put("aws_arn", description.getStreamARN());
    n.put("name", description.getStreamName());
    n.put("aws_name", description.getStreamName());
    n.put("aws_status", description.getStreamStatus());
    n.put("aws_streamCreationTimestamp", description.getStreamCreationTimestamp().getTime());
    n.put("aws_retentionPeriodHours", description.getRetentionPeriodHours());
    n.put("aws_shardCount", description.getShards().size());

    incrementEntityCount();/*from  ww  w. j  a v a2  s .  c o  m*/
    String cypher = "merge (k:AwsKinesisStream {aws_arn:{aws_arn}}) set k+={props}, k.updateTs=timestamp() return k";

    getNeoRxClient().execCypher(cypher, "aws_arn", n.path("aws_arn").asText(), "props", n);

    cypher = "match (a:AwsAccount {aws_account:{account}}), (k:AwsKinesisStream {aws_account:{account}}) MERGE (a)-[r:OWNS]->(k) set r.updateTs=timestamp()";

    getNeoRxClient().execCypher(cypher, "account", getAccountId());

}

From source file:org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner.java

License:Apache License

private List<Shard> createOrUpdate(String stream, int shards) {
    List<Shard> shardList = new ArrayList<>();

    int describeStreamRetries = 0;

    String exclusiveStartShardId = null;

    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(stream);

    while (true) {
        DescribeStreamResult describeStreamResult = null;

        try {/*from  w ww. j  a  va 2s.  c  om*/
            describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
            describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest);
            StreamDescription streamDescription = describeStreamResult.getStreamDescription();
            if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) {
                shardList.addAll(streamDescription.getShards());

                if (streamDescription.getHasMoreShards()) {
                    exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId();
                } else {
                    break;
                }
            }
        } catch (ResourceNotFoundException ex) {
            if (!this.configurationProperties.isAutoCreateStream()) {
                throw new ProvisioningException(
                        "The stream [" + stream + "] was not found and auto creation is disabled.", ex);
            }
            if (logger.isInfoEnabled()) {
                logger.info("Stream '" + stream + "' not found. Create one...");
            }

            this.amazonKinesis.createStream(stream,
                    Math.max(this.configurationProperties.getMinShardCount(), shards));
            continue;
        } catch (LimitExceededException ex) {
            logger.info(
                    "Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for ["
                            + this.configurationProperties.getDescribeStreamBackoff() + "] millis.");
        }

        if (describeStreamResult == null || !StreamStatus.ACTIVE.toString()
                .equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
            if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) {
                ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
                        "The stream [" + stream + "] isn't ACTIVE or doesn't exist.");
                resourceNotFoundException.setServiceName("Kinesis");
                throw new ProvisioningException(
                        "Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
                        resourceNotFoundException);
            }
            try {
                Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
            } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
                throw new ProvisioningException(
                        "The [describeStream] thread for the stream [" + stream + "] has been interrupted.",
                        ex);
            }
        }
    }

    int effectiveShardCount = Math.max(this.configurationProperties.getMinShardCount(), shards);

    if ((shardList.size() < effectiveShardCount) && this.configurationProperties.isAutoAddShards()) {
        return updateShardCount(stream, shardList.size(), effectiveShardCount);
    }

    return shardList;
}

From source file:org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner.java

License:Apache License

private List<Shard> updateShardCount(String streamName, int shardCount, int targetCount) {
    if (logger.isInfoEnabled()) {
        logger.info("Stream [" + streamName + "] has [" + shardCount
                + "] shards compared to a target configuration of [" + targetCount + "], creating shards...");
    }/*from   www  . ja  v a  2  s  .c  om*/

    UpdateShardCountRequest updateShardCountRequest = new UpdateShardCountRequest().withStreamName(streamName)
            .withTargetShardCount(targetCount).withScalingType(ScalingType.UNIFORM_SCALING);

    this.amazonKinesis.updateShardCount(updateShardCountRequest);

    // Wait for stream to become active again after resharding
    List<Shard> shardList = new ArrayList<>();

    int describeStreamRetries = 0;

    String exclusiveStartShardId = null;

    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(streamName);

    while (true) {
        DescribeStreamResult describeStreamResult = null;

        try {
            describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
            describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest);
            StreamDescription streamDescription = describeStreamResult.getStreamDescription();
            if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) {
                shardList.addAll(streamDescription.getShards());

                if (streamDescription.getHasMoreShards()) {
                    exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId();
                } else {
                    break;
                }
            }
        } catch (LimitExceededException ex) {
            logger.info("Got LimitExceededException when describing stream [" + streamName + "]. "
                    + "Backing off for [" + this.configurationProperties.getDescribeStreamBackoff()
                    + "] millis.");
        }

        if (describeStreamResult == null || !StreamStatus.ACTIVE.toString()
                .equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
            if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) {
                ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
                        "The stream [" + streamName + "] isn't ACTIVE or doesn't exist.");
                resourceNotFoundException.setServiceName("Kinesis");
                throw new ProvisioningException(
                        "Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
                        resourceNotFoundException);
            }
            try {
                Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
            } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
                throw new ProvisioningException(
                        "The [describeStream] thread for the stream [" + streamName + "] has been interrupted.",
                        ex);
            }
        }
    }
    return shardList;
}