Example usage for com.amazonaws.services.kinesis.model StreamStatus ACTIVE

List of usage examples for com.amazonaws.services.kinesis.model StreamStatus ACTIVE

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.kinesis.model.*
  5. StreamStatus
  6. ACTIVE

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model StreamStatus ACTIVE.

Prototype

StreamStatus ACTIVE

To view the source code for com.amazonaws.services.kinesis.model StreamStatus ACTIVE.

Click Source Link

Usage

From source file:com.gu.logback.appender.kinesis.KinesisAppender.java

License:Open Source License

/**
 * Configures this appender instance and makes it ready for use by the
 * consumers. It validates mandatory parameters and confirms if the configured
 * stream is ready for publishing data yet.
 * /*from ww  w. j  av  a  2  s  .  c  om*/
 * Error details are made available through the fallback handler for this
 * appender
 * 
 * @throws IllegalStateException
 *           if we encounter issues configuring this appender instance
 */
@Override
public void start() {
    if (layout == null) {
        initializationFailed = true;
        addError("Invalid configuration - No layout for appender: " + name);
        return;
    }

    if (streamName == null) {
        initializationFailed = true;
        addError("Invalid configuration - streamName cannot be null for appender: " + name);
        return;
    }

    ClientConfiguration clientConfiguration = new ClientConfiguration();
    clientConfiguration.setMaxErrorRetry(maxRetries);
    clientConfiguration.setRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION,
            PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, maxRetries, true));
    clientConfiguration.setUserAgent(AppenderConstants.USER_AGENT_STRING);

    BlockingQueue<Runnable> taskBuffer = new LinkedBlockingDeque<Runnable>(bufferSize);
    ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(threadCount, threadCount,
            AppenderConstants.DEFAULT_THREAD_KEEP_ALIVE_SEC, TimeUnit.SECONDS, taskBuffer,
            new BlockFastProducerPolicy());
    threadPoolExecutor.prestartAllCoreThreads();
    kinesisClient = new AmazonKinesisAsyncClient(credentials, clientConfiguration, threadPoolExecutor);

    boolean regionProvided = !Validator.isBlank(region);
    if (!regionProvided) {
        region = AppenderConstants.DEFAULT_REGION;
    }
    kinesisClient.setRegion(Region.getRegion(Regions.fromName(region)));
    if (!Validator.isBlank(endpoint)) {
        if (regionProvided) {
            addError("Received configuration for both region as well as Amazon Kinesis endpoint. (" + endpoint
                    + ") will be used as endpoint instead of default endpoint for region (" + region + ")");
        }
        kinesisClient.setEndpoint(endpoint);
    }

    DescribeStreamResult describeResult = null;
    try {
        describeResult = kinesisClient.describeStream(streamName);
        String streamStatus = describeResult.getStreamDescription().getStreamStatus();
        if (!StreamStatus.ACTIVE.name().equals(streamStatus)
                && !StreamStatus.UPDATING.name().equals(streamStatus)) {
            initializationFailed = true;
            addError(
                    "Stream " + streamName + " is not ready (in active/updating status) for appender: " + name);
        }
    } catch (ResourceNotFoundException rnfe) {
        initializationFailed = true;
        addError("Stream " + streamName + " doesn't exist for appender: " + name, rnfe);
    }

    asyncCallHander = new AsyncPutCallStatsReporter(this);

    super.start();
}

From source file:com.tcl.gateway.firehose.log4j.FirehoseAppender.java

License:Open Source License

/**
 * Configures this appender instance and makes it ready for use by the
 * consumers. It validates mandatory parameters and confirms if the configured
 * stream is ready for publishing data yet.
 * // w w  w. j  a v  a 2s.c o  m
 * Error details are made available through the fallback handler for this
 * appender
 * 
 * @throws IllegalStateException
 *           if we encounter issues configuring this appender instance
 */
@Override
public void activateOptions() {
    if (deliveryStreamName == null) {
        initializationFailed = true;
        error("Invalid configuration - streamName cannot be null for appender: " + name);
    }

    if (layout == null) {
        initializationFailed = true;
        error("Invalid configuration - No layout for appender: " + name);
    }

    ClientConfiguration clientConfiguration = new ClientConfiguration();
    clientConfiguration = setProxySettingsFromSystemProperties(clientConfiguration);

    clientConfiguration.setMaxErrorRetry(maxRetries);
    clientConfiguration.setRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION,
            PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, maxRetries, true));
    clientConfiguration.setUserAgent(AppenderConstants.USER_AGENT_STRING);

    final BlockingQueue<Runnable> taskBuffer = new LinkedBlockingDeque<Runnable>(bufferSize);
    ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(threadCount, threadCount,
            AppenderConstants.DEFAULT_THREAD_KEEP_ALIVE_SEC, TimeUnit.SECONDS, taskBuffer,
            new BlockFastProducerPolicy());
    threadPoolExecutor.prestartAllCoreThreads();
    firehoseClient = new AmazonKinesisFirehoseAsyncClient(new CustomCredentialsProviderChain(),
            clientConfiguration, threadPoolExecutor);

    boolean regionProvided = !Validator.isBlank(region);
    if (!regionProvided) {
        region = AppenderConstants.DEFAULT_REGION;
    }
    if (!Validator.isBlank(endpoint)) {
        if (regionProvided) {
            LOGGER.warn("Received configuration for both region as well as Amazon Kinesis endpoint. ("
                    + endpoint + ") will be used as endpoint instead of default endpoint for region (" + region
                    + ")");
        }
        firehoseClient.setEndpoint(endpoint);
    } else {
        firehoseClient.setRegion(Region.getRegion(Regions.fromName(region)));
    }

    DescribeDeliveryStreamResult describeResult = null;
    try {
        describeResult = firehoseClient.describeDeliveryStream(
                new DescribeDeliveryStreamRequest().withDeliveryStreamName(deliveryStreamName));
        String streamStatus = describeResult.getDeliveryStreamDescription().getDeliveryStreamStatus();
        if (!StreamStatus.ACTIVE.name().equals(streamStatus)
                && !StreamStatus.UPDATING.name().equals(streamStatus)) {
            initializationFailed = true;
            error("Delivery Stream " + deliveryStreamName
                    + " is not ready (in active/updating status) for appender: " + name);
        }
    } catch (ResourceNotFoundException rnfe) {
        initializationFailed = true;
        error("Delivery  Stream " + deliveryStreamName + " doesn't exist for appender: " + name, rnfe);
    }

    asyncCallHander = new AsyncPutCallStatsReporter(name);

    if (metric) {
        MetricRegistry registry = new MetricRegistry();
        registry.register("Gauge", new Gauge<Integer>() {

            @Override
            public Integer getValue() {
                return taskBuffer.size();
            }

        });

        ConsoleReporter.forRegistry(registry).build().start(3, TimeUnit.SECONDS);
    }
}

From source file:org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxy.java

License:Apache License

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis stream possess.
 *
 * This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher. This
 * jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation (earlier shard's infos will not appear in result)
 * @return the result of the describe stream operation
 *//*from  w  w w .  j  a  v a2s  . c om*/
private DescribeStreamResult describeStream(String streamName, @Nullable String startShardId)
        throws InterruptedException {
    final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    describeStreamRequest.setExclusiveStartShardId(startShardId);

    DescribeStreamResult describeStreamResult = null;

    // Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
    int attemptCount = 0;
    while (describeStreamResult == null) { // retry until we get a result
        try {
            describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
        } catch (LimitExceededException le) {
            long backoffMillis = fullJitterBackoff(describeStreamBaseBackoffMillis,
                    describeStreamMaxBackoffMillis, describeStreamExpConstant, attemptCount++);
            LOG.warn("Got LimitExceededException when describing stream " + streamName + ". Backing off for "
                    + backoffMillis + " millis.");
            Thread.sleep(backoffMillis);
        } catch (ResourceNotFoundException re) {
            throw new RuntimeException("Error while getting stream details", re);
        }
    }

    String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
    if (!(streamStatus.equals(StreamStatus.ACTIVE.toString())
            || streamStatus.equals(StreamStatus.UPDATING.toString()))) {
        if (LOG.isWarnEnabled()) {
            LOG.warn("The status of stream " + streamName + " is " + streamStatus + "; result of the current "
                    + "describeStream operation will not contain any shard information.");
        }
    }

    // Kinesalite (mock implementation of Kinesis) does not correctly exclude shards before the exclusive
    // start shard id in the returned shards list; check if we need to remove these erroneously returned shards
    if (startShardId != null) {
        List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
        Iterator<Shard> shardItr = shards.iterator();
        while (shardItr.hasNext()) {
            if (KinesisStreamShard.compareShardIds(shardItr.next().getShardId(), startShardId) <= 0) {
                shardItr.remove();
            }
        }
    }

    return describeStreamResult;
}

From source file:org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner.java

License:Apache License

private List<Shard> createOrUpdate(String stream, int shards) {
    List<Shard> shardList = new ArrayList<>();

    int describeStreamRetries = 0;

    String exclusiveStartShardId = null;

    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(stream);

    while (true) {
        DescribeStreamResult describeStreamResult = null;

        try {/* ww w.j  a v  a  2 s . c o m*/
            describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
            describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest);
            StreamDescription streamDescription = describeStreamResult.getStreamDescription();
            if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) {
                shardList.addAll(streamDescription.getShards());

                if (streamDescription.getHasMoreShards()) {
                    exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId();
                } else {
                    break;
                }
            }
        } catch (ResourceNotFoundException ex) {
            if (!this.configurationProperties.isAutoCreateStream()) {
                throw new ProvisioningException(
                        "The stream [" + stream + "] was not found and auto creation is disabled.", ex);
            }
            if (logger.isInfoEnabled()) {
                logger.info("Stream '" + stream + "' not found. Create one...");
            }

            this.amazonKinesis.createStream(stream,
                    Math.max(this.configurationProperties.getMinShardCount(), shards));
            continue;
        } catch (LimitExceededException ex) {
            logger.info(
                    "Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for ["
                            + this.configurationProperties.getDescribeStreamBackoff() + "] millis.");
        }

        if (describeStreamResult == null || !StreamStatus.ACTIVE.toString()
                .equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
            if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) {
                ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
                        "The stream [" + stream + "] isn't ACTIVE or doesn't exist.");
                resourceNotFoundException.setServiceName("Kinesis");
                throw new ProvisioningException(
                        "Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
                        resourceNotFoundException);
            }
            try {
                Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
            } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
                throw new ProvisioningException(
                        "The [describeStream] thread for the stream [" + stream + "] has been interrupted.",
                        ex);
            }
        }
    }

    int effectiveShardCount = Math.max(this.configurationProperties.getMinShardCount(), shards);

    if ((shardList.size() < effectiveShardCount) && this.configurationProperties.isAutoAddShards()) {
        return updateShardCount(stream, shardList.size(), effectiveShardCount);
    }

    return shardList;
}

From source file:org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner.java

License:Apache License

private List<Shard> updateShardCount(String streamName, int shardCount, int targetCount) {
    if (logger.isInfoEnabled()) {
        logger.info("Stream [" + streamName + "] has [" + shardCount
                + "] shards compared to a target configuration of [" + targetCount + "], creating shards...");
    }//from  ww w . ja v a2  s  .c o  m

    UpdateShardCountRequest updateShardCountRequest = new UpdateShardCountRequest().withStreamName(streamName)
            .withTargetShardCount(targetCount).withScalingType(ScalingType.UNIFORM_SCALING);

    this.amazonKinesis.updateShardCount(updateShardCountRequest);

    // Wait for stream to become active again after resharding
    List<Shard> shardList = new ArrayList<>();

    int describeStreamRetries = 0;

    String exclusiveStartShardId = null;

    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(streamName);

    while (true) {
        DescribeStreamResult describeStreamResult = null;

        try {
            describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
            describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest);
            StreamDescription streamDescription = describeStreamResult.getStreamDescription();
            if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) {
                shardList.addAll(streamDescription.getShards());

                if (streamDescription.getHasMoreShards()) {
                    exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId();
                } else {
                    break;
                }
            }
        } catch (LimitExceededException ex) {
            logger.info("Got LimitExceededException when describing stream [" + streamName + "]. "
                    + "Backing off for [" + this.configurationProperties.getDescribeStreamBackoff()
                    + "] millis.");
        }

        if (describeStreamResult == null || !StreamStatus.ACTIVE.toString()
                .equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
            if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) {
                ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
                        "The stream [" + streamName + "] isn't ACTIVE or doesn't exist.");
                resourceNotFoundException.setServiceName("Kinesis");
                throw new ProvisioningException(
                        "Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
                        resourceNotFoundException);
            }
            try {
                Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
            } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
                throw new ProvisioningException(
                        "The [describeStream] thread for the stream [" + streamName + "] has been interrupted.",
                        ex);
            }
        }
    }
    return shardList;
}

From source file:org.springframework.integration.aws.inbound.kinesis.KinesisMessageDrivenChannelAdapter.java

License:Apache License

private void populateShardsForStream(final String stream, final CountDownLatch shardsGatherLatch) {
    this.dispatcherExecutor.execute(new Runnable() {

        @Override/*from   w w w  .  j  av  a  2s  . c  o m*/
        public void run() {
            try {
                int describeStreamRetries = 0;
                List<Shard> shardsToConsume = new ArrayList<>();

                String exclusiveStartShardId = null;
                while (true) {
                    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
                            .withStreamName(stream).withExclusiveStartShardId(exclusiveStartShardId);

                    DescribeStreamResult describeStreamResult = null;
                    // Call DescribeStream, with backoff and retries (if we get LimitExceededException).
                    try {
                        describeStreamResult = KinesisMessageDrivenChannelAdapter.this.amazonKinesis
                                .describeStream(describeStreamRequest);
                    } catch (LimitExceededException e) {
                        logger.info("Got LimitExceededException when describing stream [" + stream + "]. "
                                + "Backing off for ["
                                + KinesisMessageDrivenChannelAdapter.this.describeStreamBackoff + "] millis.");
                    }

                    if (describeStreamResult == null || !StreamStatus.ACTIVE.toString()
                            .equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
                        if (describeStreamRetries++ > KinesisMessageDrivenChannelAdapter.this.describeStreamRetries) {
                            ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
                                    "The stream [" + stream + "] isn't ACTIVE or doesn't exist.");
                            resourceNotFoundException.setServiceName("Kinesis");
                            throw resourceNotFoundException;
                        }
                        try {
                            Thread.sleep(KinesisMessageDrivenChannelAdapter.this.describeStreamBackoff);
                            continue;
                        } catch (InterruptedException e) {
                            Thread.interrupted();
                            throw new IllegalStateException("The [describeStream] thread for the stream ["
                                    + stream + "] has been interrupted.", e);
                        }
                    }

                    List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
                    for (Shard shard : shards) {
                        String endingSequenceNumber = shard.getSequenceNumberRange().getEndingSequenceNumber();
                        if (endingSequenceNumber != null) {
                            String key = buildCheckpointKeyForShard(stream, shard.getShardId());
                            String checkpoint = KinesisMessageDrivenChannelAdapter.this.checkpointStore
                                    .get(key);

                            boolean skipClosedShard = checkpoint != null && new BigInteger(endingSequenceNumber)
                                    .compareTo(new BigInteger(checkpoint)) <= 0;

                            if (logger.isTraceEnabled()) {
                                logger.trace("The shard [" + shard + "] in stream [" + stream
                                        + "] is closed CLOSED with endingSequenceNumber ["
                                        + endingSequenceNumber + "].\nThe last processed checkpoint is ["
                                        + checkpoint + "]."
                                        + (skipClosedShard ? "\nThe shard will be skipped." : ""));
                            }

                            if (skipClosedShard) {
                                // Skip CLOSED shard which has been read before according a checkpoint
                                continue;
                            }
                        }

                        shardsToConsume.add(shard);
                    }

                    if (describeStreamResult.getStreamDescription().getHasMoreShards()) {
                        exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
                    } else {
                        break;
                    }
                }

                for (Shard shard : shardsToConsume) {
                    KinesisShardOffset shardOffset = new KinesisShardOffset(
                            KinesisMessageDrivenChannelAdapter.this.streamInitialSequence);
                    shardOffset.setShard(shard.getShardId());
                    shardOffset.setStream(stream);
                    boolean addedOffset;
                    synchronized (KinesisMessageDrivenChannelAdapter.this.shardOffsets) {
                        addedOffset = KinesisMessageDrivenChannelAdapter.this.shardOffsets.add(shardOffset);
                    }
                    if (addedOffset && shardsGatherLatch == null
                            && KinesisMessageDrivenChannelAdapter.this.active) {
                        populateConsumer(shardOffset);
                    }
                }
            } finally {
                if (shardsGatherLatch != null) {
                    shardsGatherLatch.countDown();
                }
                KinesisMessageDrivenChannelAdapter.this.inResharding.remove(stream);
            }
        }

    });
}