List of usage examples for com.amazonaws.services.kinesis.model StreamStatus UPDATING
StreamStatus UPDATING
To view the source code for com.amazonaws.services.kinesis.model StreamStatus UPDATING.
Click Source Link
From source file:com.gu.logback.appender.kinesis.KinesisAppender.java
License:Open Source License
/** * Configures this appender instance and makes it ready for use by the * consumers. It validates mandatory parameters and confirms if the configured * stream is ready for publishing data yet. * /*from w w w . ja va 2s . c o m*/ * Error details are made available through the fallback handler for this * appender * * @throws IllegalStateException * if we encounter issues configuring this appender instance */ @Override public void start() { if (layout == null) { initializationFailed = true; addError("Invalid configuration - No layout for appender: " + name); return; } if (streamName == null) { initializationFailed = true; addError("Invalid configuration - streamName cannot be null for appender: " + name); return; } ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setMaxErrorRetry(maxRetries); clientConfiguration.setRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, maxRetries, true)); clientConfiguration.setUserAgent(AppenderConstants.USER_AGENT_STRING); BlockingQueue<Runnable> taskBuffer = new LinkedBlockingDeque<Runnable>(bufferSize); ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(threadCount, threadCount, AppenderConstants.DEFAULT_THREAD_KEEP_ALIVE_SEC, TimeUnit.SECONDS, taskBuffer, new BlockFastProducerPolicy()); threadPoolExecutor.prestartAllCoreThreads(); kinesisClient = new AmazonKinesisAsyncClient(credentials, clientConfiguration, threadPoolExecutor); boolean regionProvided = !Validator.isBlank(region); if (!regionProvided) { region = AppenderConstants.DEFAULT_REGION; } kinesisClient.setRegion(Region.getRegion(Regions.fromName(region))); if (!Validator.isBlank(endpoint)) { if (regionProvided) { addError("Received configuration for both region as well as Amazon Kinesis endpoint. (" + endpoint + ") will be used as endpoint instead of default endpoint for region (" + region + ")"); } kinesisClient.setEndpoint(endpoint); } DescribeStreamResult describeResult = null; try { describeResult = kinesisClient.describeStream(streamName); String streamStatus = describeResult.getStreamDescription().getStreamStatus(); if (!StreamStatus.ACTIVE.name().equals(streamStatus) && !StreamStatus.UPDATING.name().equals(streamStatus)) { initializationFailed = true; addError( "Stream " + streamName + " is not ready (in active/updating status) for appender: " + name); } } catch (ResourceNotFoundException rnfe) { initializationFailed = true; addError("Stream " + streamName + " doesn't exist for appender: " + name, rnfe); } asyncCallHander = new AsyncPutCallStatsReporter(this); super.start(); }
From source file:com.tcl.gateway.firehose.log4j.FirehoseAppender.java
License:Open Source License
/** * Configures this appender instance and makes it ready for use by the * consumers. It validates mandatory parameters and confirms if the configured * stream is ready for publishing data yet. * /*from w ww . ja v a 2 s . c om*/ * Error details are made available through the fallback handler for this * appender * * @throws IllegalStateException * if we encounter issues configuring this appender instance */ @Override public void activateOptions() { if (deliveryStreamName == null) { initializationFailed = true; error("Invalid configuration - streamName cannot be null for appender: " + name); } if (layout == null) { initializationFailed = true; error("Invalid configuration - No layout for appender: " + name); } ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration = setProxySettingsFromSystemProperties(clientConfiguration); clientConfiguration.setMaxErrorRetry(maxRetries); clientConfiguration.setRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, maxRetries, true)); clientConfiguration.setUserAgent(AppenderConstants.USER_AGENT_STRING); final BlockingQueue<Runnable> taskBuffer = new LinkedBlockingDeque<Runnable>(bufferSize); ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(threadCount, threadCount, AppenderConstants.DEFAULT_THREAD_KEEP_ALIVE_SEC, TimeUnit.SECONDS, taskBuffer, new BlockFastProducerPolicy()); threadPoolExecutor.prestartAllCoreThreads(); firehoseClient = new AmazonKinesisFirehoseAsyncClient(new CustomCredentialsProviderChain(), clientConfiguration, threadPoolExecutor); boolean regionProvided = !Validator.isBlank(region); if (!regionProvided) { region = AppenderConstants.DEFAULT_REGION; } if (!Validator.isBlank(endpoint)) { if (regionProvided) { LOGGER.warn("Received configuration for both region as well as Amazon Kinesis endpoint. (" + endpoint + ") will be used as endpoint instead of default endpoint for region (" + region + ")"); } firehoseClient.setEndpoint(endpoint); } else { firehoseClient.setRegion(Region.getRegion(Regions.fromName(region))); } DescribeDeliveryStreamResult describeResult = null; try { describeResult = firehoseClient.describeDeliveryStream( new DescribeDeliveryStreamRequest().withDeliveryStreamName(deliveryStreamName)); String streamStatus = describeResult.getDeliveryStreamDescription().getDeliveryStreamStatus(); if (!StreamStatus.ACTIVE.name().equals(streamStatus) && !StreamStatus.UPDATING.name().equals(streamStatus)) { initializationFailed = true; error("Delivery Stream " + deliveryStreamName + " is not ready (in active/updating status) for appender: " + name); } } catch (ResourceNotFoundException rnfe) { initializationFailed = true; error("Delivery Stream " + deliveryStreamName + " doesn't exist for appender: " + name, rnfe); } asyncCallHander = new AsyncPutCallStatsReporter(name); if (metric) { MetricRegistry registry = new MetricRegistry(); registry.register("Gauge", new Gauge<Integer>() { @Override public Integer getValue() { return taskBuffer.size(); } }); ConsoleReporter.forRegistry(registry).build().start(3, TimeUnit.SECONDS); } }
From source file:org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxy.java
License:Apache License
/** * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis stream possess. * * This method is using a "full jitter" approach described in AWS's article, * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and Jitter"</a>. * This is necessary because concurrent calls will be made by all parallel subtask's fetcher. This * jitter backoff approach will help distribute calls across the fetchers over time. * * @param streamName the stream to describe * @param startShardId which shard to start with for this describe operation (earlier shard's infos will not appear in result) * @return the result of the describe stream operation *//* w w w .j a v a 2 s . c o m*/ private DescribeStreamResult describeStream(String streamName, @Nullable String startShardId) throws InterruptedException { final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(streamName); describeStreamRequest.setExclusiveStartShardId(startShardId); DescribeStreamResult describeStreamResult = null; // Call DescribeStream, with full-jitter backoff (if we get LimitExceededException). int attemptCount = 0; while (describeStreamResult == null) { // retry until we get a result try { describeStreamResult = kinesisClient.describeStream(describeStreamRequest); } catch (LimitExceededException le) { long backoffMillis = fullJitterBackoff(describeStreamBaseBackoffMillis, describeStreamMaxBackoffMillis, describeStreamExpConstant, attemptCount++); LOG.warn("Got LimitExceededException when describing stream " + streamName + ". Backing off for " + backoffMillis + " millis."); Thread.sleep(backoffMillis); } catch (ResourceNotFoundException re) { throw new RuntimeException("Error while getting stream details", re); } } String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus(); if (!(streamStatus.equals(StreamStatus.ACTIVE.toString()) || streamStatus.equals(StreamStatus.UPDATING.toString()))) { if (LOG.isWarnEnabled()) { LOG.warn("The status of stream " + streamName + " is " + streamStatus + "; result of the current " + "describeStream operation will not contain any shard information."); } } // Kinesalite (mock implementation of Kinesis) does not correctly exclude shards before the exclusive // start shard id in the returned shards list; check if we need to remove these erroneously returned shards if (startShardId != null) { List<Shard> shards = describeStreamResult.getStreamDescription().getShards(); Iterator<Shard> shardItr = shards.iterator(); while (shardItr.hasNext()) { if (KinesisStreamShard.compareShardIds(shardItr.next().getShardId(), startShardId) <= 0) { shardItr.remove(); } } } return describeStreamResult; }