List of usage examples for com.amazonaws.services.kinesis.model StreamDescription getStreamStatus
public String getStreamStatus()
The current status of the stream being described.
From source file:whgHelper.java
License:Open Source License
public static void setStream(AmazonKinesisClient kinesis, String streamName, int shardCount) { try {//from w w w. j ava 2 s . c o m // Describe the stream and check if it exists DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(streamName); StreamDescription streamDescription = kinesis.describeStream(describeStreamRequest) .getStreamDescription(); System.out.printf("Stream %s has a status of %s.\n", streamName, streamDescription.getStreamStatus()); if ("DELETING".equals(streamDescription.getStreamStatus())) { System.out.println("Stream is being deleted. This sample will now exit."); System.exit(0); } // Wait for the stream to become active if it is not yet ACTIVE. if (!"ACTIVE".equals(streamDescription.getStreamStatus())) { try { waitForStreamToBecomeAvailable(kinesis, streamName); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } catch (ResourceNotFoundException ex) { System.out.printf("Stream %s does not exist. Creating it now.\n", streamName); // Create a stream. The number of shards determines the provisioned throughput. CreateStreamRequest createStreamRequest = new CreateStreamRequest(); createStreamRequest.setStreamName(streamName); createStreamRequest.setShardCount(shardCount); kinesis.createStream(createStreamRequest); // The stream is now being created. Wait for it to become active. try { waitForStreamToBecomeAvailable(kinesis, streamName); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
From source file:kinesisAlertAnalysis.java
License:Open Source License
public static void main(String[] args) throws Exception { init();/*from w ww .j ava 2 s. co m*/ final String myStreamName = "alertsStream"; final Integer myStreamSize = 1; /* * The ProfileCredentialsProvider will return your [awsReilly] * credential profile by reading from the credentials file located at * (/Users/johnreilly/.aws/credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider("jreilly").getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (/Users/johnreilly/.aws/credentials), and is in valid format.", e); } // Describe the stream and check if it exists. DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(myStreamName); try { StreamDescription streamDescription = kinesis.describeStream(describeStreamRequest) .getStreamDescription(); System.out.printf("Stream %s has a status of %s.\n", myStreamName, streamDescription.getStreamStatus()); if ("DELETING".equals(streamDescription.getStreamStatus())) { System.out.println("Stream is being deleted. This sample will now exit."); System.exit(0); } // Wait for the stream to become active if it is not yet ACTIVE. if (!"ACTIVE".equals(streamDescription.getStreamStatus())) { waitForStreamToBecomeAvailable(myStreamName); } } catch (ResourceNotFoundException ex) { System.out.printf("Stream %s does not exist. Creating it now.\n", myStreamName); // Create a stream. The number of shards determines the provisioned throughput. CreateStreamRequest createStreamRequest = new CreateStreamRequest(); createStreamRequest.setStreamName(myStreamName); createStreamRequest.setShardCount(myStreamSize); kinesis.createStream(createStreamRequest); // The stream is now being created. Wait for it to become active. waitForStreamToBecomeAvailable(myStreamName); } // List all of my streams. ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesis.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesis.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } // Print all of my streams. System.out.println("List of my streams: "); for (int i = 0; i < streamNames.size(); i++) { System.out.println("\t- " + streamNames.get(i)); } AmazonSQS sqs = new AmazonSQSClient(credentials); Region usEast1 = Region.getRegion(Regions.US_EAST_1); sqs.setRegion(usEast1); System.out.println(""); System.out.println("==========================================="); System.out.println("Getting Started with sqsAlertCache"); System.out.println("===========================================\n"); try { String thisQueue = "alertCache"; String nextQueue = "alertReceive"; // Receive messages System.out.println("Receiving messages from " + thisQueue + "."); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(thisQueue); List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); System.out.println("Message count for " + thisQueue + ": " + messages.size() + "\n"); for (Message message : messages) { System.out.println(" Message"); System.out.println(" MessageId: " + message.getMessageId()); System.out.println(" ReceiptHandle: " + message.getReceiptHandle()); System.out.println(" MD5OfBody: " + message.getMD5OfBody()); System.out.println(" Body: " + message.getBody()); for (Entry<String, String> entry : message.getAttributes().entrySet()) { System.out.println(" Attribute"); System.out.println(" Name: " + entry.getKey()); System.out.println(" Value: " + entry.getValue()); } System.out.println(); // Write record to the stream long createTime = System.currentTimeMillis(); PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setStreamName(myStreamName); putRecordRequest.setData(ByteBuffer.wrap(String.format(message.getBody(), createTime).getBytes())); putRecordRequest.setPartitionKey(String.format("partitionKey-%d", createTime)); PutRecordResult putRecordResult = kinesis.putRecord(putRecordRequest); System.out.printf( "Successfully put record, partition key : %s, ShardID : %s, SequenceNumber : %s.\n", putRecordRequest.getPartitionKey(), putRecordResult.getShardId(), putRecordResult.getSequenceNumber()); // then send message to cache queue System.out.println("Sending messages to next queue."); sqs.sendMessage(new SendMessageRequest(nextQueue, message.getBody())); // delete message after sending to persist queue System.out.println("Deleting message from this queue.\n"); String messageRecieptHandle = message.getReceiptHandle(); sqs.deleteMessage(new DeleteMessageRequest(thisQueue, messageRecieptHandle)); } } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon SQS, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with SQS, such as not " + "being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:AmazonKinesisRecordProducerSample.java
License:Open Source License
public static void main(String[] args) throws Exception { init();/*from w w w. j a v a 2 s . c o m*/ final String myStreamName = AmazonKinesisApplicationSample.SAMPLE_APPLICATION_STREAM_NAME; final Integer myStreamSize = 1; // Describe the stream and check if it exists. DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(myStreamName); try { StreamDescription streamDescription = kinesis.describeStream(describeStreamRequest) .getStreamDescription(); System.out.printf("Stream %s has a status of %s.\n", myStreamName, streamDescription.getStreamStatus()); if ("DELETING".equals(streamDescription.getStreamStatus())) { System.out.println("Stream is being deleted. This sample will now exit."); System.exit(0); } // Wait for the stream to become active if it is not yet ACTIVE. if (!"ACTIVE".equals(streamDescription.getStreamStatus())) { waitForStreamToBecomeAvailable(myStreamName); } } catch (ResourceNotFoundException ex) { System.out.printf("Stream %s does not exist. Creating it now.\n", myStreamName); // Create a stream. The number of shards determines the provisioned throughput. CreateStreamRequest createStreamRequest = new CreateStreamRequest(); createStreamRequest.setStreamName(myStreamName); createStreamRequest.setShardCount(myStreamSize); kinesis.createStream(createStreamRequest); // The stream is now being created. Wait for it to become active. waitForStreamToBecomeAvailable(myStreamName); } // List all of my streams. ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesis.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesis.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } // Print all of my streams. System.out.println("List of my streams: "); for (int i = 0; i < streamNames.size(); i++) { System.out.println("\t- " + streamNames.get(i)); } System.out.printf("Putting records in stream : %s until this application is stopped...\n", myStreamName); System.out.println("Press CTRL-C to stop."); // Write records to the stream until this program is aborted. while (true) { long createTime = System.currentTimeMillis(); PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setStreamName(myStreamName); putRecordRequest.setData(ByteBuffer.wrap(String.format("testData-%d", createTime).getBytes())); putRecordRequest.setPartitionKey(String.format("partitionKey-%d", createTime)); PutRecordResult putRecordResult = kinesis.putRecord(putRecordRequest); System.out.printf("Successfully put record, partition key : %s, ShardID : %s, SequenceNumber : %s.\n", putRecordRequest.getPartitionKey(), putRecordResult.getShardId(), putRecordResult.getSequenceNumber()); } }
From source file:com.facebook.presto.kinesis.util.EmbeddedKinesisStream.java
License:Apache License
private String checkStreamStatus(String streamName) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(streamName); StreamDescription streamDescription = amazonKinesisClient.describeStream(describeStreamRequest) .getStreamDescription();/*from w w w .java 2 s. c o m*/ return streamDescription.getStreamStatus(); }
From source file:com.haskins.cloudtrailviewer.dialog.resourcedetail.detailpanels.KinesisStreamDetail.java
License:Open Source License
private void buildUI(DescribeStreamResult detail) { this.add(primaryScrollPane, BorderLayout.CENTER); if (detail.getStreamDescription() != null) { StreamDescription stream = detail.getStreamDescription(); if (stream.getHasMoreShards() != null) { primaryTableModel.addRow(new Object[] { "Has More Shards", stream.getHasMoreShards() }); }/*from w w w . j a va2s.c o m*/ if (stream.getStreamARN() != null) { primaryTableModel.addRow(new Object[] { "Stream Arn", stream.getStreamARN() }); } if (stream.getStreamName() != null) { primaryTableModel.addRow(new Object[] { "Stream Name", stream.getStreamName() }); } if (stream.getStreamStatus() != null) { primaryTableModel.addRow(new Object[] { "Stram Status", stream.getStreamStatus() }); } } }
From source file:com.xujinpeng.kinesis.AmazonKinesisRecordProducer.java
License:Open Source License
public void produceData(int counter) throws InterruptedException { String myStreamName = Constants.APPLICATION_STREAM_NAME; Integer myStreamSize = 1;//from w w w.j a va2 s. c o m // Describe the stream and check if it exists. DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(myStreamName); try { StreamDescription streamDescription = kinesis.describeStream(describeStreamRequest) .getStreamDescription(); System.out.printf("Stream %s has a status of %s.\n", myStreamName, streamDescription.getStreamStatus()); if ("DELETING".equals(streamDescription.getStreamStatus())) { System.out.println("Stream is being deleted. This sample will now exit."); waitForStreamToBecomeAvailable(myStreamName); // System.exit(0); } // Wait for the stream to become active if it is not yet ACTIVE. if (!"ACTIVE".equals(streamDescription.getStreamStatus())) { waitForStreamToBecomeAvailable(myStreamName); } } catch (ResourceNotFoundException ex) { System.out.printf("Stream %s does not exist. Creating it now.\n", myStreamName); // Create a stream. The number of shards determines the provisioned throughput. CreateStreamRequest createStreamRequest = new CreateStreamRequest(); createStreamRequest.setStreamName(myStreamName); createStreamRequest.setShardCount(myStreamSize); kinesis.createStream(createStreamRequest); // The stream is now being created. Wait for it to become active. waitForStreamToBecomeAvailable(myStreamName); } // List all of my streams. ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesis.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesis.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } // Print all of my streams. System.out.println("List of my streams: "); for (int i = 0; i < streamNames.size(); i++) { System.out.println("\t- " + streamNames.get(i)); } System.out.printf("Putting records in stream : %s until this application is stopped...\n", myStreamName); System.out.println("Press CTRL-C to stop."); // Write records to the stream until this program is aborted. for (int i = 0; i < counter; i++) { long createTime = System.currentTimeMillis(); PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setStreamName(myStreamName); Date loginDate = new Date(System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1)); DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss a"); System.out.println(dateFormat.format(loginDate)); String randomData = String.format( "{\"username\":\"%s\", \"loginTime\":\"%s\", \"gameDuration\":\"%s\", \"gameResult\":\"%s\", \"isBlackJack\":\"%s\"}\n", getRandomUserName(), dateFormat.format(loginDate), getRandomGameDuration(), getRandomGameResult(), getRandomBlackJackResult()); putRecordRequest.setData(ByteBuffer.wrap(randomData.getBytes())); putRecordRequest.setPartitionKey(String.format("partitionKey-%d", createTime)); kinesis.putRecord(putRecordRequest); System.out.printf(randomData); } }
From source file:org.lendingclub.mercator.aws.KinesisScanner.java
License:Apache License
private void project(StreamDescription description) { ObjectNode n = mapper.createObjectNode(); n.put("aws_account", getAccountId()); n.put("aws_region", getRegion().getName()); n.put("aws_arn", description.getStreamARN()); n.put("name", description.getStreamName()); n.put("aws_name", description.getStreamName()); n.put("aws_status", description.getStreamStatus()); n.put("aws_streamCreationTimestamp", description.getStreamCreationTimestamp().getTime()); n.put("aws_retentionPeriodHours", description.getRetentionPeriodHours()); n.put("aws_shardCount", description.getShards().size()); incrementEntityCount();/*from ww w. j av a 2 s . c om*/ String cypher = "merge (k:AwsKinesisStream {aws_arn:{aws_arn}}) set k+={props}, k.updateTs=timestamp() return k"; getNeoRxClient().execCypher(cypher, "aws_arn", n.path("aws_arn").asText(), "props", n); cypher = "match (a:AwsAccount {aws_account:{account}}), (k:AwsKinesisStream {aws_account:{account}}) MERGE (a)-[r:OWNS]->(k) set r.updateTs=timestamp()"; getNeoRxClient().execCypher(cypher, "account", getAccountId()); }
From source file:org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner.java
License:Apache License
private List<Shard> createOrUpdate(String stream, int shards) { List<Shard> shardList = new ArrayList<>(); int describeStreamRetries = 0; String exclusiveStartShardId = null; DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(stream); while (true) { DescribeStreamResult describeStreamResult = null; try {//from w ww.j av a 2s.c o m describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId); describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest); StreamDescription streamDescription = describeStreamResult.getStreamDescription(); if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) { shardList.addAll(streamDescription.getShards()); if (streamDescription.getHasMoreShards()) { exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId(); } else { break; } } } catch (ResourceNotFoundException ex) { if (!this.configurationProperties.isAutoCreateStream()) { throw new ProvisioningException( "The stream [" + stream + "] was not found and auto creation is disabled.", ex); } if (logger.isInfoEnabled()) { logger.info("Stream '" + stream + "' not found. Create one..."); } this.amazonKinesis.createStream(stream, Math.max(this.configurationProperties.getMinShardCount(), shards)); continue; } catch (LimitExceededException ex) { logger.info( "Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for [" + this.configurationProperties.getDescribeStreamBackoff() + "] millis."); } if (describeStreamResult == null || !StreamStatus.ACTIVE.toString() .equals(describeStreamResult.getStreamDescription().getStreamStatus())) { if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) { ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException( "The stream [" + stream + "] isn't ACTIVE or doesn't exist."); resourceNotFoundException.setServiceName("Kinesis"); throw new ProvisioningException( "Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error", resourceNotFoundException); } try { Thread.sleep(this.configurationProperties.getDescribeStreamBackoff()); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new ProvisioningException( "The [describeStream] thread for the stream [" + stream + "] has been interrupted.", ex); } } } int effectiveShardCount = Math.max(this.configurationProperties.getMinShardCount(), shards); if ((shardList.size() < effectiveShardCount) && this.configurationProperties.isAutoAddShards()) { return updateShardCount(stream, shardList.size(), effectiveShardCount); } return shardList; }
From source file:org.springframework.cloud.stream.binder.kinesis.provisioning.KinesisStreamProvisioner.java
License:Apache License
private List<Shard> updateShardCount(String streamName, int shardCount, int targetCount) { if (logger.isInfoEnabled()) { logger.info("Stream [" + streamName + "] has [" + shardCount + "] shards compared to a target configuration of [" + targetCount + "], creating shards..."); }/*w ww .j a v a2 s. com*/ UpdateShardCountRequest updateShardCountRequest = new UpdateShardCountRequest().withStreamName(streamName) .withTargetShardCount(targetCount).withScalingType(ScalingType.UNIFORM_SCALING); this.amazonKinesis.updateShardCount(updateShardCountRequest); // Wait for stream to become active again after resharding List<Shard> shardList = new ArrayList<>(); int describeStreamRetries = 0; String exclusiveStartShardId = null; DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(streamName); while (true) { DescribeStreamResult describeStreamResult = null; try { describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId); describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest); StreamDescription streamDescription = describeStreamResult.getStreamDescription(); if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) { shardList.addAll(streamDescription.getShards()); if (streamDescription.getHasMoreShards()) { exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId(); } else { break; } } } catch (LimitExceededException ex) { logger.info("Got LimitExceededException when describing stream [" + streamName + "]. " + "Backing off for [" + this.configurationProperties.getDescribeStreamBackoff() + "] millis."); } if (describeStreamResult == null || !StreamStatus.ACTIVE.toString() .equals(describeStreamResult.getStreamDescription().getStreamStatus())) { if (describeStreamRetries++ > this.configurationProperties.getDescribeStreamRetries()) { ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException( "The stream [" + streamName + "] isn't ACTIVE or doesn't exist."); resourceNotFoundException.setServiceName("Kinesis"); throw new ProvisioningException( "Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error", resourceNotFoundException); } try { Thread.sleep(this.configurationProperties.getDescribeStreamBackoff()); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new ProvisioningException( "The [describeStream] thread for the stream [" + streamName + "] has been interrupted.", ex); } } } return shardList; }