List of usage examples for com.amazonaws.services.kinesis.model DescribeStreamRequest DescribeStreamRequest
DescribeStreamRequest
From source file:whgHelper.java
License:Open Source License
public static void setStream(AmazonKinesisClient kinesis, String streamName, int shardCount) { try {/* w ww . jav a 2 s. co m*/ // Describe the stream and check if it exists DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(streamName); StreamDescription streamDescription = kinesis.describeStream(describeStreamRequest) .getStreamDescription(); System.out.printf("Stream %s has a status of %s.\n", streamName, streamDescription.getStreamStatus()); if ("DELETING".equals(streamDescription.getStreamStatus())) { System.out.println("Stream is being deleted. This sample will now exit."); System.exit(0); } // Wait for the stream to become active if it is not yet ACTIVE. if (!"ACTIVE".equals(streamDescription.getStreamStatus())) { try { waitForStreamToBecomeAvailable(kinesis, streamName); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } catch (ResourceNotFoundException ex) { System.out.printf("Stream %s does not exist. Creating it now.\n", streamName); // Create a stream. The number of shards determines the provisioned throughput. CreateStreamRequest createStreamRequest = new CreateStreamRequest(); createStreamRequest.setStreamName(streamName); createStreamRequest.setShardCount(shardCount); kinesis.createStream(createStreamRequest); // The stream is now being created. Wait for it to become active. try { waitForStreamToBecomeAvailable(kinesis, streamName); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
From source file:whgHelper.java
License:Open Source License
private static void waitForStreamToBecomeAvailable(AmazonKinesisClient kinesis, String myStreamName) throws InterruptedException { System.out.printf("Waiting for %s to become ACTIVE...\n", myStreamName); long startTime = System.currentTimeMillis(); long endTime = startTime + TimeUnit.MINUTES.toMillis(10); while (System.currentTimeMillis() < endTime) { Thread.sleep(TimeUnit.SECONDS.toMillis(20)); try {//from w ww .j a va 2 s . c o m DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(myStreamName); // ask for no more than 10 shards at a time -- this is an optional parameter describeStreamRequest.setLimit(10); DescribeStreamResult describeStreamResponse = kinesis.describeStream(describeStreamRequest); String streamStatus = describeStreamResponse.getStreamDescription().getStreamStatus(); System.out.printf("\t- current state: %s\n", streamStatus); if ("ACTIVE".equals(streamStatus)) { return; } } catch (ResourceNotFoundException ex) { // ResourceNotFound means the stream doesn't exist yet, // so ignore this error and just keep polling. } catch (AmazonServiceException ase) { throw ase; } } throw new RuntimeException(String.format("Stream %s never became active", myStreamName)); }
From source file:AmazonKinesisSample.java
License:Open Source License
private static void waitForStreamToBecomeAvailable(String myStreamName) { System.out.println("Waiting for " + myStreamName + " to become ACTIVE..."); long startTime = System.currentTimeMillis(); long endTime = startTime + (10 * 60 * 1000); while (System.currentTimeMillis() < endTime) { try {// www . ja v a 2 s . c o m Thread.sleep(1000 * 20); } catch (InterruptedException e) { // Ignore interruption (doesn't impact stream creation) } try { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(myStreamName); // ask for no more than 10 shards at a time -- this is an optional parameter describeStreamRequest.setLimit(10); DescribeStreamResult describeStreamResponse = kinesisClient.describeStream(describeStreamRequest); String streamStatus = describeStreamResponse.getStreamDescription().getStreamStatus(); System.out.println(" - current state: " + streamStatus); if (streamStatus.equals("ACTIVE")) { return; } } catch (AmazonServiceException ase) { if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) { throw ase; } throw new RuntimeException("Stream " + myStreamName + " never went active"); } } }
From source file:kinesisAlertAnalysis.java
License:Open Source License
public static void main(String[] args) throws Exception { init();// w ww . j a v a 2s .c o m final String myStreamName = "alertsStream"; final Integer myStreamSize = 1; /* * The ProfileCredentialsProvider will return your [awsReilly] * credential profile by reading from the credentials file located at * (/Users/johnreilly/.aws/credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider("jreilly").getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (/Users/johnreilly/.aws/credentials), and is in valid format.", e); } // Describe the stream and check if it exists. DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(myStreamName); try { StreamDescription streamDescription = kinesis.describeStream(describeStreamRequest) .getStreamDescription(); System.out.printf("Stream %s has a status of %s.\n", myStreamName, streamDescription.getStreamStatus()); if ("DELETING".equals(streamDescription.getStreamStatus())) { System.out.println("Stream is being deleted. This sample will now exit."); System.exit(0); } // Wait for the stream to become active if it is not yet ACTIVE. if (!"ACTIVE".equals(streamDescription.getStreamStatus())) { waitForStreamToBecomeAvailable(myStreamName); } } catch (ResourceNotFoundException ex) { System.out.printf("Stream %s does not exist. Creating it now.\n", myStreamName); // Create a stream. The number of shards determines the provisioned throughput. CreateStreamRequest createStreamRequest = new CreateStreamRequest(); createStreamRequest.setStreamName(myStreamName); createStreamRequest.setShardCount(myStreamSize); kinesis.createStream(createStreamRequest); // The stream is now being created. Wait for it to become active. waitForStreamToBecomeAvailable(myStreamName); } // List all of my streams. ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesis.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesis.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } // Print all of my streams. System.out.println("List of my streams: "); for (int i = 0; i < streamNames.size(); i++) { System.out.println("\t- " + streamNames.get(i)); } AmazonSQS sqs = new AmazonSQSClient(credentials); Region usEast1 = Region.getRegion(Regions.US_EAST_1); sqs.setRegion(usEast1); System.out.println(""); System.out.println("==========================================="); System.out.println("Getting Started with sqsAlertCache"); System.out.println("===========================================\n"); try { String thisQueue = "alertCache"; String nextQueue = "alertReceive"; // Receive messages System.out.println("Receiving messages from " + thisQueue + "."); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(thisQueue); List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); System.out.println("Message count for " + thisQueue + ": " + messages.size() + "\n"); for (Message message : messages) { System.out.println(" Message"); System.out.println(" MessageId: " + message.getMessageId()); System.out.println(" ReceiptHandle: " + message.getReceiptHandle()); System.out.println(" MD5OfBody: " + message.getMD5OfBody()); System.out.println(" Body: " + message.getBody()); for (Entry<String, String> entry : message.getAttributes().entrySet()) { System.out.println(" Attribute"); System.out.println(" Name: " + entry.getKey()); System.out.println(" Value: " + entry.getValue()); } System.out.println(); // Write record to the stream long createTime = System.currentTimeMillis(); PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setStreamName(myStreamName); putRecordRequest.setData(ByteBuffer.wrap(String.format(message.getBody(), createTime).getBytes())); putRecordRequest.setPartitionKey(String.format("partitionKey-%d", createTime)); PutRecordResult putRecordResult = kinesis.putRecord(putRecordRequest); System.out.printf( "Successfully put record, partition key : %s, ShardID : %s, SequenceNumber : %s.\n", putRecordRequest.getPartitionKey(), putRecordResult.getShardId(), putRecordResult.getSequenceNumber()); // then send message to cache queue System.out.println("Sending messages to next queue."); sqs.sendMessage(new SendMessageRequest(nextQueue, message.getBody())); // delete message after sending to persist queue System.out.println("Deleting message from this queue.\n"); String messageRecieptHandle = message.getReceiptHandle(); sqs.deleteMessage(new DeleteMessageRequest(thisQueue, messageRecieptHandle)); } } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon SQS, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with SQS, such as not " + "being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:kinesisAlertAnalysis.java
License:Open Source License
private static void waitForStreamToBecomeAvailable(String myStreamName) throws InterruptedException { System.out.printf("Waiting for %s to become ACTIVE...\n", myStreamName); long startTime = System.currentTimeMillis(); long endTime = startTime + TimeUnit.MINUTES.toMillis(10); while (System.currentTimeMillis() < endTime) { Thread.sleep(TimeUnit.SECONDS.toMillis(20)); try {/*w w w . j av a 2s . co m*/ DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(myStreamName); // ask for no more than 10 shards at a time -- this is an optional parameter describeStreamRequest.setLimit(10); DescribeStreamResult describeStreamResponse = kinesis.describeStream(describeStreamRequest); String streamStatus = describeStreamResponse.getStreamDescription().getStreamStatus(); System.out.printf("\t- current state: %s\n", streamStatus); if ("ACTIVE".equals(streamStatus)) { return; } } catch (ResourceNotFoundException ex) { // ResourceNotFound means the stream doesn't exist yet, // so ignore this error and just keep polling. } catch (AmazonServiceException ase) { throw ase; } } throw new RuntimeException(String.format("Stream %s never became active", myStreamName)); }
From source file:AmazonKinesisGet.java
License:Open Source License
public static void main(String[] args) throws Exception { init();/*from ww w. j a va 2 s . c o m*/ final String myStreamName = "philsteststream"; final Integer myStreamSize = 1; // list all of my streams ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesisClient.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } LOG.info("Printing my list of streams : "); // print all of my streams. if (!streamNames.isEmpty()) { System.out.println("List of my streams: "); } for (int i = 0; i < streamNames.size(); i++) { System.out.println(streamNames.get(i)); } //System.out.println(streamNames.get(0)); String myownstream = streamNames.get(0); // Retrieve the Shards from a Stream DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(myownstream); DescribeStreamResult describeStreamResult; List<Shard> shards = new ArrayList<>(); String lastShardId = null; do { describeStreamRequest.setExclusiveStartShardId(lastShardId); describeStreamResult = kinesisClient.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (shards.size() > 0) { lastShardId = shards.get(shards.size() - 1).getShardId(); } } while (describeStreamResult.getStreamDescription().getHasMoreShards()); // Get Data from the Shards in a Stream // Hard-coded to use only 1 shard String shardIterator; GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(myownstream); //get(0) shows hardcoded to 1 stream getShardIteratorRequest.setShardId(shards.get(0).getShardId()); // using TRIM_HORIZON but could use alternatives getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON"); GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest); shardIterator = getShardIteratorResult.getShardIterator(); // Continuously read data records from shard. List<Record> records; while (true) { // Create new GetRecordsRequest with existing shardIterator. // Set maximum records to return to 1000. GetRecordsRequest getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(1000); GetRecordsResult result = kinesisClient.getRecords(getRecordsRequest); // Put result into record list. Result may be empty. records = result.getRecords(); // Print records for (Record record : records) { ByteBuffer byteBuffer = record.getData(); System.out.println(String.format("Seq No: %s - %s", record.getSequenceNumber(), new String(byteBuffer.array()))); } try { Thread.sleep(1000); } catch (InterruptedException exception) { throw new RuntimeException(exception); } shardIterator = result.getNextShardIterator(); } }
From source file:AmazonKinesisRecordProducerSample.java
License:Open Source License
public static void main(String[] args) throws Exception { init();/*from w w w.ja v a2 s. c om*/ final String myStreamName = AmazonKinesisApplicationSample.SAMPLE_APPLICATION_STREAM_NAME; final Integer myStreamSize = 1; // Describe the stream and check if it exists. DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(myStreamName); try { StreamDescription streamDescription = kinesis.describeStream(describeStreamRequest) .getStreamDescription(); System.out.printf("Stream %s has a status of %s.\n", myStreamName, streamDescription.getStreamStatus()); if ("DELETING".equals(streamDescription.getStreamStatus())) { System.out.println("Stream is being deleted. This sample will now exit."); System.exit(0); } // Wait for the stream to become active if it is not yet ACTIVE. if (!"ACTIVE".equals(streamDescription.getStreamStatus())) { waitForStreamToBecomeAvailable(myStreamName); } } catch (ResourceNotFoundException ex) { System.out.printf("Stream %s does not exist. Creating it now.\n", myStreamName); // Create a stream. The number of shards determines the provisioned throughput. CreateStreamRequest createStreamRequest = new CreateStreamRequest(); createStreamRequest.setStreamName(myStreamName); createStreamRequest.setShardCount(myStreamSize); kinesis.createStream(createStreamRequest); // The stream is now being created. Wait for it to become active. waitForStreamToBecomeAvailable(myStreamName); } // List all of my streams. ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesis.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesis.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } // Print all of my streams. System.out.println("List of my streams: "); for (int i = 0; i < streamNames.size(); i++) { System.out.println("\t- " + streamNames.get(i)); } System.out.printf("Putting records in stream : %s until this application is stopped...\n", myStreamName); System.out.println("Press CTRL-C to stop."); // Write records to the stream until this program is aborted. while (true) { long createTime = System.currentTimeMillis(); PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setStreamName(myStreamName); putRecordRequest.setData(ByteBuffer.wrap(String.format("testData-%d", createTime).getBytes())); putRecordRequest.setPartitionKey(String.format("partitionKey-%d", createTime)); PutRecordResult putRecordResult = kinesis.putRecord(putRecordRequest); System.out.printf("Successfully put record, partition key : %s, ShardID : %s, SequenceNumber : %s.\n", putRecordRequest.getPartitionKey(), putRecordResult.getShardId(), putRecordResult.getSequenceNumber()); } }
From source file:com.amazon.services.awsrum.utils.KinesisUtils.java
License:Open Source License
/** * Helper method to determine if a Kinesis stream exists. * /* ww w . ja v a 2s . c o m*/ * @param kinesisClient * The {@link AmazonKinesisClient} with Kinesis read privileges * @param streamName * The Kinesis stream to check for * @return true if the Kinesis stream exists, otherwise return false */ private static boolean streamExists(AmazonKinesisClient kinesisClient, String streamName) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(streamName); try { kinesisClient.describeStream(describeStreamRequest); return true; } catch (ResourceNotFoundException e) { return false; } }
From source file:com.amazon.services.awsrum.utils.KinesisUtils.java
License:Open Source License
/** * Return the state of a Kinesis stream. * /* ww w. j av a2 s. c o m*/ * @param kinesisClient * The {@link AmazonKinesisClient} with Kinesis read privileges * @param streamName * The Kinesis stream to get the state of * @return String representation of the Stream state */ private static String streamState(AmazonKinesisClient kinesisClient, String streamName) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(streamName); try { return kinesisClient.describeStream(describeStreamRequest).getStreamDescription().getStreamStatus(); } catch (AmazonServiceException e) { return null; } }
From source file:com.boundary.aws.kinesis.Sample.java
License:Open Source License
private static void waitForStreamToBecomeAvailable(String myStreamName) { System.out.println("Waiting for " + myStreamName + " to become ACTIVE..."); long startTime = System.currentTimeMillis(); long endTime = startTime + (10 * 60 * 1000); while (System.currentTimeMillis() < endTime) { try {/* w ww . ja v a 2 s .c om*/ Thread.sleep(1000 * 20); } catch (InterruptedException e) { // Ignore interruption (doesn't impact stream creation) } try { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(myStreamName); // ask for no more than 10 shards at a time -- this is an // optional parameter describeStreamRequest.setLimit(10); DescribeStreamResult describeStreamResponse = kinesisClient.describeStream(describeStreamRequest); String streamStatus = describeStreamResponse.getStreamDescription().getStreamStatus(); System.out.println(" - current state: " + streamStatus); if (streamStatus.equals("ACTIVE")) { return; } } catch (AmazonServiceException ase) { if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) { throw ase; } throw new RuntimeException("Stream " + myStreamName + " never went active"); } } }