Example usage for com.amazonaws.services.kinesis.model PutRecordRequest getPartitionKey

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model PutRecordRequest getPartitionKey.

Prototype

public String getPartitionKey() 

Source Link

Document

Determines which shard in the stream the data record is assigned to.

Usage

From source file:AmazonKinesisSample.java

License:Open Source License

public static void main(String[] args) throws Exception {
    init();//from  ww w.ja  v a  2s  . c  om

    final String myStreamName = "myFirstStream";
    final Integer myStreamSize = 1;

    // Create a stream. The number of shards determines the provisioned throughput.

    CreateStreamRequest createStreamRequest = new CreateStreamRequest();
    createStreamRequest.setStreamName(myStreamName);
    createStreamRequest.setShardCount(myStreamSize);

    kinesisClient.createStream(createStreamRequest);
    // The stream is now being created.
    LOG.info("Creating Stream : " + myStreamName);
    waitForStreamToBecomeAvailable(myStreamName);

    // list all of my streams
    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    listStreamsRequest.setLimit(10);
    ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
    List<String> streamNames = listStreamsResult.getStreamNames();
    while (listStreamsResult.isHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }

        listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
        streamNames.addAll(listStreamsResult.getStreamNames());

    }
    LOG.info("Printing my list of streams : ");

    // print all of my streams.
    if (!streamNames.isEmpty()) {
        System.out.println("List of my streams: ");
    }
    for (int i = 0; i < streamNames.size(); i++) {
        System.out.println(streamNames.get(i));
    }

    LOG.info("Putting records in stream : " + myStreamName);
    // Write 10 records to the stream
    for (int j = 0; j < 10; j++) {
        PutRecordRequest putRecordRequest = new PutRecordRequest();
        putRecordRequest.setStreamName(myStreamName);
        putRecordRequest.setData(ByteBuffer.wrap(String.format("testData-%d", j).getBytes()));
        putRecordRequest.setPartitionKey(String.format("partitionKey-%d", j));
        PutRecordResult putRecordResult = kinesisClient.putRecord(putRecordRequest);
        System.out.println("Successfully putrecord, partition key : " + putRecordRequest.getPartitionKey()
                + ", ShardID : " + putRecordResult.getShardId());
    }

    // Delete the stream.
    LOG.info("Deleting stream : " + myStreamName);
    DeleteStreamRequest deleteStreamRequest = new DeleteStreamRequest();
    deleteStreamRequest.setStreamName(myStreamName);

    kinesisClient.deleteStream(deleteStreamRequest);
    // The stream is now being deleted.
    LOG.info("Stream is now being deleted : " + myStreamName);
}

---

From source file:kinesisAlertAnalysis.java

License:Open Source License

public static void main(String[] args) throws Exception {

    init();/*from  w w w .  ja va 2 s.c  om*/

    final String myStreamName = "alertsStream";
    final Integer myStreamSize = 1;

    /*
     * The ProfileCredentialsProvider will return your [awsReilly]
     * credential profile by reading from the credentials file located at
     * (/Users/johnreilly/.aws/credentials).
     */
    AWSCredentials credentials = null;
    try {
        credentials = new ProfileCredentialsProvider("jreilly").getCredentials();
    } catch (Exception e) {
        throw new AmazonClientException("Cannot load the credentials from the credential profiles file. "
                + "Please make sure that your credentials file is at the correct "
                + "location (/Users/johnreilly/.aws/credentials), and is in valid format.", e);
    }

    // Describe the stream and check if it exists.
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(myStreamName);
    try {
        StreamDescription streamDescription = kinesis.describeStream(describeStreamRequest)
                .getStreamDescription();
        System.out.printf("Stream %s has a status of %s.\n", myStreamName, streamDescription.getStreamStatus());

        if ("DELETING".equals(streamDescription.getStreamStatus())) {
            System.out.println("Stream is being deleted. This sample will now exit.");
            System.exit(0);
        }

        // Wait for the stream to become active if it is not yet ACTIVE.
        if (!"ACTIVE".equals(streamDescription.getStreamStatus())) {
            waitForStreamToBecomeAvailable(myStreamName);
        }
    } catch (ResourceNotFoundException ex) {
        System.out.printf("Stream %s does not exist. Creating it now.\n", myStreamName);

        // Create a stream. The number of shards determines the provisioned throughput.
        CreateStreamRequest createStreamRequest = new CreateStreamRequest();
        createStreamRequest.setStreamName(myStreamName);
        createStreamRequest.setShardCount(myStreamSize);
        kinesis.createStream(createStreamRequest);
        // The stream is now being created. Wait for it to become active.
        waitForStreamToBecomeAvailable(myStreamName);
    }

    // List all of my streams.
    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    listStreamsRequest.setLimit(10);
    ListStreamsResult listStreamsResult = kinesis.listStreams(listStreamsRequest);
    List<String> streamNames = listStreamsResult.getStreamNames();
    while (listStreamsResult.isHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }

        listStreamsResult = kinesis.listStreams(listStreamsRequest);
        streamNames.addAll(listStreamsResult.getStreamNames());
    }
    // Print all of my streams.
    System.out.println("List of my streams: ");
    for (int i = 0; i < streamNames.size(); i++) {
        System.out.println("\t- " + streamNames.get(i));
    }

    AmazonSQS sqs = new AmazonSQSClient(credentials);
    Region usEast1 = Region.getRegion(Regions.US_EAST_1);
    sqs.setRegion(usEast1);

    System.out.println("");
    System.out.println("===========================================");
    System.out.println("Getting Started with sqsAlertCache");
    System.out.println("===========================================\n");

    try {

        String thisQueue = "alertCache";
        String nextQueue = "alertReceive";

        // Receive messages
        System.out.println("Receiving messages from " + thisQueue + ".");
        ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(thisQueue);
        List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages();
        System.out.println("Message count for " + thisQueue + ": " + messages.size() + "\n");

        for (Message message : messages) {

            System.out.println("  Message");
            System.out.println("    MessageId:     " + message.getMessageId());
            System.out.println("    ReceiptHandle: " + message.getReceiptHandle());
            System.out.println("    MD5OfBody:     " + message.getMD5OfBody());
            System.out.println("    Body:          " + message.getBody());
            for (Entry<String, String> entry : message.getAttributes().entrySet()) {
                System.out.println("  Attribute");
                System.out.println("    Name:  " + entry.getKey());
                System.out.println("    Value: " + entry.getValue());
            }
            System.out.println();

            // Write record to the stream
            long createTime = System.currentTimeMillis();
            PutRecordRequest putRecordRequest = new PutRecordRequest();
            putRecordRequest.setStreamName(myStreamName);
            putRecordRequest.setData(ByteBuffer.wrap(String.format(message.getBody(), createTime).getBytes()));
            putRecordRequest.setPartitionKey(String.format("partitionKey-%d", createTime));
            PutRecordResult putRecordResult = kinesis.putRecord(putRecordRequest);
            System.out.printf(
                    "Successfully put record, partition key : %s, ShardID : %s, SequenceNumber : %s.\n",
                    putRecordRequest.getPartitionKey(), putRecordResult.getShardId(),
                    putRecordResult.getSequenceNumber());

            // then send message to cache queue
            System.out.println("Sending messages to next queue.");
            sqs.sendMessage(new SendMessageRequest(nextQueue, message.getBody()));

            // delete message after sending to persist queue
            System.out.println("Deleting message from this queue.\n");
            String messageRecieptHandle = message.getReceiptHandle();
            sqs.deleteMessage(new DeleteMessageRequest(thisQueue, messageRecieptHandle));
        }

    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException, which means your request made it "
                + "to Amazon SQS, but was rejected with an error response for some reason.");
        System.out.println("Error Message:    " + ase.getMessage());
        System.out.println("HTTP Status Code: " + ase.getStatusCode());
        System.out.println("AWS Error Code:   " + ase.getErrorCode());
        System.out.println("Error Type:       " + ase.getErrorType());
        System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println("Caught an AmazonClientException, which means the client encountered "
                + "a serious internal problem while trying to communicate with SQS, such as not "
                + "being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    }
}

---

From source file:sqsAlertStream.java

License:Open Source License

public static void main(String[] args) throws Exception {

    // get credentials
    String user = "jreilly";
    AWSCredentials credentials = whgHelper.getCred(user);

    // use credentials to set access to SQS
    AmazonSQS sqs = whgHelper.setQueueAccess(credentials);

    // define queue that messages will be retrieved from
    String thisQueue = "alertStream";
    String nextQueue = "alertErrorHandling";

    // set access to stream instance
    kinesis = new AmazonKinesisClient(credentials);

    final String streamName = "alertsStream";
    final Integer streamSize = 1;

    while (1 > 0) {

        // pull list of current messages (up to 10) in the queue
        List<Message> messages = whgHelper.getMessagesFromQueue(thisQueue, sqs);
        System.out.println("Count of messages in " + thisQueue + ": " + messages.size());

        try {/*from w ww  .  j a v a2  s  . c o  m*/

            for (Message message : messages) {

                whgHelper.printMessage(message);
                for (Entry<String, String> entry : message.getAttributes().entrySet()) {
                    whgHelper.printMessageEntry(entry);
                }

                // Write record to the stream
                long createTime = System.currentTimeMillis();
                PutRecordRequest putRecordRequest = new PutRecordRequest();
                putRecordRequest.setStreamName(streamName);
                putRecordRequest
                        .setData(ByteBuffer.wrap(String.format(message.getBody(), createTime).getBytes()));
                putRecordRequest.setPartitionKey(String.format("partitionKey-%d", createTime));
                PutRecordResult putRecordResult = kinesis.putRecord(putRecordRequest);
                System.out.printf(
                        "Successfully put record, partition key : %s, ShardID : %s, SequenceNumber : %s.\n",
                        putRecordRequest.getPartitionKey(), putRecordResult.getShardId(),
                        putRecordResult.getSequenceNumber());

                // then send message to cache queue
                System.out.println("Sending messages to next queue.");
                sqs.sendMessage(new SendMessageRequest(nextQueue, message.getBody()));

                // delete message after sending to persist queue
                System.out.println("Deleting message from this queue.\n");
                String messageRecieptHandle = message.getReceiptHandle();
                sqs.deleteMessage(new DeleteMessageRequest(thisQueue, messageRecieptHandle));
            }
            Thread.sleep(20000); // do nothing for 1000 miliseconds (1 second)

        } catch (AmazonServiceException ase) {
            whgHelper.errorMessagesAse(ase);
        } catch (AmazonClientException ace) {
            whgHelper.errorMessagesAce(ace);
        }
    }
}

---

From source file:AmazonKinesisCreate.java

License:Open Source License

public static void main(String[] args) throws Exception {
    init();//from  w w  w  . j a  v  a2 s.co  m

    final String myStreamName = "philsteststream";
    final Integer myStreamSize = 1;

    // Create a stream. The number of shards determines the provisioned throughput.

    CreateStreamRequest createStreamRequest = new CreateStreamRequest();
    createStreamRequest.setStreamName(myStreamName);
    createStreamRequest.setShardCount(myStreamSize);

    // pt
    kinesisClient.createStream(createStreamRequest);

    // The stream is now being created.
    LOG.info("Creating Stream : " + myStreamName);
    waitForStreamToBecomeAvailable(myStreamName);

    // list all of my streams
    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    listStreamsRequest.setLimit(10);
    ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
    List<String> streamNames = listStreamsResult.getStreamNames();
    while (listStreamsResult.isHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }

        listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
        streamNames.addAll(listStreamsResult.getStreamNames());

    }
    LOG.info("Printing my list of streams : ");

    // print all of my streams.
    if (!streamNames.isEmpty()) {
        System.out.println("List of my streams: ");
    }
    for (int i = 0; i < streamNames.size(); i++) {
        System.out.println(streamNames.get(i));
    }

    LOG.info("Putting records in stream : " + myStreamName);
    // Write 10 records to the stream
    for (int j = 0; j < 10; j++) {

        try {
            PutRecordRequest putRecordRequest = new PutRecordRequest();
            putRecordRequest.setStreamName(myStreamName);
            putRecordRequest.setData(ByteBuffer.wrap(String.format("testData-%d", j).getBytes()));
            putRecordRequest.setPartitionKey(String.format("partitionKey-%d", j));
            PutRecordResult putRecordResult = kinesisClient.putRecord(putRecordRequest);
            System.out.println("Successfully putrecord, partition key : " + putRecordRequest.getPartitionKey()
                    + ", ShardID : " + putRecordResult.getShardId());
            Thread.sleep(1000);

        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    // Delete the stream.

    /*
    LOG.info("Deleting stream : " + myStreamName);
    DeleteStreamRequest deleteStreamRequest = new DeleteStreamRequest();
    deleteStreamRequest.setStreamName(myStreamName);
            
    kinesisClient.deleteStream(deleteStreamRequest);
    // The stream is now being deleted.
    LOG.info("Stream is now being deleted : " + myStreamName);
            
    LOG.info("Streaming completed" + myStreamName);
    */

}

---

From source file:KinesisStreamDataProducer.java

License:Open Source License

public static void main(String[] args) throws Exception {
    init();/*from  w ww  .  j a  va  2s  . c  o m*/

    final String myStreamName = "sparkStream";

    logger.info("Putting records in stream : " + myStreamName);
    // Write records to the stream
    for (int j = 0; j < 100; j++) {
        PutRecordRequest putRecordRequest = new PutRecordRequest();
        putRecordRequest.setStreamName(myStreamName);
        putRecordRequest.setData(
                ByteBuffer.wrap(String.format("testData-%d testData-%d testData-%d", j, j, j).getBytes()));
        putRecordRequest.setPartitionKey(String.format("partitionKey-%d", j));
        PutRecordResult putRecordResult = kinesisClient.putRecord(putRecordRequest);
        System.out.println("Successfully putrecord, partition key : " + putRecordRequest.getPartitionKey()
                + ", ShardID : " + putRecordResult.getShardId());
    }
}

---

From source file:AmazonKinesisRecordProducerSample.java

License:Open Source License

public static void main(String[] args) throws Exception {
    init();//from  w w w  .  jav  a  2s  .  c  om

    final String myStreamName = AmazonKinesisApplicationSample.SAMPLE_APPLICATION_STREAM_NAME;
    final Integer myStreamSize = 1;

    // Describe the stream and check if it exists.
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(myStreamName);
    try {
        StreamDescription streamDescription = kinesis.describeStream(describeStreamRequest)
                .getStreamDescription();
        System.out.printf("Stream %s has a status of %s.\n", myStreamName, streamDescription.getStreamStatus());

        if ("DELETING".equals(streamDescription.getStreamStatus())) {
            System.out.println("Stream is being deleted. This sample will now exit.");
            System.exit(0);
        }

        // Wait for the stream to become active if it is not yet ACTIVE.
        if (!"ACTIVE".equals(streamDescription.getStreamStatus())) {
            waitForStreamToBecomeAvailable(myStreamName);
        }
    } catch (ResourceNotFoundException ex) {
        System.out.printf("Stream %s does not exist. Creating it now.\n", myStreamName);

        // Create a stream. The number of shards determines the provisioned throughput.
        CreateStreamRequest createStreamRequest = new CreateStreamRequest();
        createStreamRequest.setStreamName(myStreamName);
        createStreamRequest.setShardCount(myStreamSize);
        kinesis.createStream(createStreamRequest);
        // The stream is now being created. Wait for it to become active.
        waitForStreamToBecomeAvailable(myStreamName);
    }

    // List all of my streams.
    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    listStreamsRequest.setLimit(10);
    ListStreamsResult listStreamsResult = kinesis.listStreams(listStreamsRequest);
    List<String> streamNames = listStreamsResult.getStreamNames();
    while (listStreamsResult.isHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }

        listStreamsResult = kinesis.listStreams(listStreamsRequest);
        streamNames.addAll(listStreamsResult.getStreamNames());
    }
    // Print all of my streams.
    System.out.println("List of my streams: ");
    for (int i = 0; i < streamNames.size(); i++) {
        System.out.println("\t- " + streamNames.get(i));
    }

    System.out.printf("Putting records in stream : %s until this application is stopped...\n", myStreamName);
    System.out.println("Press CTRL-C to stop.");
    // Write records to the stream until this program is aborted.
    while (true) {
        long createTime = System.currentTimeMillis();
        PutRecordRequest putRecordRequest = new PutRecordRequest();
        putRecordRequest.setStreamName(myStreamName);
        putRecordRequest.setData(ByteBuffer.wrap(String.format("testData-%d", createTime).getBytes()));
        putRecordRequest.setPartitionKey(String.format("partitionKey-%d", createTime));
        PutRecordResult putRecordResult = kinesis.putRecord(putRecordRequest);
        System.out.printf("Successfully put record, partition key : %s, ShardID : %s, SequenceNumber : %s.\n",
                putRecordRequest.getPartitionKey(), putRecordResult.getShardId(),
                putRecordResult.getSequenceNumber());
    }
}

---

From source file:com.boundary.aws.kinesis.Sample.java

License:Open Source License

public static void main(String[] args) throws Exception {
    init();//from  w w  w  .  ja  v a 2  s.  c  o m

    final String myStreamName = "boundary-test-stream";
    final Integer myStreamSize = 1;

    // Create a stream. The number of shards determines the provisioned
    // throughput.

    CreateStreamRequest createStreamRequest = new CreateStreamRequest();
    createStreamRequest.setStreamName(myStreamName);
    createStreamRequest.setShardCount(myStreamSize);

    kinesisClient.createStream(createStreamRequest);
    // The stream is now being created.
    LOG.info("Creating Stream : " + myStreamName);
    waitForStreamToBecomeAvailable(myStreamName);

    // list all of my streams
    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    listStreamsRequest.setLimit(10);
    ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
    List<String> streamNames = listStreamsResult.getStreamNames();
    while (listStreamsResult.isHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }

        listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
        streamNames.addAll(listStreamsResult.getStreamNames());

    }
    LOG.info("Printing my list of streams : ");

    // print all of my streams.
    if (!streamNames.isEmpty()) {
        System.out.println("List of my streams: ");
    }
    for (int i = 0; i < streamNames.size(); i++) {
        System.out.println(streamNames.get(i));
    }

    LOG.info("Putting records in stream : " + myStreamName);
    // Write 10 records to the stream
    for (int j = 0; j < 100; j++) {
        PutRecordRequest putRecordRequest = new PutRecordRequest();
        putRecordRequest.setStreamName(myStreamName);
        putRecordRequest.setData(ByteBuffer.wrap(String.format("testData-%d", j).getBytes()));
        putRecordRequest.setPartitionKey(String.format("partitionKey-%d", j));
        PutRecordResult putRecordResult = kinesisClient.putRecord(putRecordRequest);
        System.out.println("Successfully putrecord, partition key : " + putRecordRequest.getPartitionKey()
                + ", ShardID : " + putRecordResult.getShardId());
    }

    // Delete the stream.
    LOG.info("Deleting stream : " + myStreamName);
    DeleteStreamRequest deleteStreamRequest = new DeleteStreamRequest();
    deleteStreamRequest.setStreamName(myStreamName);

    kinesisClient.deleteStream(deleteStreamRequest);
    // The stream is now being deleted.
    LOG.info("Stream is now being deleted : " + myStreamName);
}

---

From source file:com.facebook.presto.kinesis.util.MockKinesisClient.java

License:Apache License

@Override
public PutRecordResult putRecord(PutRecordRequest putRecordRequest)
        throws AmazonServiceException, AmazonClientException {
    // Setup method to add a new record:
    InternalStream theStream = this.getStream(putRecordRequest.getStreamName());
    if (theStream != null) {
        PutRecordResult result = theStream.putRecord(putRecordRequest.getData(),
                putRecordRequest.getPartitionKey());
        return result;
    } else {//w w  w .j a  v a  2 s.  co m
        throw new AmazonClientException("This stream does not exist!");
    }
}

---

From source file:org.selman.tweetamo.TweetamoClient.java

License:Apache License

public static void main(String[] args) throws Exception {

    if (args.length != 2) {
        System.out.println("Usage: [language] [search topic]");
    }// w  ww . j  a va  2 s .  com

    kinesisClient = new AmazonKinesisClient(new ClasspathPropertiesFileCredentialsProvider());
    waitForStreamToBecomeAvailable(STREAM_NAME);

    LOG.info("Publishing tweets to stream : " + STREAM_NAME);
    StatusListener listener = new StatusListener() {
        public void onStatus(Status status) {
            try {
                PutRecordRequest putRecordRequest = new PutRecordRequest();
                putRecordRequest.setStreamName(STREAM_NAME);
                putRecordRequest.setData(TweetSerializer.toBytes(status));
                putRecordRequest.setPartitionKey(status.getUser().getScreenName());
                PutRecordResult putRecordResult = kinesisClient.putRecord(putRecordRequest);
                LOG.info("Successfully putrecord, partition key : " + putRecordRequest.getPartitionKey()
                        + ", ShardID : " + putRecordResult.getShardId());

            } catch (Exception e) {
                LOG.error("Failed to putrecord", e);
            }
        }

        public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
        }

        public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
        }

        public void onException(Exception ex) {
            ex.printStackTrace();
        }

        @Override
        public void onScrubGeo(long arg0, long arg1) {
        }

        @Override
        public void onStallWarning(StallWarning arg0) {
        }
    };

    ClasspathTwitterCredentialsProvider provider = new ClasspathTwitterCredentialsProvider();
    TwitterCredentials credentials = provider.getTwitterCredentials();

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true).setOAuthConsumerKey(credentials.getConsumerKey())
            .setOAuthConsumerSecret(credentials.getConsumerSecret())
            .setOAuthAccessToken(credentials.getAccessToken())
            .setOAuthAccessTokenSecret(credentials.getAccessTokenSecret());
    TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
    twitterStream.addListener(listener);
    FilterQuery filterQuery = new FilterQuery();
    filterQuery.language(new String[] { args[0] });
    filterQuery.track(new String[] { args[1] });
    twitterStream.filter(filterQuery);
}

---