List of usage examples for com.amazonaws.services.sqs.model Message getBody
public String getBody()
The message's contents (not URL-encoded).
From source file:com.plumbee.flume.source.sqs.BatchConsumer.java
License:Apache License
public Status process() throws EventDeliveryException { // Check if we've met the criteria to flush events. if (batchDeleteRequestEntries.size() >= batchSize) { flush();/*from w w w. ja v a 2s. c o m*/ } else if ((flushInterval > 0) && ((System.currentTimeMillis() - lastFlush) > flushInterval)) { flush(); } // The number of messages pending insertion to the channels should // always by the same as the number of messages to delete from SQS! assert (batchEventList.size() == batchDeleteRequestEntries.size()); // Determine the maximum number of messages to request from SQS. We // never exceed the capacity of the internal buffers. if ((batchDeleteRequestEntries.size() + queueRecvBatchSize) > batchSize) { receiveMessageRequest.setMaxNumberOfMessages(batchSize - batchDeleteRequestEntries.size()); } else { receiveMessageRequest.setMaxNumberOfMessages(queueRecvBatchSize); } // Retrieve messages. List<Message> messages = client.receiveMessage(receiveMessageRequest).getMessages(); sourceCounter.incrementBatchReceiveRequestAttemptCount(); for (Message message : messages) { // Extract SQS message attributes. long sentTimestamp = Long.parseLong(message.getAttributes().get(SQS_ATTR_SENTTIMESTAMP)); long approximateReceiveCount = Long.parseLong(message.getAttributes().get(SQS_ATTR_APPROXRECEIVECOUNT)); // Update statistics. if (approximateReceiveCount > 1) { sourceCounter.incrementEventReprocessedCount(); } // By default the timestamp of the message is set to the // timestamp in UTC that the message was added to the SQS queue as // opposed to the time it was extracted. Map<String, String> headers = new HashMap<String, String>(); headers.put("timestamp", String.valueOf(sentTimestamp)); batchEventList.add(EventBuilder.withBody(message.getBody(), Charsets.UTF_8, headers)); batchDeleteRequestEntries.add(new DeleteMessageBatchRequestEntry() .withId(Long.toString(batchEventList.size())).withReceiptHandle(message.getReceiptHandle())); } sourceCounter.incrementBatchReceiveRequestSuccessCount(); sourceCounter.addToEventReceivedCount((long) messages.size()); // If the payload was less than 90% of the maximum batch size, // instruct the runner to throttle polling. if (messages.size() < (queueRecvBatchSize * 0.9)) { return Status.BACKOFF; } return Status.READY; }
From source file:com.rodosaenz.samples.aws.sqs.AwsSqsSimpleExample.java
License:Open Source License
public static void main(String[] args) throws Exception { /*/* www . j a v a 2s . co m*/ * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (~/.aws/credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider().getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (~/.aws/credentials), and is in valid format.", e); } AmazonSQS sqs = new AmazonSQSClient(credentials); Region usWest2 = Region.getRegion(Regions.US_EAST_1); sqs.setRegion(usWest2); System.out.println("==========================================="); System.out.println("Getting Started with Amazon SQS"); System.out.println("===========================================\n"); String queue_name = "com-rodosaenz-examples-aws-sqs"; try { // Create a queue System.out.println("Creating a new SQS queue called " + queue_name + ".\n"); CreateQueueRequest createQueueRequest = new CreateQueueRequest(queue_name); String myQueueUrl = sqs.createQueue(createQueueRequest).getQueueUrl(); // List queues System.out.println("Listing all queues in your account.\n"); for (String queueUrl : sqs.listQueues().getQueueUrls()) { System.out.println(" QueueUrl: " + queueUrl); } System.out.println(); // Send a message System.out.println("Sending a message to " + queue_name + ".\n"); sqs.sendMessage(new SendMessageRequest(myQueueUrl, "This is my message text.")); // Receive messages System.out.println("Receiving messages from MyQueue.\n"); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myQueueUrl); receiveMessageRequest.setMaxNumberOfMessages(1); List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); for (Message message : messages) { System.out.println(" Message"); System.out.println(" MessageId: " + message.getMessageId()); System.out.println(" ReceiptHandle: " + message.getReceiptHandle()); System.out.println(" MD5OfBody: " + message.getMD5OfBody()); System.out.println(" Body: " + message.getBody()); for (Entry<String, String> entry : message.getAttributes().entrySet()) { System.out.println(" Attribute"); System.out.println(" Name: " + entry.getKey()); System.out.println(" Value: " + entry.getValue()); } } System.out.println(); // Delete a message System.out.println("Deleting a message.\n"); String messageReceiptHandle = messages.get(0).getReceiptHandle(); sqs.deleteMessage(new DeleteMessageRequest(myQueueUrl, messageReceiptHandle)); //Get attributes GetQueueAttributesRequest request = new GetQueueAttributesRequest(myQueueUrl).withAttributeNames("All"); final Map<String, String> attributes = sqs.getQueueAttributes(request).getAttributes(); System.out.println(" Policy: " + attributes.get("Policy")); System.out.println(" MessageRetentionPeriod: " + attributes.get("MessageRetentionPeriod")); System.out.println(" MaximumMessageSize: " + attributes.get("MaximumMessageSize")); System.out.println(" CreatedTimestamp: " + attributes.get("CreatedTimestamp")); System.out.println(" VisibilityTimeout: " + attributes.get("VisibilityTimeout")); System.out.println(" QueueArn: " + attributes.get("QueueArn")); System.out.println(" ApproximateNumberOfMessages: " + attributes.get("ApproximateNumberOfMessages")); System.out.println(" ApproximateNumberOfMessagesNotVisible: " + attributes.get("ApproximateNumberOfMessagesNotVisible")); System.out.println(" DelaySeconds: " + attributes.get("DelaySeconds")); // Delete a queue System.out.println("Deleting the test queue.\n"); sqs.deleteQueue(new DeleteQueueRequest(myQueueUrl)); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon SQS, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with SQS, such as not " + "being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.sag.tn.storm.stormmaven.spoutsources.AWSSQSSpoutSource.java
License:Open Source License
@Override public List<TuplePair<VTDNav, String>> fetch() throws IOException { List<TuplePair<VTDNav, String>> list = new ArrayList<>(); /*//w ww.j a va2s. co m * get duplicate check - ignore as it needs a centralized cache process delete */ try { List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); for (Message message : messages) { this.vg.setDoc(message.getBody().getBytes()); this.vg.parse(true); list.add(new TuplePair<>(vg.getNav(), message.getMessageId())); sqs.deleteMessage(new DeleteMessageRequest(this.sqsQueueURL, message.getReceiptHandle())); this.vg.clear(); } } catch (Exception e) { e.printStackTrace(); } return list; }
From source file:com.sjsu.cmpe281.team06.NovaMiaas.SimpleQueueServiceSample.java
License:Open Source License
public static void main(String[] args) throws Exception { /*/* w w w . j av a 2s. co m*/ * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonSQS sqs = new AmazonSQSClient(new ClasspathPropertiesFileCredentialsProvider()); Region usWest1 = Region.getRegion(Regions.US_WEST_1); sqs.setRegion(usWest1); System.out.println("==========================================="); System.out.println("Getting Started with Amazon SQS"); System.out.println("===========================================\n"); try { // Create a queue System.out.println("Creating a new SQS queue called MyQueue.\n"); CreateQueueRequest createQueueRequest = new CreateQueueRequest("MyQueue"); String myQueueUrl = sqs.createQueue(createQueueRequest).getQueueUrl(); // List queues System.out.println("Listing all queues in your account.\n"); for (String queueUrl : sqs.listQueues().getQueueUrls()) { System.out.println(" QueueUrl: " + queueUrl); } System.out.println(); // Send a message System.out.println("Sending a message to MyQueue.\n"); sqs.sendMessage(new SendMessageRequest(myQueueUrl, "This is my new message text.")); // Receive messages System.out.println("Receiving messages from MyQueue.\n"); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myQueueUrl); List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); for (Message message : messages) { System.out.println(" Message"); System.out.println(" MessageId: " + message.getMessageId()); System.out.println(" ReceiptHandle: " + message.getReceiptHandle()); System.out.println(" MD5OfBody: " + message.getMD5OfBody()); System.out.println(" Body: " + message.getBody()); for (Entry<String, String> entry : message.getAttributes().entrySet()) { System.out.println(" Attribute"); System.out.println(" Name: " + entry.getKey()); System.out.println(" Value: " + entry.getValue()); } } System.out.println(); // Delete a message System.out.println("Deleting a message.\n"); String messageRecieptHandle = messages.get(0).getReceiptHandle(); sqs.deleteMessage(new DeleteMessageRequest(myQueueUrl, messageRecieptHandle)); // Delete a queue System.out.println("Deleting the test queue.\n"); sqs.deleteQueue(new DeleteQueueRequest(myQueueUrl)); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon SQS, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with SQS, such as not " + "being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.smoketurner.pipeline.application.core.MessageProcessor.java
License:Apache License
/** * Process an SQS {@link Message} by parsing the SNS notification out of the * message body. Then download the S3 object out of the SNS notification, * decompress the object, then broadcast each event. * /*from www.j a v a 2s .c o m*/ * @param message * SQS message * @return true if the file was fully processed (and the message can be * deleted from SQS), otherwise false. */ @Override public boolean test(@Nullable final Message message) { if (message == null) { return false; } if (LOGGER.isTraceEnabled()) { LOGGER.trace("Received SQS message: {}", message); } else if (LOGGER.isDebugEnabled()) { LOGGER.debug("Received SQS message: {}", message.getMessageId()); } if (broadcaster.isEmpty()) { LOGGER.debug("No connections found, skipping SQS message processing"); return false; } final AmazonSNSNotification notification; try { notification = MAPPER.readValue(message.getBody(), AmazonSNSNotification.class); } catch (IOException e) { LOGGER.error("Failed to parse SNS notification, deleting SQS message", e); return true; } LOGGER.debug("SNS notification created at: {} ({} behind)", notification.getTimestamp(), notification.getDelayDuration()); // if we don't have a valid SNS notification, try parsing the body as S3 // event records final String body; if (notification.isValid()) { body = notification.getMessage(); } else { body = message.getBody(); } final S3EventNotification records; try { records = S3EventNotification.parseJson(body); } catch (AmazonClientException e) { LOGGER.error("Failed to parse S3 event records, deleting SQS message", e); return true; } final int recordCount = records.getRecords().size(); recordCounts.update(recordCount); LOGGER.debug("Parsed {} S3 event records from SNS notification", recordCount); if (recordCount < 1) { LOGGER.debug("No S3 event records found in SNS notification, deleting SQS message"); return true; } int recordsProcessed = 0; for (S3EventNotificationRecord record : records.getRecords()) { if (broadcaster.isEmpty()) { LOGGER.debug("No connections found, not downloading from S3"); return false; } if (processRecord(record)) { recordsProcessed++; } } // if we've processed all of the records, which includes skipping over // empty S3 files, the message has been fully processed. if (recordsProcessed == recordCount) { LOGGER.debug("Processed {} of {} records, deleting SQS message", recordsProcessed, recordCount); return true; } LOGGER.debug("Processed {} of {} records, not deleting SQS message: {}", recordsProcessed, recordCount, message.getMessageId()); return false; }
From source file:com.springboot.jms.AWSJmsSendAndReceive.java
License:Open Source License
public static void testAWSQueue() throws Exception { /*/*from w ww . j a v a2 s .c o m*/ * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (~/.aws/credentials). */ /* AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider().getCredentials(); } catch (Exception e) { throw new AmazonClientException( "Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (~/.aws/credentials), and is in valid format.", e); } AmazonSQS sqs = new AmazonSQSClient(credentials);*/ AmazonSQS sqs = new AmazonSQSClient(new BasicAWSCredentials("", "")); Region usWest2 = Region.getRegion(Regions.US_EAST_1); sqs.setRegion(usWest2); System.out.println("==========================================="); System.out.println("Getting Started with Amazon SQS"); System.out.println("===========================================\n"); try { /* // Create a queue System.out.println("Creating a new SQS queue called MyQueue.\n"); CreateQueueRequest createQueueRequest = new CreateQueueRequest("PetQueue"); String myQueueUrl = sqs.createQueue(createQueueRequest).getQueueUrl();*/ String myQueueUrl = "https://sqs.us-east-1.amazonaws.com/918558451804/PetQueue"; // List queues System.out.println("Listing all queues in your account.\n"); for (String queueUrl : sqs.listQueues().getQueueUrls()) { System.out.println(" QueueUrl: " + queueUrl); } System.out.println(); // Send a message System.out.println("Sending a message to MyQueue.\n"); sqs.sendMessage(new SendMessageRequest(myQueueUrl, "This is my message text.")); // Receive messages System.out.println("Receiving messages from MyQueue.\n"); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myQueueUrl); List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); for (Message message : messages) { System.out.println(" Message"); System.out.println(" MessageId: " + message.getMessageId()); System.out.println(" ReceiptHandle: " + message.getReceiptHandle()); System.out.println(" MD5OfBody: " + message.getMD5OfBody()); System.out.println(" Body: " + message.getBody()); for (Entry<String, String> entry : message.getAttributes().entrySet()) { System.out.println(" Attribute"); System.out.println(" Name: " + entry.getKey()); System.out.println(" Value: " + entry.getValue()); } } System.out.println(); // Delete a message System.out.println("Deleting a message.\n"); String messageReceiptHandle = messages.get(0).getReceiptHandle(); sqs.deleteMessage(new DeleteMessageRequest(myQueueUrl, messageReceiptHandle)); /* // Delete a queue System.out.println("Deleting the test queue.\n"); sqs.deleteQueue(new DeleteQueueRequest(myQueueUrl));*/ } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon SQS, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with SQS, such as not " + "being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.streamsets.pipeline.origin.sqs.SampleSource.java
License:Apache License
/** * {@inheritDoc}/* w ww .j ava2s .c o m*/ */ @Override public String produce(String lastSourceOffset, int maxBatchSize, BatchMaker batchMaker) throws StageException { // Offsets can vary depending on the data source. Here we use an integer as an example only. long nextSourceOffset = 0; if (lastSourceOffset != null) { nextSourceOffset = Long.parseLong(lastSourceOffset); } int numRecords = 0; // TODO: As the developer, implement your logic that reads from a data source in this method. // Create records and add to batch. Records must have a string id. This can include the source offset // or other metadata to help uniquely identify the record itself. AWSSQSUtil awssqsUtil = new AWSSQSUtil(getAccessKey(), getSecreteKey(), getQueueName(), getEndPoint()); String queuName = awssqsUtil.getQueueName(); String queueUrl = awssqsUtil.getQueueUrl(queuName); //maximum number of meesage that can be retrieve in one request int maxMessagCount = 10; List<Message> messages = awssqsUtil.getMessagesFromQueue(queueUrl, maxMessagCount); for (Message message : messages) { Record record = getContext().createRecord("messageId::" + message.getMessageId()); Map<String, Field> map = new HashMap<>(); map.put("receipt_handle", Field.create(message.getReceiptHandle())); map.put("md5_of_body", Field.create(message.getMD5OfBody())); map.put("body", Field.create(message.getBody())); JSONObject attributeJson = new JSONObject(); for (Map.Entry<String, String> entry : message.getAttributes().entrySet()) { attributeJson.put(entry.getKey(), entry.getValue()); } map.put("attribute_list", Field.create(attributeJson.toString())); record.set(Field.create(map)); batchMaker.addRecord(record); ++nextSourceOffset; ++numRecords; if (getDeleteFlag()) { awssqsUtil.deleteMessageFromQueue(queueUrl, message); } } return String.valueOf(nextSourceOffset); }
From source file:com.streamsets.pipeline.stage.origin.sqs.SqsConsumerWorkerCallable.java
License:Apache License
@Override public Exception call() throws Exception { Exception terminatingException = null; final AmazonSQSAsync asyncConsumer = sqsAsync; final ArrayBlockingQueue<String> urlQueue = new ArrayBlockingQueue(queueUrlToNamePrefix.size(), false, queueUrlToNamePrefix.keySet()); while (!context.isStopped() && terminatingException == null) { String nextQueueUrl = null; try {/*from www . ja v a 2s .c o m*/ nextQueueUrl = urlQueue.take(); final ReceiveMessageRequest receiveRequest = new ReceiveMessageRequest() .withMaxNumberOfMessages(numMessagesPerRequest).withQueueUrl(nextQueueUrl); if (pollWaitTimeSeconds > 0) { receiveRequest.setWaitTimeSeconds(pollWaitTimeSeconds); } if (messageAttributeNames.size() > 0) { receiveRequest.setMessageAttributeNames(messageAttributeNames); } Future<ReceiveMessageResult> resultFuture = asyncConsumer.receiveMessageAsync(receiveRequest); ReceiveMessageResult result = resultFuture.get(); for (Message message : result.getMessages()) { final String recordId = getRecordId(message, nextQueueUrl); DataParser parser = null; try { parser = context.getService(DataFormatParserService.class).getParser(recordId, message.getBody()); Record record = null; do { try { record = parser.parse(); } catch (RecoverableDataParserException e) { // log the error and keep trying to parse this message LOG.error(Errors.SQS_04.getMessage(), e.getMessage(), e); terminatingException = new StageException(Errors.SQS_04, e.getMessage(), e); } catch (DataParserException e) { // log the error and stop trying to parse this message LOG.error(Errors.SQS_07.getMessage(), e.getMessage(), e); errorRecordHandler.onError(Errors.SQS_07, e.getMessage(), e); break; } } while (record == null); setSqsAttributesOnRecord(message, record, nextQueueUrl, queueUrlToNamePrefix.get(nextQueueUrl)); startBatchIfNeeded(); batchContext.getBatchMaker().addRecord(record); commitQueueUrlsToMessages.put(nextQueueUrl, message); if (++batchRecordCount > maxBatchSize) { cycleBatch(); } } catch (DataParserException e) { LOG.error(Errors.SQS_05.getMessage(), e.getMessage(), e); terminatingException = new StageException(Errors.SQS_05, e.getMessage(), e); break; } finally { if (parser != null) { parser.close(); } } } boolean batchMaxTimeElapsed = Clock.systemUTC().millis() > lastBatchStartTimestamp + maxBatchWaitTimeMs; if (batchMaxTimeElapsed) { cycleBatch(); } } catch (InterruptedException e) { LOG.error("InterruptedException trying to get SQS messages: {}", e.getMessage(), e); Thread.currentThread().interrupt(); break; } finally { if (nextQueueUrl != null) { urlQueue.put(nextQueueUrl); } } } flushBatch(); Optional.ofNullable(asyncConsumer).ifPresent(AmazonSQSAsync::shutdown); return terminatingException; }
From source file:com.streamsets.pipeline.stage.origin.sqs.SqsConsumerWorkerCallable.java
License:Apache License
private void setSqsAttributesOnRecord(Message message, Record record, String queueUrl, String queueNamePrefix) { final Record.Header header = record.getHeader(); switch (sqsAttributesOption) { case ALL:// w w w . j a va 2s. co m header.setAttribute(SQS_QUEUE_URL_ATTRIBUTE, queueUrl); Optional.of(message.getMessageAttributes()).ifPresent(attrs -> { attrs.forEach((name, val) -> { final String stringValue = val.getStringValue(); if (stringValue != null) { header.setAttribute(SQS_MESSAGE_ATTRIBUTE_PREFIX + name, stringValue); } }); }); final String body = message.getBody(); if (body != null) { header.setAttribute(SQS_MESSAGE_BODY_ATTRIBUTE, body); } final String bodyMd5 = message.getMD5OfBody(); if (bodyMd5 != null) { header.setAttribute(SQS_MESSAGE_BODY_MD5_ATTRIBUTE, bodyMd5); } final String attrsMd5 = message.getMD5OfMessageAttributes(); if (attrsMd5 != null) { header.setAttribute(SQS_MESSAGE_ATTRIBUTE_MD5_ATTRIBUTE, attrsMd5); } // fall through case BASIC: header.setAttribute(SQS_MESSAGE_ID_ATTRIBUTE, message.getMessageId()); header.setAttribute(SQS_QUEUE_NAME_PREFIX_ATTRIBUTE, queueNamePrefix); header.setAttribute(SQS_REGION_ATTRIBUTE, awsRegionLabel); break; case NONE: // empty block break; } }
From source file:com.threewks.thundr.deferred.provider.SqsQueueProvider.java
License:Apache License
@Override public List<String> receive() { ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(queueUrl); List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); List<String> received = new ArrayList<String>(); for (Message message : messages) { received.add(message.getBody()); // Delete message once read off queue sqs.deleteMessage(new DeleteMessageRequest(queueUrl, message.getReceiptHandle())); }/*from w w w .j av a 2 s. c om*/ return received; }