List of usage examples for com.amazonaws.services.sqs.model ReceiveMessageRequest setQueueUrl
public void setQueueUrl(String queueUrl)
The URL of the Amazon SQS queue from which messages are received.
From source file:com.comcast.cmb.test.tools.CMBTutorial.java
License:Apache License
public static void main(String[] args) { try {/*from w ww.j a v a 2 s.c om*/ Util.initLog4jTest(); //TODO: set user id and credentials for two distinct users // user "cqs_test_1" //BasicAWSCredentials user1Credentials = new BasicAWSCredentials("<access_key>", "<secret_key>"); BasicAWSCredentials user1Credentials = new BasicAWSCredentials("Z2DVBFRNZ2C2SSXDWS5F", "bH2UQiJkpctBaE3eaDob19fj5J9Q1FVafrZantBp"); // user "cqs_test_2" //String user2Id = "<user_id>"; String user2Id = "389653920093"; //BasicAWSCredentials user2Credentials = new BasicAWSCredentials("<access_key>", "<secret_key>"); BasicAWSCredentials user2Credentials = new BasicAWSCredentials("QL8Q1VOBCSJC5FZ2DMIU", "n6a82gyJZ04Z+Xqp7OgfqPtbbKqVc3UbuOTNrF+7"); // service urls //TODO: add service URLs //String cqsServerUrl = "http://<host>:<port>"; //String cnsServerUrl = "http://<host>:<port>"; String cqsServerUrl = "http://localhost:6059"; String cnsServerUrl = "http://localhost:6061"; // initialize service AmazonSQSClient sqs = new AmazonSQSClient(user1Credentials); sqs.setEndpoint(cqsServerUrl); AmazonSNSClient sns = new AmazonSNSClient(user2Credentials); sns.setEndpoint(cnsServerUrl); // create queue Random randomGenerator = new Random(); String queueName = QUEUE_PREFIX + randomGenerator.nextLong(); HashMap<String, String> attributeParams = new HashMap<String, String>(); CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName); createQueueRequest.setAttributes(attributeParams); String queueUrl = sqs.createQueue(createQueueRequest).getQueueUrl(); AddPermissionRequest addPermissionRequest = new AddPermissionRequest(); addPermissionRequest.setQueueUrl(queueUrl); addPermissionRequest.setActions(Arrays.asList("SendMessage")); addPermissionRequest.setLabel(UUID.randomUUID().toString()); addPermissionRequest.setAWSAccountIds(Arrays.asList(user2Id)); sqs.addPermission(addPermissionRequest); // create topic String topicName = "TSTT" + randomGenerator.nextLong(); CreateTopicRequest createTopicRequest = new CreateTopicRequest(topicName); CreateTopicResult createTopicResult = sns.createTopic(createTopicRequest); String topicArn = createTopicResult.getTopicArn(); // subscribe and confirm cqs endpoint SubscribeRequest subscribeRequest = new SubscribeRequest(); String queueArn = getArnForQueueUrl(queueUrl); subscribeRequest.setEndpoint(queueArn); subscribeRequest.setProtocol("cqs"); subscribeRequest.setTopicArn(topicArn); SubscribeResult subscribeResult = sns.subscribe(subscribeRequest); String subscriptionArn = subscribeResult.getSubscriptionArn(); if (subscriptionArn.equals("pending confirmation")) { Thread.sleep(500); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(); receiveMessageRequest.setQueueUrl(queueUrl); receiveMessageRequest.setMaxNumberOfMessages(1); ReceiveMessageResult receiveMessageResult = sqs.receiveMessage(receiveMessageRequest); List<Message> messages = receiveMessageResult.getMessages(); if (messages != null && messages.size() == 1) { JSONObject o = new JSONObject(messages.get(0).getBody()); if (!o.has("SubscribeURL")) { throw new Exception("message is not a confirmation messsage"); } String subscriptionUrl = o.getString("SubscribeURL"); httpGet(subscriptionUrl); DeleteMessageRequest deleteMessageRequest = new DeleteMessageRequest(); deleteMessageRequest.setReceiptHandle(messages.get(0).getReceiptHandle()); deleteMessageRequest.setQueueUrl(queueUrl); sqs.deleteMessage(deleteMessageRequest); } else { throw new Exception("no confirmation message found"); } } // publish and receive message PublishRequest publishRequest = new PublishRequest(); String messageText = "quamvis sint sub aqua, sub aqua maledicere temptant"; publishRequest.setMessage(messageText); publishRequest.setSubject("unit test message"); publishRequest.setTopicArn(topicArn); sns.publish(publishRequest); Thread.sleep(500); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(); receiveMessageRequest.setQueueUrl(queueUrl); receiveMessageRequest.setMaxNumberOfMessages(1); ReceiveMessageResult receiveMessageResult = sqs.receiveMessage(receiveMessageRequest); List<Message> messages = receiveMessageResult.getMessages(); if (messages != null && messages.size() == 1) { String messageBody = messages.get(0).getBody(); if (!messageBody.contains(messageText)) { throw new Exception("message text not found"); } DeleteMessageRequest deleteMessageRequest = new DeleteMessageRequest(); deleteMessageRequest.setReceiptHandle(messages.get(0).getReceiptHandle()); deleteMessageRequest.setQueueUrl(queueUrl); sqs.deleteMessage(deleteMessageRequest); } else { throw new Exception("no messages found"); } // subscribe and confirm http endpoint String id = randomGenerator.nextLong() + ""; String endPointUrl = CMBTestingConstants.HTTP_ENDPOINT_BASE_URL + "recv/" + id; String lastMessageUrl = CMBTestingConstants.HTTP_ENDPOINT_BASE_URL + "info/" + id + "?showLast=true"; subscribeRequest = new SubscribeRequest(); subscribeRequest.setEndpoint(endPointUrl); subscribeRequest.setProtocol("http"); subscribeRequest.setTopicArn(topicArn); subscribeResult = sns.subscribe(subscribeRequest); subscriptionArn = subscribeResult.getSubscriptionArn(); if (subscriptionArn.equals("pending confirmation")) { Thread.sleep(500); String response = httpGet(lastMessageUrl); JSONObject o = new JSONObject(response); if (!o.has("SubscribeURL")) { throw new Exception("message is not a confirmation messsage"); } String subscriptionUrl = o.getString("SubscribeURL"); response = httpGet(subscriptionUrl); } // publish and receive message publishRequest = new PublishRequest(); publishRequest.setMessage(messageText); publishRequest.setSubject("unit test message"); publishRequest.setTopicArn(topicArn); sns.publish(publishRequest); Thread.sleep(500); String response = httpGet(lastMessageUrl); if (response != null && response.length() > 0) { if (!response.contains(messageText)) { throw new Exception("message text not found"); } } else { throw new Exception("no messages found"); } // delete queue and topic DeleteTopicRequest deleteTopicRequest = new DeleteTopicRequest(topicArn); sns.deleteTopic(deleteTopicRequest); sqs.deleteQueue(new DeleteQueueRequest(queueUrl)); System.out.println("OK"); } catch (Exception ex) { ex.printStackTrace(); } }
From source file:com.eucalyptus.portal.SimpleQueueClientManager.java
License:Open Source License
public List<Message> receiveAllMessages(final String queueName, final boolean shouldDelete) throws Exception { try {//from ww w . j a v a 2 s . co m final int visibilityTimeout = 600; final int visibilityBuffer = 300; final long startTime = System.currentTimeMillis(); final List<Message> messages = Lists.newArrayList(); while ((System.currentTimeMillis() - startTime) < ((visibilityTimeout - visibilityBuffer) * 1000L)) { final ReceiveMessageRequest req = new ReceiveMessageRequest(); req.setQueueUrl(getQueueUrl(queueName)); req.setMaxNumberOfMessages(10); req.setWaitTimeSeconds(0); req.setVisibilityTimeout(visibilityTimeout); final ReceiveMessageResult result = getSimpleQueueClient().receiveMessage(req); final List<Message> received = result.getMessages(); if (received == null || received.size() <= 0) break; messages.addAll(received); } // TODO: Use PurgeQueue if (shouldDelete) { for (final List<Message> partition : Iterables.partition(messages, 10)) { final DeleteMessageBatchRequest delReq = new DeleteMessageBatchRequest(); delReq.setQueueUrl(getQueueUrl(queueName)); delReq.setEntries(partition.stream().map(m -> new DeleteMessageBatchRequestEntry() .withId(m.getMessageId()).withReceiptHandle(m.getReceiptHandle())) .collect(Collectors.toList())); getSimpleQueueClient().deleteMessageBatch(delReq); } } return messages; } catch (final AmazonServiceException ex) { throw new Exception("Failed to receive messages due to service error", ex); } catch (final AmazonClientException ex) { throw new Exception("Failed to receive messages due to client error", ex); } }
From source file:org.apache.nifi.processors.aws.sqs.GetSQS.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { final String queueUrl = context.getProperty(DYNAMIC_QUEUE_URL).evaluateAttributeExpressions().getValue(); final AmazonSQSClient client = getClient(); final ReceiveMessageRequest request = new ReceiveMessageRequest(); request.setAttributeNames(Collections.singleton("All")); request.setMessageAttributeNames(Collections.singleton("All")); request.setMaxNumberOfMessages(context.getProperty(BATCH_SIZE).asInteger()); request.setVisibilityTimeout(/*from ww w . j a v a 2 s . c o m*/ context.getProperty(VISIBILITY_TIMEOUT).asTimePeriod(TimeUnit.SECONDS).intValue()); request.setQueueUrl(queueUrl); request.setWaitTimeSeconds( context.getProperty(RECEIVE_MSG_WAIT_TIME).asTimePeriod(TimeUnit.SECONDS).intValue()); final Charset charset = Charset.forName(context.getProperty(CHARSET).getValue()); final ReceiveMessageResult result; try { result = client.receiveMessage(request); } catch (final Exception e) { getLogger().error("Failed to receive messages from Amazon SQS due to {}", new Object[] { e }); context.yield(); return; } final List<Message> messages = result.getMessages(); if (messages.isEmpty()) { context.yield(); return; } final boolean autoDelete = context.getProperty(AUTO_DELETE).asBoolean(); for (final Message message : messages) { FlowFile flowFile = session.create(); final Map<String, String> attributes = new HashMap<>(); for (final Map.Entry<String, String> entry : message.getAttributes().entrySet()) { attributes.put("sqs." + entry.getKey(), entry.getValue()); } for (final Map.Entry<String, MessageAttributeValue> entry : message.getMessageAttributes().entrySet()) { attributes.put("sqs." + entry.getKey(), entry.getValue().getStringValue()); } attributes.put("hash.value", message.getMD5OfBody()); attributes.put("hash.algorithm", "md5"); attributes.put("sqs.message.id", message.getMessageId()); attributes.put("sqs.receipt.handle", message.getReceiptHandle()); flowFile = session.putAllAttributes(flowFile, attributes); flowFile = session.write(flowFile, new OutputStreamCallback() { @Override public void process(final OutputStream out) throws IOException { out.write(message.getBody().getBytes(charset)); } }); session.transfer(flowFile, REL_SUCCESS); session.getProvenanceReporter().receive(flowFile, queueUrl); getLogger().info("Successfully received {} from Amazon SQS", new Object[] { flowFile }); } if (autoDelete) { // If we want to auto-delete messages, we must fist commit the session to ensure that the data // is persisted in NiFi's repositories. session.commit(); final DeleteMessageBatchRequest deleteRequest = new DeleteMessageBatchRequest(); deleteRequest.setQueueUrl(queueUrl); final List<DeleteMessageBatchRequestEntry> deleteRequestEntries = new ArrayList<>(); for (final Message message : messages) { final DeleteMessageBatchRequestEntry entry = new DeleteMessageBatchRequestEntry(); entry.setId(message.getMessageId()); entry.setReceiptHandle(message.getReceiptHandle()); deleteRequestEntries.add(entry); } deleteRequest.setEntries(deleteRequestEntries); try { client.deleteMessageBatch(deleteRequest); } catch (final Exception e) { getLogger().error( "Received {} messages from Amazon SQS but failed to delete the messages; these messages" + " may be duplicated. Reason for deletion failure: {}", new Object[] { messages.size(), e }); } } }
From source file:org.apache.usergrid.apm.service.MetricsInjestionServiceSQSImpl.java
License:Apache License
protected List<ReceiveMessageResult> getClientDataForApp(String fullAppName) { ArrayList<ReceiveMessageResult> messageResults = new ArrayList<ReceiveMessageResult>( MAX_NUMBER_OF_REQUEST_TO_PROCESS); try {/* ww w . j av a 2s . c om*/ GetQueueAttributesRequest queueAttributesRequest = new GetQueueAttributesRequest(); ArrayList<String> attributeNames = new ArrayList<String>(2); attributeNames.add("ApproximateNumberOfMessages"); attributeNames.add("LastModifiedTimestamp"); //attributeNames.add("All"); queueAttributesRequest.setAttributeNames(attributeNames); String qUrl = AWSUtil.formFullQueueUrl(fullAppName); log.info("Getting APM data from SQS queue" + qUrl); queueAttributesRequest.setQueueUrl(qUrl); GetQueueAttributesResult queueAttributeResult = sqsClient.getQueueAttributes(queueAttributesRequest); String numMessagesString = queueAttributeResult.getAttributes().get("ApproximateNumberOfMessages"); //Calculate number of "ReceiveMessage" requests need to be made. //TODO might need to use AsyncClient in the future to make multiple requests int numMessages = Integer.parseInt(numMessagesString); log.info("Num Messages to Process " + numMessages + " messages"); if (numMessages > 0) { int receiveMessages = 0; int receiveRequest = 0; int lastNumberOfRecievedMessages = 1; while ((receiveMessages < numMessages) && (receiveRequest < MAX_NUMBER_OF_REQUEST_TO_PROCESS) && (lastNumberOfRecievedMessages != 0)) { ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(); receiveMessageRequest.setMaxNumberOfMessages(MAX_NUMBER_OF_MESSAGES); receiveMessageRequest.setQueueUrl(qUrl); ArrayList<String> requestMessageAttributeNames = new ArrayList<String>(1); requestMessageAttributeNames.add("All"); receiveMessageRequest.setAttributeNames(requestMessageAttributeNames); try { ReceiveMessageResult receiveMessageResult = sqsClient.receiveMessage(receiveMessageRequest); log.info("For application " + fullAppName + " Received " + receiveMessageResult.getMessages().size() + " messages."); receiveMessages += receiveMessageResult.getMessages().size(); //check if any of these messages have been downloaded already. Since SQS is distributed and injestor //could have failed before deleting particular message, we check for message read count to 3. In odd //case, some messages could get processed at most 3 times. List<Message> messages = receiveMessageResult.getMessages(); String receiveCount = null; Message m = null; for (Iterator<Message> iter = messages.iterator(); iter.hasNext();) { m = iter.next(); receiveCount = m.getAttributes().get("ApproximateReceiveCount"); if (receiveCount != null && Integer.valueOf(receiveCount) > 3) { log.warn("ReceiveCount of message for app " + fullAppName + " is greater than 3 so going to delete this message before further processing"); sqsClient.deleteMessage(new DeleteMessageRequest(qUrl, m.getReceiptHandle())); iter.remove(); } } lastNumberOfRecievedMessages = receiveMessageResult.getMessages().size(); if (lastNumberOfRecievedMessages > 0) { messageResults.add(receiveMessageResult); receiveRequest++; } ; } catch (Exception ex) { log.error("Problem getting messages for " + fullAppName, ex); } } } } catch (AmazonServiceException ce) { log.error("Problem pulling message from SQS for " + fullAppName, ce); } catch (Exception e) { log.error("Problem getting messages for " + fullAppName, e); } return messageResults; }
From source file:org.lendingclub.reflex.aws.sqs.SQSAdapter.java
License:Apache License
@SuppressWarnings("unchecked") public synchronized <T extends SQSAdapter> T start() { if (running.get()) { logger.warn("already running"); return (T) this; }/*from w w w. j av a 2 s . c o m*/ Preconditions.checkArgument(sqs != null, "SQSClient must be set"); if (urlSupplier == null && queueName != null) { urlSupplier = Suppliers.memoize(new SQSUrlSupplier(sqs, queueName)); } if (urlSupplier == null) { throw new IllegalArgumentException("queueUrl or queueName must be set"); } Runnable r = new Runnable() { public void run() { running.set(true); while (running.get()) { try { if (isRunning()) { ReceiveMessageRequest rmr = new ReceiveMessageRequest(); rmr.setWaitTimeSeconds(getWaitTimeSeconds()); rmr.setMaxNumberOfMessages(getMessagesPerRequest()); if (urlSupplier == null) { throw new IllegalArgumentException("queueUrl or queueName must be set"); } rmr.setQueueUrl(urlSupplier.get()); ReceiveMessageResult result = sqs.receiveMessage(rmr); List<Message> list = result.getMessages(); if (list != null) { for (Message message : list) { try { messageReceiveCount.incrementAndGet(); if (logger.isDebugEnabled()) { logger.debug("received: {}", message.getMessageId()); } SQSMessage sqs = new SQSMessage(); sqs.message = message; subject.onNext(sqs); if (autoDelete) { delete(message); } dispatchSuccessCount.incrementAndGet(); resetFailureCount(); } catch (Exception e) { handleException(e); } } } } else { logger.info("{} is paused", this); Thread.sleep(10000); } } catch (Throwable e) { Exceptions.throwIfFatal(e); handleException(e); } } logger.info("stopped"); } }; String threadNameFormat = String.format("%s-%s", "SQSAdapter", (Strings.isNullOrEmpty(name) ? Integer.toHexString(hashCode()) : name)) + "-%d"; ThreadFactoryBuilder tfb = new ThreadFactoryBuilder().setDaemon(true).setNameFormat(threadNameFormat); Thread t = tfb.build().newThread(r); logger.info("starting thread: {}", t); t.start(); return (T) this; }
From source file:org.mule.modules.sqs.SQSConnector.java
License:Open Source License
/** * Attempts to receive a message from the queue. Every attribute of the incoming * message will be added as an inbound property. Also the following properties * will also be added:/* w ww.j a v a 2 s .co m*/ * <p/> * sqs.message.id = containing the message identification * sqs.message.receipt.handle = containing the message identification * <p/> * {@sample.xml ../../../doc/mule-module-sqs.xml.sample sqs:receive-messages} * * @param callback Callback to call when a new message is available. * @param visibilityTimeout the duration (in seconds) the retrieved message is hidden from * subsequent calls to retrieve. * @param preserveMessages Flag that indicates if you want to preserve the messages * in the queue. False by default, so the messages are * going to be deleted. * @param pollPeriod time in milliseconds to wait between polls (when no messages were retrieved). * Default period is 1000 ms. * @param numberOfMessages the number of messages to be retrieved on each call (10 messages max). * By default, 1 message will be retrieved. * @throws AmazonClientException * If any internal errors are encountered inside the client while * attempting to make the request or handle the response. For example * if a network connection is not available. * @throws AmazonServiceException * If an error response is returned by AmazonSQS indicating * either a problem with the data in the request, or a server side issue. */ @Source @InvalidateConnectionOn(exception = AmazonClientException.class) public void receiveMessages(SourceCallback callback, @Optional @Default("30") Integer visibilityTimeout, @Optional @Default("false") Boolean preserveMessages, @Optional @Default("1000") Long pollPeriod, @Optional @Default("1") Integer numberOfMessages) throws AmazonServiceException { List<Message> messages; ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(); receiveMessageRequest.setQueueUrl(getQueueUrl()); if (visibilityTimeout != null) { receiveMessageRequest.setVisibilityTimeout(visibilityTimeout); } receiveMessageRequest.setMaxNumberOfMessages(numberOfMessages); while (!Thread.interrupted()) { messages = msgQueue.receiveMessage(receiveMessageRequest).getMessages(); try { if (messages.size() == 0) { Thread.sleep(pollPeriod); continue; } for (Message msg : messages) { callback.process(msg.getBody(), createProperties(msg)); if (!preserveMessages) { msgQueue.deleteMessage(new DeleteMessageRequest(getQueueUrl(), msg.getReceiptHandle())); } } } catch (InterruptedException e) { logger.error(e.getMessage(), e); } catch (Exception e) { throw new AmazonClientException("Error while processing message.", e); } } }