List of usage examples for com.amazonaws.services.sqs.model ReceiveMessageRequest setVisibilityTimeout
public void setVisibilityTimeout(Integer visibilityTimeout)
The duration (in seconds) that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request.
From source file:com.eucalyptus.portal.SimpleQueueClientManager.java
License:Open Source License
public List<Message> receiveAllMessages(final String queueName, final boolean shouldDelete) throws Exception { try {// www. j a va2s.c om final int visibilityTimeout = 600; final int visibilityBuffer = 300; final long startTime = System.currentTimeMillis(); final List<Message> messages = Lists.newArrayList(); while ((System.currentTimeMillis() - startTime) < ((visibilityTimeout - visibilityBuffer) * 1000L)) { final ReceiveMessageRequest req = new ReceiveMessageRequest(); req.setQueueUrl(getQueueUrl(queueName)); req.setMaxNumberOfMessages(10); req.setWaitTimeSeconds(0); req.setVisibilityTimeout(visibilityTimeout); final ReceiveMessageResult result = getSimpleQueueClient().receiveMessage(req); final List<Message> received = result.getMessages(); if (received == null || received.size() <= 0) break; messages.addAll(received); } // TODO: Use PurgeQueue if (shouldDelete) { for (final List<Message> partition : Iterables.partition(messages, 10)) { final DeleteMessageBatchRequest delReq = new DeleteMessageBatchRequest(); delReq.setQueueUrl(getQueueUrl(queueName)); delReq.setEntries(partition.stream().map(m -> new DeleteMessageBatchRequestEntry() .withId(m.getMessageId()).withReceiptHandle(m.getReceiptHandle())) .collect(Collectors.toList())); getSimpleQueueClient().deleteMessageBatch(delReq); } } return messages; } catch (final AmazonServiceException ex) { throw new Exception("Failed to receive messages due to service error", ex); } catch (final AmazonClientException ex) { throw new Exception("Failed to receive messages due to client error", ex); } }
From source file:com.pocketdealhunter.HotDealsMessagesUtil.java
License:Open Source License
public List<Message> getMessageQueue() { try {//from w ww. ja va 2s .c o m ReceiveMessageRequest rmr = new ReceiveMessageRequest(this.queueUrl); rmr.setMaxNumberOfMessages(10); rmr.setVisibilityTimeout(2); List<Message> messages = null; ArrayList<Message> allMessages = new ArrayList<Message>(100); do { ReceiveMessageResult result = this.sqsClient.receiveMessage(rmr); messages = result.getMessages(); allMessages.addAll(messages); try { Thread.sleep(1 * 1000); } catch (Exception exception) { } } while (!messages.isEmpty()); return allMessages; } catch (Exception exception) { System.out.println("Exception = " + exception); return Collections.emptyList(); } }
From source file:org.apache.camel.component.aws.sqs.SqsConsumer.java
License:Apache License
@Override protected int poll() throws Exception { // must reset for each poll shutdownRunningTask = null;//from ww w. ja v a 2 s.co m pendingExchanges = 0; ReceiveMessageRequest request = new ReceiveMessageRequest(getQueueUrl()); request.setMaxNumberOfMessages(getMaxMessagesPerPoll() > 0 ? getMaxMessagesPerPoll() : null); request.setVisibilityTimeout( getConfiguration().getVisibilityTimeout() != null ? getConfiguration().getVisibilityTimeout() : null); request.setAttributeNames( getConfiguration().getAttributeNames() != null ? getConfiguration().getAttributeNames() : null); request.setWaitTimeSeconds( getConfiguration().getWaitTimeSeconds() != null ? getConfiguration().getWaitTimeSeconds() : null); LOG.trace("Receiving messages with request [{}]...", request); ReceiveMessageResult messageResult = null; try { messageResult = getClient().receiveMessage(request); } catch (QueueDoesNotExistException e) { LOG.info("Queue does not exist....recreating now..."); reConnectToQueue(); messageResult = getClient().receiveMessage(request); } if (LOG.isTraceEnabled()) { LOG.trace("Received {} messages", messageResult.getMessages().size()); } Queue<Exchange> exchanges = createExchanges(messageResult.getMessages()); return processBatch(CastUtils.cast(exchanges)); }
From source file:org.apache.nifi.processors.aws.sqs.GetSQS.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { final String queueUrl = context.getProperty(DYNAMIC_QUEUE_URL).evaluateAttributeExpressions().getValue(); final AmazonSQSClient client = getClient(); final ReceiveMessageRequest request = new ReceiveMessageRequest(); request.setAttributeNames(Collections.singleton("All")); request.setMessageAttributeNames(Collections.singleton("All")); request.setMaxNumberOfMessages(context.getProperty(BATCH_SIZE).asInteger()); request.setVisibilityTimeout( context.getProperty(VISIBILITY_TIMEOUT).asTimePeriod(TimeUnit.SECONDS).intValue()); request.setQueueUrl(queueUrl);//from w w w . ja v a 2s. c o m request.setWaitTimeSeconds( context.getProperty(RECEIVE_MSG_WAIT_TIME).asTimePeriod(TimeUnit.SECONDS).intValue()); final Charset charset = Charset.forName(context.getProperty(CHARSET).getValue()); final ReceiveMessageResult result; try { result = client.receiveMessage(request); } catch (final Exception e) { getLogger().error("Failed to receive messages from Amazon SQS due to {}", new Object[] { e }); context.yield(); return; } final List<Message> messages = result.getMessages(); if (messages.isEmpty()) { context.yield(); return; } final boolean autoDelete = context.getProperty(AUTO_DELETE).asBoolean(); for (final Message message : messages) { FlowFile flowFile = session.create(); final Map<String, String> attributes = new HashMap<>(); for (final Map.Entry<String, String> entry : message.getAttributes().entrySet()) { attributes.put("sqs." + entry.getKey(), entry.getValue()); } for (final Map.Entry<String, MessageAttributeValue> entry : message.getMessageAttributes().entrySet()) { attributes.put("sqs." + entry.getKey(), entry.getValue().getStringValue()); } attributes.put("hash.value", message.getMD5OfBody()); attributes.put("hash.algorithm", "md5"); attributes.put("sqs.message.id", message.getMessageId()); attributes.put("sqs.receipt.handle", message.getReceiptHandle()); flowFile = session.putAllAttributes(flowFile, attributes); flowFile = session.write(flowFile, new OutputStreamCallback() { @Override public void process(final OutputStream out) throws IOException { out.write(message.getBody().getBytes(charset)); } }); session.transfer(flowFile, REL_SUCCESS); session.getProvenanceReporter().receive(flowFile, queueUrl); getLogger().info("Successfully received {} from Amazon SQS", new Object[] { flowFile }); } if (autoDelete) { // If we want to auto-delete messages, we must fist commit the session to ensure that the data // is persisted in NiFi's repositories. session.commit(); final DeleteMessageBatchRequest deleteRequest = new DeleteMessageBatchRequest(); deleteRequest.setQueueUrl(queueUrl); final List<DeleteMessageBatchRequestEntry> deleteRequestEntries = new ArrayList<>(); for (final Message message : messages) { final DeleteMessageBatchRequestEntry entry = new DeleteMessageBatchRequestEntry(); entry.setId(message.getMessageId()); entry.setReceiptHandle(message.getReceiptHandle()); deleteRequestEntries.add(entry); } deleteRequest.setEntries(deleteRequestEntries); try { client.deleteMessageBatch(deleteRequest); } catch (final Exception e) { getLogger().error( "Received {} messages from Amazon SQS but failed to delete the messages; these messages" + " may be duplicated. Reason for deletion failure: {}", new Object[] { messages.size(), e }); } } }
From source file:org.apache.usergrid.persistence.queue.impl.SNSQueueManagerImpl.java
License:Apache License
@Override public List<LegacyQueueMessage> getMessages(final int limit, final Class klass) { if (sqs == null) { logger.error("SQS is null - was not initialized properly"); return new ArrayList<>(0); }/*from w w w .j a va 2s . c o m*/ String url = getReadQueue().getUrl(); if (logger.isTraceEnabled()) { logger.trace("Getting up to {} messages from {}", limit, url); } ArrayList<String> requestMessageAttributeNames = new ArrayList<String>(1); requestMessageAttributeNames.add("ApproximateReceiveCount"); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(url); receiveMessageRequest.setAttributeNames(requestMessageAttributeNames); receiveMessageRequest.setMaxNumberOfMessages(limit); receiveMessageRequest .setVisibilityTimeout(Math.max(MIN_VISIBILITY_TIMEOUT, fig.getVisibilityTimeout() / 1000)); int longPollTimeout = Math.min(20000, fig.getQueuePollTimeout()); // 20000 is the SQS maximum // ensure the client's socket timeout is not less than the configure long poll timeout if (fig.getQueueClientSocketTimeout() < longPollTimeout) { longPollTimeout = Math.max(0, fig.getQueueClientSocketTimeout() - 1000); } receiveMessageRequest.setWaitTimeSeconds(longPollTimeout / 1000); // convert to seconds try { ReceiveMessageResult result = sqs.receiveMessage(receiveMessageRequest); List<Message> messages = result.getMessages(); if (logger.isTraceEnabled()) { logger.trace("Received {} messages from {}", messages.size(), url); } List<LegacyQueueMessage> queueMessages = new ArrayList<>(messages.size()); for (Message message : messages) { Object payload; final String originalBody = message.getBody(); try { final JsonNode bodyNode = mapper.readTree(message.getBody()); /** * When a message originates from SNS it has a "Message" we have to extract * it and then process it separately */ if (bodyNode.has("Message")) { final String snsNode = bodyNode.get("Message").asText(); payload = deSerializeSQSMessage(snsNode, klass); } else { payload = deSerializeSQSMessage(originalBody, klass); } } catch (Exception e) { logger.error("failed to deserialize message: {}", message.getBody(), e); throw new RuntimeException(e); } LegacyQueueMessage queueMessage = new LegacyQueueMessage(message.getMessageId(), message.getReceiptHandle(), payload, message.getAttributes().get("type")); queueMessage.setStringBody(originalBody); int receiveCount = Integer.valueOf(message.getAttributes().get("ApproximateReceiveCount")); queueMessage.setReceiveCount(receiveCount); queueMessages.add(queueMessage); } return queueMessages; } catch (com.amazonaws.services.sqs.model.QueueDoesNotExistException dne) { logger.error("Queue does not exist! [{}]", url, dne); } catch (Exception e) { logger.error("Programming error getting messages from queue=[{}] exist!", url, e); } return new ArrayList<>(0); }
From source file:org.mule.modules.sqs.SQSConnector.java
License:Open Source License
/** * Attempts to receive a message from the queue. Every attribute of the incoming * message will be added as an inbound property. Also the following properties * will also be added://from w ww. j a v a 2 s. c o m * <p/> * sqs.message.id = containing the message identification * sqs.message.receipt.handle = containing the message identification * <p/> * {@sample.xml ../../../doc/mule-module-sqs.xml.sample sqs:receive-messages} * * @param callback Callback to call when a new message is available. * @param visibilityTimeout the duration (in seconds) the retrieved message is hidden from * subsequent calls to retrieve. * @param preserveMessages Flag that indicates if you want to preserve the messages * in the queue. False by default, so the messages are * going to be deleted. * @param pollPeriod time in milliseconds to wait between polls (when no messages were retrieved). * Default period is 1000 ms. * @param numberOfMessages the number of messages to be retrieved on each call (10 messages max). * By default, 1 message will be retrieved. * @throws AmazonClientException * If any internal errors are encountered inside the client while * attempting to make the request or handle the response. For example * if a network connection is not available. * @throws AmazonServiceException * If an error response is returned by AmazonSQS indicating * either a problem with the data in the request, or a server side issue. */ @Source @InvalidateConnectionOn(exception = AmazonClientException.class) public void receiveMessages(SourceCallback callback, @Optional @Default("30") Integer visibilityTimeout, @Optional @Default("false") Boolean preserveMessages, @Optional @Default("1000") Long pollPeriod, @Optional @Default("1") Integer numberOfMessages) throws AmazonServiceException { List<Message> messages; ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(); receiveMessageRequest.setQueueUrl(getQueueUrl()); if (visibilityTimeout != null) { receiveMessageRequest.setVisibilityTimeout(visibilityTimeout); } receiveMessageRequest.setMaxNumberOfMessages(numberOfMessages); while (!Thread.interrupted()) { messages = msgQueue.receiveMessage(receiveMessageRequest).getMessages(); try { if (messages.size() == 0) { Thread.sleep(pollPeriod); continue; } for (Message msg : messages) { callback.process(msg.getBody(), createProperties(msg)); if (!preserveMessages) { msgQueue.deleteMessage(new DeleteMessageRequest(getQueueUrl(), msg.getReceiptHandle())); } } } catch (InterruptedException e) { logger.error(e.getMessage(), e); } catch (Exception e) { throw new AmazonClientException("Error while processing message.", e); } } }
From source file:pa3.RemoteClientSQS.java
License:Open Source License
public static void main(String[] args) throws Exception { initSQSandDynamoDB();/* w w w .j a v a 2s. co m*/ double startTime, endTime, totalTime; // String fileName = // "C:/Cloud/Assignment/LocalQueueAsst3/src/cloud/asst3/queue/10KSleepTasks.txt"; // int noOfThreads = 16 ; // String localOrRemote = "REMOTE"; // String clientOrWorker = "CLIENT"; String fileName = args[4]; String requestQueueName = "MyRequestQueue" + args[2]; String responseQueueName = "MyResponseQueue" + args[2]; String clientOrWorker = args[0]; System.out.println("==========================================="); System.out.println("Lets get started with Amazon SQS"); System.out.println("===========================================\n"); try { // Create a queue System.out.println("Creating a new SQS queue called MyRequestQueue.\n"); CreateQueueRequest createRequestQueue = new CreateQueueRequest(requestQueueName); String myRequestQueueUrl = sqs.createQueue(createRequestQueue).getQueueUrl(); System.out.println("Creating a new SQS queue called MyResponseQueue.\n"); CreateQueueRequest createResponseQueue = new CreateQueueRequest(responseQueueName); String myResponseQueueUrl = sqs.createQueue(createResponseQueue).getQueueUrl(); // List queues System.out.println("Listing all queues in your account.\n"); for (String queueUrl : sqs.listQueues().getQueueUrls()) { System.out.println(" QueueUrl: " + queueUrl); } // Send a message System.out.println("Sending a message to " + requestQueueName); int taskID = 1; FileReader fileReader; BufferedReader bufferedReader = null; startTime = System.nanoTime(); try { fileReader = new FileReader(fileName); bufferedReader = new BufferedReader(fileReader); String eachTaskLine = null; mySubmittedTasks = new ConcurrentHashMap<Integer, String>(); while ((eachTaskLine = bufferedReader.readLine()) != null) { sqs.sendMessage(new SendMessageRequest(myRequestQueueUrl, taskID + " " + eachTaskLine)); mySubmittedTasks.put(taskID, eachTaskLine); taskID++; } bufferedReader.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.out.println("Sent all the messages to " + requestQueueName); try { ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myResponseQueueUrl); receiveMessageRequest.setVisibilityTimeout(900); receiveMessageRequest.setWaitTimeSeconds(20); while (!mySubmittedTasks.isEmpty()) { List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); for (Message message : messages) { if (mySubmittedTasks.containsKey(Integer.parseInt(message.getBody().toString()))) { mySubmittedTasks.remove(Integer.parseInt(message.getBody().toString())); String messageReceiptHandle = message.getReceiptHandle(); sqs.deleteMessage(new DeleteMessageRequest(myResponseQueueUrl, messageReceiptHandle)); } } } } catch (Exception ex) { ex.printStackTrace(); } if (!mySubmittedTasks.isEmpty()) { System.out.println("Some tasks have failed"); } endTime = System.nanoTime(); totalTime = (endTime - startTime) / 1000000000.0; System.out.println("This is " + clientOrWorker.toUpperCase() + " running with workers."); System.out.println("Total time taken: " + totalTime); System.out.println("Received all the messages from MyResponseQueue.\n"); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon SQS, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with SQS, such as not " + "being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:pa3.RemoteWorkerSQS.java
License:Open Source License
public static void main(String[] args) throws Exception { initSQSandDynamoDB();//from w ww. j a v a2 s .c om try { String tableName = "PA3" + args[2]; // Create table if it does not exist yet if (Tables.doesTableExist(dynamoDB, tableName)) { System.out.println("Table " + tableName + " is already ACTIVE"); } else { // Create a table with a primary hash key named 'name', which // holds a string CreateTableRequest createTableRequest = new CreateTableRequest().withTableName(tableName) .withKeySchema(new KeySchemaElement().withAttributeName("taskID").withKeyType(KeyType.HASH)) .withAttributeDefinitions(new AttributeDefinition().withAttributeName("taskID") .withAttributeType(ScalarAttributeType.S)) .withProvisionedThroughput( new ProvisionedThroughput().withReadCapacityUnits(1L).withWriteCapacityUnits(1L)); TableDescription createdTableDescription = dynamoDB.createTable(createTableRequest) .getTableDescription(); System.out.println("Created Table: " + createdTableDescription); // Wait for it to become active System.out.println("Waiting for " + tableName + " to become ACTIVE..."); Tables.awaitTableToBecomeActive(dynamoDB, tableName); } String noOfWorkers = args[4]; String requestQueueName = "TaskQueue" + args[2]; String responseQueueName = "TaskResponseQueue" + args[2]; String clientOrWorker = args[0]; // Create a queue //System.out.println("Accessing SQS queue: "+requestQueueName); String myRequestQueueUrl = sqs.createQueue(requestQueueName).getQueueUrl(); //System.out.println("Creating a new SQS queue called MyResponseQueue.\n"); String myResponseQueueUrl = sqs.createQueue(responseQueueName).getQueueUrl(); // Receive the messages try { ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myRequestQueueUrl); receiveMessageRequest.setVisibilityTimeout(900); receiveMessageRequest.setWaitTimeSeconds(20); while (true) { List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); // Throw exception when queue gets empty if (messages.isEmpty()) { break; } for (Message message : messages) { try { String[] splitTask = message.getBody().split(" "); //DynamoDB Map<String, AttributeValue> item = newItem(splitTask[0]); PutItemRequest putItemRequest = new PutItemRequest(tableName, item) .withConditionExpression("attribute_not_exists(taskID)"); dynamoDB.putItem(putItemRequest); //System.out.println(splitTask[0]+" : "+splitTask[2]); // Execute Task Thread.sleep(Long.parseLong(splitTask[2])); sqs.sendMessage(new SendMessageRequest(myResponseQueueUrl, splitTask[0])); // Delete the message String messageReceiptHandle = message.getReceiptHandle(); sqs.deleteMessage(new DeleteMessageRequest(myRequestQueueUrl, messageReceiptHandle)); } catch (ConditionalCheckFailedException e) { //e.printStackTrace(); } } } } catch (Exception ex) { //ex.printStackTrace(); } } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to AWS, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with AWS, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }