List of usage examples for com.amazonaws.services.sqs.model ReceiveMessageRequest setAttributeNames
public void setAttributeNames(java.util.Collection<String> attributeNames)
A list of attributes that need to be returned along with each message.
From source file:org.apache.beam.sdk.io.aws.sqs.SqsUnboundedReader.java
License:Apache License
private void pull() { final ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(source.getRead().queueUrl()); receiveMessageRequest.setMaxNumberOfMessages(MAX_NUMBER_OF_MESSAGES); receiveMessageRequest.setAttributeNames(Arrays.asList(MessageSystemAttributeName.SentTimestamp.toString())); final ReceiveMessageResult receiveMessageResult = source.getSqs().receiveMessage(receiveMessageRequest); final List<Message> messages = receiveMessageResult.getMessages(); if (messages == null || messages.isEmpty()) { return;/*w w w . ja v a 2s . c o m*/ } for (Message message : messages) { messagesNotYetRead.add(message); } }
From source file:org.apache.camel.component.aws.sqs.SqsConsumer.java
License:Apache License
@Override protected int poll() throws Exception { // must reset for each poll shutdownRunningTask = null;/* w ww . j av a 2 s. com*/ pendingExchanges = 0; ReceiveMessageRequest request = new ReceiveMessageRequest(getQueueUrl()); request.setMaxNumberOfMessages(getMaxMessagesPerPoll() > 0 ? getMaxMessagesPerPoll() : null); request.setVisibilityTimeout( getConfiguration().getVisibilityTimeout() != null ? getConfiguration().getVisibilityTimeout() : null); request.setAttributeNames( getConfiguration().getAttributeNames() != null ? getConfiguration().getAttributeNames() : null); request.setWaitTimeSeconds( getConfiguration().getWaitTimeSeconds() != null ? getConfiguration().getWaitTimeSeconds() : null); LOG.trace("Receiving messages with request [{}]...", request); ReceiveMessageResult messageResult = null; try { messageResult = getClient().receiveMessage(request); } catch (QueueDoesNotExistException e) { LOG.info("Queue does not exist....recreating now..."); reConnectToQueue(); messageResult = getClient().receiveMessage(request); } if (LOG.isTraceEnabled()) { LOG.trace("Received {} messages", messageResult.getMessages().size()); } Queue<Exchange> exchanges = createExchanges(messageResult.getMessages()); return processBatch(CastUtils.cast(exchanges)); }
From source file:org.apache.nifi.processors.aws.sqs.GetSQS.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { final String queueUrl = context.getProperty(DYNAMIC_QUEUE_URL).evaluateAttributeExpressions().getValue(); final AmazonSQSClient client = getClient(); final ReceiveMessageRequest request = new ReceiveMessageRequest(); request.setAttributeNames(Collections.singleton("All")); request.setMessageAttributeNames(Collections.singleton("All")); request.setMaxNumberOfMessages(context.getProperty(BATCH_SIZE).asInteger()); request.setVisibilityTimeout(/*from w w w .j a v a 2s .com*/ context.getProperty(VISIBILITY_TIMEOUT).asTimePeriod(TimeUnit.SECONDS).intValue()); request.setQueueUrl(queueUrl); request.setWaitTimeSeconds( context.getProperty(RECEIVE_MSG_WAIT_TIME).asTimePeriod(TimeUnit.SECONDS).intValue()); final Charset charset = Charset.forName(context.getProperty(CHARSET).getValue()); final ReceiveMessageResult result; try { result = client.receiveMessage(request); } catch (final Exception e) { getLogger().error("Failed to receive messages from Amazon SQS due to {}", new Object[] { e }); context.yield(); return; } final List<Message> messages = result.getMessages(); if (messages.isEmpty()) { context.yield(); return; } final boolean autoDelete = context.getProperty(AUTO_DELETE).asBoolean(); for (final Message message : messages) { FlowFile flowFile = session.create(); final Map<String, String> attributes = new HashMap<>(); for (final Map.Entry<String, String> entry : message.getAttributes().entrySet()) { attributes.put("sqs." + entry.getKey(), entry.getValue()); } for (final Map.Entry<String, MessageAttributeValue> entry : message.getMessageAttributes().entrySet()) { attributes.put("sqs." + entry.getKey(), entry.getValue().getStringValue()); } attributes.put("hash.value", message.getMD5OfBody()); attributes.put("hash.algorithm", "md5"); attributes.put("sqs.message.id", message.getMessageId()); attributes.put("sqs.receipt.handle", message.getReceiptHandle()); flowFile = session.putAllAttributes(flowFile, attributes); flowFile = session.write(flowFile, new OutputStreamCallback() { @Override public void process(final OutputStream out) throws IOException { out.write(message.getBody().getBytes(charset)); } }); session.transfer(flowFile, REL_SUCCESS); session.getProvenanceReporter().receive(flowFile, queueUrl); getLogger().info("Successfully received {} from Amazon SQS", new Object[] { flowFile }); } if (autoDelete) { // If we want to auto-delete messages, we must fist commit the session to ensure that the data // is persisted in NiFi's repositories. session.commit(); final DeleteMessageBatchRequest deleteRequest = new DeleteMessageBatchRequest(); deleteRequest.setQueueUrl(queueUrl); final List<DeleteMessageBatchRequestEntry> deleteRequestEntries = new ArrayList<>(); for (final Message message : messages) { final DeleteMessageBatchRequestEntry entry = new DeleteMessageBatchRequestEntry(); entry.setId(message.getMessageId()); entry.setReceiptHandle(message.getReceiptHandle()); deleteRequestEntries.add(entry); } deleteRequest.setEntries(deleteRequestEntries); try { client.deleteMessageBatch(deleteRequest); } catch (final Exception e) { getLogger().error( "Received {} messages from Amazon SQS but failed to delete the messages; these messages" + " may be duplicated. Reason for deletion failure: {}", new Object[] { messages.size(), e }); } } }
From source file:org.apache.usergrid.apm.service.MetricsInjestionServiceSQSImpl.java
License:Apache License
protected List<ReceiveMessageResult> getClientDataForApp(String fullAppName) { ArrayList<ReceiveMessageResult> messageResults = new ArrayList<ReceiveMessageResult>( MAX_NUMBER_OF_REQUEST_TO_PROCESS); try {//from w ww . java2s .c o m GetQueueAttributesRequest queueAttributesRequest = new GetQueueAttributesRequest(); ArrayList<String> attributeNames = new ArrayList<String>(2); attributeNames.add("ApproximateNumberOfMessages"); attributeNames.add("LastModifiedTimestamp"); //attributeNames.add("All"); queueAttributesRequest.setAttributeNames(attributeNames); String qUrl = AWSUtil.formFullQueueUrl(fullAppName); log.info("Getting APM data from SQS queue" + qUrl); queueAttributesRequest.setQueueUrl(qUrl); GetQueueAttributesResult queueAttributeResult = sqsClient.getQueueAttributes(queueAttributesRequest); String numMessagesString = queueAttributeResult.getAttributes().get("ApproximateNumberOfMessages"); //Calculate number of "ReceiveMessage" requests need to be made. //TODO might need to use AsyncClient in the future to make multiple requests int numMessages = Integer.parseInt(numMessagesString); log.info("Num Messages to Process " + numMessages + " messages"); if (numMessages > 0) { int receiveMessages = 0; int receiveRequest = 0; int lastNumberOfRecievedMessages = 1; while ((receiveMessages < numMessages) && (receiveRequest < MAX_NUMBER_OF_REQUEST_TO_PROCESS) && (lastNumberOfRecievedMessages != 0)) { ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(); receiveMessageRequest.setMaxNumberOfMessages(MAX_NUMBER_OF_MESSAGES); receiveMessageRequest.setQueueUrl(qUrl); ArrayList<String> requestMessageAttributeNames = new ArrayList<String>(1); requestMessageAttributeNames.add("All"); receiveMessageRequest.setAttributeNames(requestMessageAttributeNames); try { ReceiveMessageResult receiveMessageResult = sqsClient.receiveMessage(receiveMessageRequest); log.info("For application " + fullAppName + " Received " + receiveMessageResult.getMessages().size() + " messages."); receiveMessages += receiveMessageResult.getMessages().size(); //check if any of these messages have been downloaded already. Since SQS is distributed and injestor //could have failed before deleting particular message, we check for message read count to 3. In odd //case, some messages could get processed at most 3 times. List<Message> messages = receiveMessageResult.getMessages(); String receiveCount = null; Message m = null; for (Iterator<Message> iter = messages.iterator(); iter.hasNext();) { m = iter.next(); receiveCount = m.getAttributes().get("ApproximateReceiveCount"); if (receiveCount != null && Integer.valueOf(receiveCount) > 3) { log.warn("ReceiveCount of message for app " + fullAppName + " is greater than 3 so going to delete this message before further processing"); sqsClient.deleteMessage(new DeleteMessageRequest(qUrl, m.getReceiptHandle())); iter.remove(); } } lastNumberOfRecievedMessages = receiveMessageResult.getMessages().size(); if (lastNumberOfRecievedMessages > 0) { messageResults.add(receiveMessageResult); receiveRequest++; } ; } catch (Exception ex) { log.error("Problem getting messages for " + fullAppName, ex); } } } } catch (AmazonServiceException ce) { log.error("Problem pulling message from SQS for " + fullAppName, ce); } catch (Exception e) { log.error("Problem getting messages for " + fullAppName, e); } return messageResults; }
From source file:org.apache.usergrid.persistence.queue.impl.SNSQueueManagerImpl.java
License:Apache License
@Override public List<LegacyQueueMessage> getMessages(final int limit, final Class klass) { if (sqs == null) { logger.error("SQS is null - was not initialized properly"); return new ArrayList<>(0); }//w ww . j a v a2 s . co m String url = getReadQueue().getUrl(); if (logger.isTraceEnabled()) { logger.trace("Getting up to {} messages from {}", limit, url); } ArrayList<String> requestMessageAttributeNames = new ArrayList<String>(1); requestMessageAttributeNames.add("ApproximateReceiveCount"); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(url); receiveMessageRequest.setAttributeNames(requestMessageAttributeNames); receiveMessageRequest.setMaxNumberOfMessages(limit); receiveMessageRequest .setVisibilityTimeout(Math.max(MIN_VISIBILITY_TIMEOUT, fig.getVisibilityTimeout() / 1000)); int longPollTimeout = Math.min(20000, fig.getQueuePollTimeout()); // 20000 is the SQS maximum // ensure the client's socket timeout is not less than the configure long poll timeout if (fig.getQueueClientSocketTimeout() < longPollTimeout) { longPollTimeout = Math.max(0, fig.getQueueClientSocketTimeout() - 1000); } receiveMessageRequest.setWaitTimeSeconds(longPollTimeout / 1000); // convert to seconds try { ReceiveMessageResult result = sqs.receiveMessage(receiveMessageRequest); List<Message> messages = result.getMessages(); if (logger.isTraceEnabled()) { logger.trace("Received {} messages from {}", messages.size(), url); } List<LegacyQueueMessage> queueMessages = new ArrayList<>(messages.size()); for (Message message : messages) { Object payload; final String originalBody = message.getBody(); try { final JsonNode bodyNode = mapper.readTree(message.getBody()); /** * When a message originates from SNS it has a "Message" we have to extract * it and then process it separately */ if (bodyNode.has("Message")) { final String snsNode = bodyNode.get("Message").asText(); payload = deSerializeSQSMessage(snsNode, klass); } else { payload = deSerializeSQSMessage(originalBody, klass); } } catch (Exception e) { logger.error("failed to deserialize message: {}", message.getBody(), e); throw new RuntimeException(e); } LegacyQueueMessage queueMessage = new LegacyQueueMessage(message.getMessageId(), message.getReceiptHandle(), payload, message.getAttributes().get("type")); queueMessage.setStringBody(originalBody); int receiveCount = Integer.valueOf(message.getAttributes().get("ApproximateReceiveCount")); queueMessage.setReceiveCount(receiveCount); queueMessages.add(queueMessage); } return queueMessages; } catch (com.amazonaws.services.sqs.model.QueueDoesNotExistException dne) { logger.error("Queue does not exist! [{}]", url, dne); } catch (Exception e) { logger.error("Programming error getting messages from queue=[{}] exist!", url, e); } return new ArrayList<>(0); }