List of usage examples for com.amazonaws.services.s3.event S3EventNotification parseJson
public static S3EventNotification parseJson(String json)
Parse the JSON string into a S3EventNotification object.
From source file:com.nextdoor.bender.handler.s3.SNSS3Handler.java
License:Apache License
@Override public void handler(SNSEvent event, Context context) throws HandlerException { if (!initialized) { init(context);/*from w w w. j a v a 2s. co m*/ SNSS3HandlerConfig handlerConfig = (SNSS3HandlerConfig) this.config.getHandlerConfig(); this.logTrigger = handlerConfig.getLogSnsTrigger(); } this.source = this.sources.get(0); this.inputFiles = new ArrayList<String>(0); if (this.logTrigger) { logger.info("trigger: " + gson.toJson(event)); } for (SNSRecord record : event.getRecords()) { /* * Parse SNS as a S3 notification */ String json = record.getSNS().getMessage(); S3EventNotification s3Event = S3EventNotification.parseJson(json); /* * Validate the S3 file matches the regex */ List<S3EventNotificationRecord> toProcess = new ArrayList<S3EventNotificationRecord>( s3Event.getRecords()); for (S3EventNotificationRecord s3Record : s3Event.getRecords()) { String s3Path = String.format("s3://%s/%s", s3Record.getS3().getBucket().getName(), s3Record.getS3().getObject().getKey()); try { this.source = SourceUtils.getSource(s3Path, this.sources); } catch (SourceNotFoundException e) { logger.warn("skipping processing " + s3Path); toProcess.remove(s3Record); } } if (toProcess.size() == 0) { logger.warn("Nothing to process"); return; } this.inputFiles.addAll(toProcess.stream().map(m -> { return m.getS3().getObject().getKey(); }).collect(Collectors.toList())); this.recordIterator = new S3EventIterator(new LambdaContext(context), toProcess, s3ClientFactory); super.process(context); } }
From source file:com.smoketurner.pipeline.application.core.MessageProcessor.java
License:Apache License
/** * Process an SQS {@link Message} by parsing the SNS notification out of the * message body. Then download the S3 object out of the SNS notification, * decompress the object, then broadcast each event. * //from w w w . j a va2s.c o m * @param message * SQS message * @return true if the file was fully processed (and the message can be * deleted from SQS), otherwise false. */ @Override public boolean test(@Nullable final Message message) { if (message == null) { return false; } if (LOGGER.isTraceEnabled()) { LOGGER.trace("Received SQS message: {}", message); } else if (LOGGER.isDebugEnabled()) { LOGGER.debug("Received SQS message: {}", message.getMessageId()); } if (broadcaster.isEmpty()) { LOGGER.debug("No connections found, skipping SQS message processing"); return false; } final AmazonSNSNotification notification; try { notification = MAPPER.readValue(message.getBody(), AmazonSNSNotification.class); } catch (IOException e) { LOGGER.error("Failed to parse SNS notification, deleting SQS message", e); return true; } LOGGER.debug("SNS notification created at: {} ({} behind)", notification.getTimestamp(), notification.getDelayDuration()); // if we don't have a valid SNS notification, try parsing the body as S3 // event records final String body; if (notification.isValid()) { body = notification.getMessage(); } else { body = message.getBody(); } final S3EventNotification records; try { records = S3EventNotification.parseJson(body); } catch (AmazonClientException e) { LOGGER.error("Failed to parse S3 event records, deleting SQS message", e); return true; } final int recordCount = records.getRecords().size(); recordCounts.update(recordCount); LOGGER.debug("Parsed {} S3 event records from SNS notification", recordCount); if (recordCount < 1) { LOGGER.debug("No S3 event records found in SNS notification, deleting SQS message"); return true; } int recordsProcessed = 0; for (S3EventNotificationRecord record : records.getRecords()) { if (broadcaster.isEmpty()) { LOGGER.debug("No connections found, not downloading from S3"); return false; } if (processRecord(record)) { recordsProcessed++; } } // if we've processed all of the records, which includes skipping over // empty S3 files, the message has been fully processed. if (recordsProcessed == recordCount) { LOGGER.debug("Processed {} of {} records, deleting SQS message", recordsProcessed, recordCount); return true; } LOGGER.debug("Processed {} of {} records, not deleting SQS message: {}", recordsProcessed, recordCount, message.getMessageId()); return false; }
From source file:org.finra.dm.service.helper.DmJmsMessageListener.java
License:Apache License
/** * Process the message as S3 notification. * * @param payload the JMS message payload. * * @return boolean whether message was processed. *///w w w .j a v a2 s . c o m private boolean processS3Notification(String payload) { boolean messageProcessed = false; try { // Process messages coming from S3 bucket. S3EventNotification s3EventNotification = S3EventNotification.parseJson(payload); String objectKey = URLDecoder.decode( s3EventNotification.getRecords().get(0).getS3().getObject().getKey(), CharEncoding.UTF_8); // Perform the complete upload single file. CompleteUploadSingleMessageResult returnValues = uploadDownloadService .performCompleteUploadSingleMessage(objectKey); LOGGER.debug(String.format( "completeUploadSingleMessageResult- SourceBusinessObjectDataKey: \"%s\", sourceOldStatus: \"%s\", " + "sourceNewStatus: \"%s\", TargetBusinessObjectDataKey: \"%s\", targetOldStatus: \"%s\", targetNewStatus: \"%s\"", dmHelper.businessObjectDataKeyToString(returnValues.getSourceBusinessObjectDataKey()), returnValues.getSourceOldStatus(), returnValues.getSourceNewStatus(), dmHelper.businessObjectDataKeyToString(returnValues.getTargetBusinessObjectDataKey()), returnValues.getTargetOldStatus(), returnValues.getTargetNewStatus())); messageProcessed = true; } catch (Exception e) { LOGGER.debug(String.format( "Failed to process JMS message from \"%s\" queue. Payload: \"%s\" for an S3 notification.", DmJmsDestinationResolver.SQS_DESTINATION_DM_INCOMING, payload), e); } return messageProcessed; }
From source file:org.finra.herd.service.helper.HerdJmsMessageListener.java
License:Apache License
/** * Process the message as S3 notification. * * @param payload the JMS message payload. * * @return boolean whether message was processed. */// www . ja va2 s . c om private boolean processS3Notification(String payload) { boolean messageProcessed = false; try { // Process messages coming from S3 bucket. S3EventNotification s3EventNotification = S3EventNotification.parseJson(payload); String objectKey = URLDecoder.decode( s3EventNotification.getRecords().get(0).getS3().getObject().getKey(), CharEncoding.UTF_8); // Perform the complete upload single file. CompleteUploadSingleMessageResult completeUploadSingleMessageResult = uploadDownloadService .performCompleteUploadSingleMessage(objectKey); if (LOGGER.isDebugEnabled()) { LOGGER.debug("completeUploadSingleMessageResult={}", jsonHelper.objectToJson(completeUploadSingleMessageResult)); } messageProcessed = true; } catch (RuntimeException | UnsupportedEncodingException e) { // The logging is set to DEBUG level, since the method is expected to fail when message is not of the expected type. LOGGER.debug( "Failed to process message from the JMS queue for an S3 notification. jmsQueueName=\"{}\" jmsMessagePayload={}", HerdJmsDestinationResolver.SQS_DESTINATION_HERD_INCOMING, payload, e); } return messageProcessed; }
From source file:org.finra.herd.service.helper.SampleDataJmsMessageListener.java
License:Apache License
/** * Processes a JMS message./*from w w w. j a v a 2 s .co m*/ * * @param payload the message payload * @param allHeaders the JMS headers */ @JmsListener(id = HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE, containerFactory = "jmsListenerContainerFactory", destination = HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE) public void processMessage(String payload, @Headers Map<Object, Object> allHeaders) { LOGGER.info( "Message received from the JMS queue. jmsQueueName=\"{}\" jmsMessageHeaders=\"{}\" jmsMessagePayload={}", HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE, allHeaders, payload); try { // Process messages coming from S3 bucket. S3EventNotification s3EventNotification = S3EventNotification.parseJson(payload); String objectKey = URLDecoder.decode( s3EventNotification.getRecords().get(0).getS3().getObject().getKey(), CharEncoding.UTF_8); long fileSize = s3EventNotification.getRecords().get(0).getS3().getObject().getSizeAsLong(); // parse the objectKey, it should be in the format of namespace/businessObjectDefinitionName/fileName String[] objectKeyArrays = objectKey.split("/"); Assert.isTrue(objectKeyArrays.length == 3, String.format("S3 notification message %s is not in expected format", objectKey)); String namespace = objectKeyArrays[0]; String businessObjectDefinitionName = objectKeyArrays[1]; String fileName = objectKeyArrays[2]; String path = namespace + "/" + businessObjectDefinitionName + "/"; BusinessObjectDefinitionSampleFileUpdateDto businessObjectDefinitionSampleFileUpdateDto = new BusinessObjectDefinitionSampleFileUpdateDto( path, fileName, fileSize); String convertedNamespaece = convertS3KeyFormat(namespace); String convertedBusinessObjectDefinitionName = convertS3KeyFormat(businessObjectDefinitionName); BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey( convertedNamespaece, convertedBusinessObjectDefinitionName); try { businessObjectDefinitionService.updateBusinessObjectDefinitionEntitySampleFile( businessObjectDefinitionKey, businessObjectDefinitionSampleFileUpdateDto); } catch (ObjectNotFoundException ex) { LOGGER.info( "Failed to find the business object definition, next try the original namespace and business oject defination name " + ex); // if Business object definition is not found, use the original name space and bdef name businessObjectDefinitionKey = new BusinessObjectDefinitionKey(namespace, businessObjectDefinitionName); businessObjectDefinitionService.updateBusinessObjectDefinitionEntitySampleFile( businessObjectDefinitionKey, businessObjectDefinitionSampleFileUpdateDto); } } catch (RuntimeException | IOException e) { LOGGER.error("Failed to process message from the JMS queue. jmsQueueName=\"{}\" jmsMessagePayload={}", HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE, payload, e); } }