List of usage examples for com.amazonaws.services.s3.model ObjectMetadata getContentMD5
public String getContentMD5()
Gets the base64 encoded 128-bit MD5 digest of the associated object (content - not including headers) according to RFC 1864.
From source file:com.emc.ecs.sync.model.object.S3SyncObject.java
License:Open Source License
protected SyncMetadata toSyncMeta(ObjectMetadata s3meta) { SyncMetadata meta = new SyncMetadata(); meta.setCacheControl(s3meta.getCacheControl()); meta.setContentDisposition(s3meta.getContentDisposition()); meta.setContentEncoding(s3meta.getContentEncoding()); if (s3meta.getContentMD5() != null) meta.setChecksum(new Checksum("MD5", s3meta.getContentMD5())); meta.setContentType(s3meta.getContentType()); meta.setHttpExpires(s3meta.getHttpExpiresDate()); meta.setExpirationDate(s3meta.getExpirationTime()); meta.setModificationTime(s3meta.getLastModified()); meta.setContentLength(s3meta.getContentLength()); meta.setUserMetadata(toMetaMap(s3meta.getUserMetadata())); return meta;//from w w w . j a va2 s . c o m }
From source file:com.emc.vipr.services.s3.ViPRS3Client.java
License:Open Source License
/** * Executes a (Subclass of) PutObjectRequest. In particular, we check for subclasses * of the UpdateObjectRequest and inject the value of the Range header. This version * also returns the raw ObjectMetadata for the response so callers can construct * their own result objects.// w w w . ja va2 s. co m * @param putObjectRequest the request to execute * @return an ObjectMetadata containing the response headers. */ protected ObjectMetadata doPut(PutObjectRequest putObjectRequest) { assertParameterNotNull(putObjectRequest, "The PutObjectRequest parameter must be specified when uploading an object"); String bucketName = putObjectRequest.getBucketName(); String key = putObjectRequest.getKey(); ObjectMetadata metadata = putObjectRequest.getMetadata(); InputStream input = putObjectRequest.getInputStream(); if (metadata == null) metadata = new ObjectMetadata(); assertParameterNotNull(bucketName, "The bucket name parameter must be specified when uploading an object"); assertParameterNotNull(key, "The key parameter must be specified when uploading an object"); /* * This is compatible with progress listener set by either the legacy * method GetObjectRequest#setProgressListener or the new method * GetObjectRequest#setGeneralProgressListener. */ com.amazonaws.event.ProgressListener progressListener = putObjectRequest.getGeneralProgressListener(); ProgressListenerCallbackExecutor progressListenerCallbackExecutor = ProgressListenerCallbackExecutor .wrapListener(progressListener); // If a file is specified for upload, we need to pull some additional // information from it to auto-configure a few options if (putObjectRequest.getFile() != null) { File file = putObjectRequest.getFile(); // Always set the content length, even if it's already set metadata.setContentLength(file.length()); // Only set the content type if it hasn't already been set if (metadata.getContentType() == null) { metadata.setContentType(Mimetypes.getInstance().getMimetype(file)); } FileInputStream fileInputStream = null; try { fileInputStream = new FileInputStream(file); byte[] md5Hash = Md5Utils.computeMD5Hash(fileInputStream); metadata.setContentMD5(BinaryUtils.toBase64(md5Hash)); } catch (Exception e) { throw new AmazonClientException("Unable to calculate MD5 hash: " + e.getMessage(), e); } finally { try { fileInputStream.close(); } catch (Exception e) { } } try { input = new RepeatableFileInputStream(file); } catch (FileNotFoundException fnfe) { throw new AmazonClientException("Unable to find file to upload", fnfe); } } Request<PutObjectRequest> request = createRequest(bucketName, key, putObjectRequest, HttpMethodName.PUT); if (putObjectRequest.getAccessControlList() != null) { addAclHeaders(request, putObjectRequest.getAccessControlList()); } else if (putObjectRequest.getCannedAcl() != null) { request.addHeader(Headers.S3_CANNED_ACL, putObjectRequest.getCannedAcl().toString()); } if (putObjectRequest.getStorageClass() != null) { request.addHeader(Headers.STORAGE_CLASS, putObjectRequest.getStorageClass()); } if (putObjectRequest.getRedirectLocation() != null) { request.addHeader(Headers.REDIRECT_LOCATION, putObjectRequest.getRedirectLocation()); if (input == null) { input = new ByteArrayInputStream(new byte[0]); } } // Use internal interface to differentiate 0 from unset. if (metadata.getRawMetadata().get(Headers.CONTENT_LENGTH) == null) { /* * There's nothing we can do except for let the HTTP client buffer * the input stream contents if the caller doesn't tell us how much * data to expect in a stream since we have to explicitly tell * Amazon S3 how much we're sending before we start sending any of * it. */ log.warn("No content length specified for stream data. " + "Stream contents will be buffered in memory and could result in " + "out of memory errors."); } if (progressListenerCallbackExecutor != null) { com.amazonaws.event.ProgressReportingInputStream progressReportingInputStream = new com.amazonaws.event.ProgressReportingInputStream( input, progressListenerCallbackExecutor); fireProgressEvent(progressListenerCallbackExecutor, com.amazonaws.event.ProgressEvent.STARTED_EVENT_CODE); } if (!input.markSupported()) { int streamBufferSize = Constants.DEFAULT_STREAM_BUFFER_SIZE; String bufferSizeOverride = System.getProperty("com.amazonaws.sdk.s3.defaultStreamBufferSize"); if (bufferSizeOverride != null) { try { streamBufferSize = Integer.parseInt(bufferSizeOverride); } catch (Exception e) { log.warn("Unable to parse buffer size override from value: " + bufferSizeOverride); } } input = new RepeatableInputStream(input, streamBufferSize); } MD5DigestCalculatingInputStream md5DigestStream = null; if (metadata.getContentMD5() == null) { /* * If the user hasn't set the content MD5, then we don't want to * buffer the whole stream in memory just to calculate it. Instead, * we can calculate it on the fly and validate it with the returned * ETag from the object upload. */ try { md5DigestStream = new MD5DigestCalculatingInputStream(input); input = md5DigestStream; } catch (NoSuchAlgorithmException e) { log.warn("No MD5 digest algorithm available. Unable to calculate " + "checksum and verify data integrity.", e); } } if (metadata.getContentType() == null) { /* * Default to the "application/octet-stream" if the user hasn't * specified a content type. */ metadata.setContentType(Mimetypes.MIMETYPE_OCTET_STREAM); } populateRequestMetadata(request, metadata); request.setContent(input); if (putObjectRequest instanceof UpdateObjectRequest) { request.addHeader(Headers.RANGE, "bytes=" + ((UpdateObjectRequest) putObjectRequest).getUpdateRange()); } ObjectMetadata returnedMetadata = null; try { returnedMetadata = invoke(request, new S3MetadataResponseHandler(), bucketName, key); } catch (AmazonClientException ace) { fireProgressEvent(progressListenerCallbackExecutor, com.amazonaws.event.ProgressEvent.FAILED_EVENT_CODE); throw ace; } finally { try { input.close(); } catch (Exception e) { log.warn("Unable to cleanly close input stream: " + e.getMessage(), e); } } String contentMd5 = metadata.getContentMD5(); if (md5DigestStream != null) { contentMd5 = BinaryUtils.toBase64(md5DigestStream.getMd5Digest()); } // Can't verify MD5 on appends/update (yet). if (!(putObjectRequest instanceof UpdateObjectRequest)) { if (returnedMetadata != null && contentMd5 != null) { byte[] clientSideHash = BinaryUtils.fromBase64(contentMd5); byte[] serverSideHash = BinaryUtils.fromHex(returnedMetadata.getETag()); if (!Arrays.equals(clientSideHash, serverSideHash)) { fireProgressEvent(progressListenerCallbackExecutor, com.amazonaws.event.ProgressEvent.FAILED_EVENT_CODE); throw new AmazonClientException("Unable to verify integrity of data upload. " + "Client calculated content hash didn't match hash calculated by Amazon S3. " + "You may need to delete the data stored in Amazon S3."); } } } fireProgressEvent(progressListenerCallbackExecutor, com.amazonaws.event.ProgressEvent.COMPLETED_EVENT_CODE); return returnedMetadata; }
From source file:io.konig.camel.aws.s3.DeleteObjectEndpoint.java
License:Apache License
public Exchange createExchange(ExchangePattern pattern, final S3Object s3Object) { LOG.trace("Getting object with key [{}] from bucket [{}]...", s3Object.getKey(), s3Object.getBucketName()); ObjectMetadata objectMetadata = s3Object.getObjectMetadata(); LOG.trace("Got object [{}]", s3Object); Exchange exchange = super.createExchange(pattern); Message message = exchange.getIn();/*from w w w.j a v a2 s. c om*/ if (configuration.isIncludeBody()) { message.setBody(s3Object.getObjectContent()); } else { message.setBody(null); } message.setHeader(S3Constants.KEY, s3Object.getKey()); message.setHeader(S3Constants.BUCKET_NAME, s3Object.getBucketName()); message.setHeader(S3Constants.E_TAG, objectMetadata.getETag()); message.setHeader(S3Constants.LAST_MODIFIED, objectMetadata.getLastModified()); message.setHeader(S3Constants.VERSION_ID, objectMetadata.getVersionId()); message.setHeader(S3Constants.CONTENT_TYPE, objectMetadata.getContentType()); message.setHeader(S3Constants.CONTENT_MD5, objectMetadata.getContentMD5()); message.setHeader(S3Constants.CONTENT_LENGTH, objectMetadata.getContentLength()); message.setHeader(S3Constants.CONTENT_ENCODING, objectMetadata.getContentEncoding()); message.setHeader(S3Constants.CONTENT_DISPOSITION, objectMetadata.getContentDisposition()); message.setHeader(S3Constants.CACHE_CONTROL, objectMetadata.getCacheControl()); message.setHeader(S3Constants.S3_HEADERS, objectMetadata.getRawMetadata()); message.setHeader(S3Constants.SERVER_SIDE_ENCRYPTION, objectMetadata.getSSEAlgorithm()); message.setHeader(S3Constants.USER_METADATA, objectMetadata.getUserMetadata()); message.setHeader(S3Constants.EXPIRATION_TIME, objectMetadata.getExpirationTime()); message.setHeader(S3Constants.REPLICATION_STATUS, objectMetadata.getReplicationStatus()); message.setHeader(S3Constants.STORAGE_CLASS, objectMetadata.getStorageClass()); /** * If includeBody != true, it is safe to close the object here. If * includeBody == true, the caller is responsible for closing the stream * and object once the body has been fully consumed. As of 2.17, the * consumer does not close the stream or object on commit. */ if (!configuration.isIncludeBody()) { IOHelper.close(s3Object); } else { if (configuration.isAutocloseBody()) { exchange.addOnCompletion(new SynchronizationAdapter() { @Override public void onDone(Exchange exchange) { IOHelper.close(s3Object); } }); } } return exchange; }
From source file:org.apache.camel.component.aws.s3.S3Endpoint.java
License:Apache License
public Exchange createExchange(ExchangePattern pattern, S3Object s3Object) { LOG.trace("Getting object with key [{}] from bucket [{}]...", s3Object.getKey(), s3Object.getBucketName()); ObjectMetadata objectMetadata = s3Object.getObjectMetadata(); LOG.trace("Got object [{}]", s3Object); Exchange exchange = new DefaultExchange(this, pattern); Message message = exchange.getIn();//from w w w.j ava 2 s . c o m message.setBody(s3Object.getObjectContent()); message.setHeader(S3Constants.KEY, s3Object.getKey()); message.setHeader(S3Constants.BUCKET_NAME, s3Object.getBucketName()); message.setHeader(S3Constants.E_TAG, objectMetadata.getETag()); message.setHeader(S3Constants.LAST_MODIFIED, objectMetadata.getLastModified()); message.setHeader(S3Constants.VERSION_ID, objectMetadata.getVersionId()); message.setHeader(S3Constants.CONTENT_TYPE, objectMetadata.getContentType()); message.setHeader(S3Constants.CONTENT_MD5, objectMetadata.getContentMD5()); message.setHeader(S3Constants.CONTENT_LENGTH, objectMetadata.getContentLength()); message.setHeader(S3Constants.CONTENT_ENCODING, objectMetadata.getContentEncoding()); message.setHeader(S3Constants.CONTENT_DISPOSITION, objectMetadata.getContentDisposition()); message.setHeader(S3Constants.CACHE_CONTROL, objectMetadata.getCacheControl()); return exchange; }
From source file:org.apache.nifi.processors.aws.s3.FetchS3Object.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { FlowFile flowFile = session.get();/* www .j av a 2 s. c om*/ if (flowFile == null) { return; } final long startNanos = System.nanoTime(); final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue(); final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue(); final String versionId = context.getProperty(VERSION_ID).evaluateAttributeExpressions(flowFile).getValue(); final AmazonS3 client = getClient(); final GetObjectRequest request; if (versionId == null) { request = new GetObjectRequest(bucket, key); } else { request = new GetObjectRequest(bucket, key, versionId); } final Map<String, String> attributes = new HashMap<>(); try (final S3Object s3Object = client.getObject(request)) { flowFile = session.importFrom(s3Object.getObjectContent(), flowFile); attributes.put("s3.bucket", s3Object.getBucketName()); final ObjectMetadata metadata = s3Object.getObjectMetadata(); if (metadata.getContentDisposition() != null) { final String fullyQualified = metadata.getContentDisposition(); final int lastSlash = fullyQualified.lastIndexOf("/"); if (lastSlash > -1 && lastSlash < fullyQualified.length() - 1) { attributes.put(CoreAttributes.PATH.key(), fullyQualified.substring(0, lastSlash)); attributes.put(CoreAttributes.ABSOLUTE_PATH.key(), fullyQualified); attributes.put(CoreAttributes.FILENAME.key(), fullyQualified.substring(lastSlash + 1)); } else { attributes.put(CoreAttributes.FILENAME.key(), metadata.getContentDisposition()); } } if (metadata.getContentMD5() != null) { attributes.put("hash.value", metadata.getContentMD5()); attributes.put("hash.algorithm", "MD5"); } if (metadata.getContentType() != null) { attributes.put(CoreAttributes.MIME_TYPE.key(), metadata.getContentType()); } if (metadata.getETag() != null) { attributes.put("s3.etag", metadata.getETag()); } if (metadata.getExpirationTime() != null) { attributes.put("s3.expirationTime", String.valueOf(metadata.getExpirationTime().getTime())); } if (metadata.getExpirationTimeRuleId() != null) { attributes.put("s3.expirationTimeRuleId", metadata.getExpirationTimeRuleId()); } if (metadata.getUserMetadata() != null) { attributes.putAll(metadata.getUserMetadata()); } if (metadata.getSSEAlgorithm() != null) { attributes.put("s3.sseAlgorithm", metadata.getSSEAlgorithm()); } if (metadata.getVersionId() != null) { attributes.put("s3.version", metadata.getVersionId()); } } catch (final IOException | AmazonClientException ioe) { getLogger().error("Failed to retrieve S3 Object for {}; routing to failure", new Object[] { flowFile, ioe }); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); return; } if (!attributes.isEmpty()) { flowFile = session.putAllAttributes(flowFile, attributes); } session.transfer(flowFile, REL_SUCCESS); final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); getLogger().info("Successfully retrieved S3 Object for {} in {} millis; routing to success", new Object[] { flowFile, transferMillis }); session.getProvenanceReporter().fetch(flowFile, "http://" + bucket + ".amazonaws.com/" + key, transferMillis); }
From source file:org.springframework.integration.aws.outbound.S3MessageHandler.java
License:Apache License
private Transfer upload(Message<?> requestMessage) { Object payload = requestMessage.getPayload(); String bucketName = obtainBucket(requestMessage); String key = null;/*from ww w . j a va 2 s . com*/ if (this.keyExpression != null) { key = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class); } if (payload instanceof File && ((File) payload).isDirectory()) { File fileToUpload = (File) payload; if (key == null) { key = fileToUpload.getName(); } return this.transferManager.uploadDirectory(bucketName, key, fileToUpload, true, new MessageHeadersObjectMetadataProvider(requestMessage.getHeaders())); } else { ObjectMetadata metadata = new ObjectMetadata(); if (this.uploadMetadataProvider != null) { this.uploadMetadataProvider.populateMetadata(metadata, requestMessage); } InputStream inputStream; if (payload instanceof InputStream) { inputStream = (InputStream) payload; } else if (payload instanceof File) { File fileToUpload = (File) payload; if (key == null) { key = fileToUpload.getName(); } try { inputStream = new FileInputStream(fileToUpload); if (metadata.getContentLength() == 0) { metadata.setContentLength(fileToUpload.length()); } if (metadata.getContentType() == null) { metadata.setContentType(Mimetypes.getInstance().getMimetype(fileToUpload)); } } catch (FileNotFoundException e) { throw new AmazonClientException(e); } } else if (payload instanceof byte[]) { inputStream = new ByteArrayInputStream((byte[]) payload); } else { throw new IllegalArgumentException("Unsupported payload type: [" + payload.getClass() + "]. The only supported payloads for the upload request are " + "java.io.File, java.io.InputStream, byte[] and PutObjectRequest."); } Assert.state(key != null, "The 'keyExpression' must not be null for non-File payloads and can't evaluate to null. " + "Root object is: " + requestMessage); if (key == null) { if (this.keyExpression != null) { throw new IllegalStateException( "The 'keyExpression' [" + this.keyExpression.getExpressionString() + "] must not evaluate to null. Root object is: " + requestMessage); } else { throw new IllegalStateException("Specify a 'keyExpression' for non-java.io.File payloads"); } } if (metadata.getContentMD5() == null) { String contentMd5 = null; try { contentMd5 = Md5Utils.md5AsBase64(StreamUtils.copyToByteArray(inputStream)); if (inputStream.markSupported()) { inputStream.reset(); } metadata.setContentMD5(contentMd5); } catch (IOException e) { throw new MessageHandlingException(requestMessage, e); } } PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, inputStream, metadata); S3ProgressListener progressListener = this.s3ProgressListener; if (this.objectAclExpression != null) { Object acl = this.objectAclExpression.getValue(this.evaluationContext, requestMessage); Assert.state(acl instanceof AccessControlList || acl instanceof CannedAccessControlList, "The 'objectAclExpression' [" + this.objectAclExpression.getExpressionString() + "] must evaluate to com.amazonaws.services.s3.model.AccessControlList " + "or must evaluate to com.amazonaws.services.s3.model.CannedAccessControlList. " + "Gotten: [" + acl + "]"); SetObjectAclRequest aclRequest; if (acl instanceof AccessControlList) { aclRequest = new SetObjectAclRequest(bucketName, key, (AccessControlList) acl); } else { aclRequest = new SetObjectAclRequest(bucketName, key, (CannedAccessControlList) acl); } final SetObjectAclRequest theAclRequest = aclRequest; progressListener = new S3ProgressListener() { @Override public void onPersistableTransfer(PersistableTransfer persistableTransfer) { } @Override public void progressChanged(ProgressEvent progressEvent) { if (ProgressEventType.TRANSFER_COMPLETED_EVENT.equals(progressEvent.getEventType())) { S3MessageHandler.this.transferManager.getAmazonS3Client().setObjectAcl(theAclRequest); } } }; if (this.s3ProgressListener != null) { progressListener = new S3ProgressListenerChain(this.s3ProgressListener, progressListener); } } if (progressListener != null) { return this.transferManager.upload(putObjectRequest, progressListener); } else { return this.transferManager.upload(putObjectRequest); } } }
From source file:org.xmlsh.aws.util.AWSS3Command.java
License:BSD License
protected void writeMeta(ObjectMetadata m) throws InvalidArgumentException, XMLStreamException, SaxonApiException { mLogger.entry(m);//from w ww.ja v a2s. c om startElement(sMetaDataElem); attribute("cache-control", m.getCacheControl()); attribute("content-disposition", m.getContentDisposition()); attribute("content-encoding", m.getContentEncoding()); attribute("md5", m.getContentMD5()); attribute("etag", m.getETag()); attribute("version-id", m.getVersionId()); attribute("content-length", String.valueOf(m.getContentLength())); attribute("last-modified", Util.formatXSDateTime(m.getLastModified())); attribute("expiration-time", Util.formatXSDateTime(m.getExpirationTime())); attribute("expiration-time-rule-id", m.getExpirationTimeRuleId()); attribute("httpExpiresDate", Util.formatXSDateTime(m.getHttpExpiresDate())); attribute("ongoingRestore", m.getOngoingRestore()); attribute("restore-expiration-time", Util.formatXSDateTime(m.getRestoreExpirationTime())); attribute("instance-length", m.getInstanceLength()); attribute("sse-algorithm", m.getSSEAlgorithm()); attribute("sse-aws-kms-key-id", m.getSSEAwsKmsKeyId()); attribute("sse-customer-algorithm", m.getSSECustomerAlgorithm()); attribute("sse-customer-key-md5", m.getSSECustomerKeyMd5()); attribute("storage-class", m.getStorageClass()); startElement("user-metadata"); for (Entry<String, String> user : m.getUserMetadata().entrySet()) { startElement(sUserMetaDataElem); attribute("name", user.getKey()); attribute("value", user.getValue()); endElement(); } endElement(); endElement(); mLogger.exit(); }