List of usage examples for com.amazonaws.services.s3.model ObjectMetadata setSSEAlgorithm
@Override public void setSSEAlgorithm(String algorithm)
From source file:alluxio.underfs.s3a.S3ALowLevelOutputStream.java
License:Apache License
/** * Initializes multipart upload.//from w ww . j av a 2 s. c o m */ private void initMultiPartUpload() throws IOException { // Generate the object metadata by setting server side encryption, md5 checksum, // and encoding as octet stream since no assumptions are made about the file type ObjectMetadata meta = new ObjectMetadata(); if (mSseEnabled) { meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); } if (mHash != null) { meta.setContentMD5(Base64.encodeAsString(mHash.digest())); } meta.setContentType(Mimetypes.MIMETYPE_OCTET_STREAM); AmazonClientException lastException; InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(mBucketName, mKey) .withObjectMetadata(meta); do { try { mUploadId = mClient.initiateMultipartUpload(initRequest).getUploadId(); return; } catch (AmazonClientException e) { lastException = e; } } while (mRetryPolicy.attempt()); // This point is only reached if the operation failed more // than the allowed retry count throw new IOException("Unable to init multipart upload to " + mKey, lastException); }
From source file:alluxio.underfs.s3a.S3AOutputStream.java
License:Apache License
@Override public void close() throws IOException { if (mClosed) { return;//from ww w.j av a2 s .c o m } mLocalOutputStream.close(); try { // Generate the object metadata by setting server side encryption, md5 checksum, the file // length, and encoding as octet stream since no assumptions are made about the file type ObjectMetadata meta = new ObjectMetadata(); if (SSE_ENABLED) { meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); } if (mHash != null) { meta.setContentMD5(new String(Base64.encode(mHash.digest()))); } meta.setContentLength(mFile.length()); meta.setContentEncoding(Mimetypes.MIMETYPE_OCTET_STREAM); // Generate the put request and wait for the transfer manager to complete the upload, then // delete the temporary file on the local machine PutObjectRequest putReq = new PutObjectRequest(mBucketName, mKey, mFile).withMetadata(meta); mManager.upload(putReq).waitForUploadResult(); if (!mFile.delete()) { LOG.error("Failed to delete temporary file @ {}", mFile.getPath()); } } catch (Exception e) { LOG.error("Failed to upload {}. Temporary file @ {}", mKey, mFile.getPath()); throw new IOException(e); } // Set the closed flag, close can be retried until mFile.delete is called successfully mClosed = true; }
From source file:alluxio.underfs.s3a.S3AUnderFileSystem.java
License:Apache License
/** * Copies an object to another key./*from w ww .ja va 2 s. c om*/ * * @param src the source key to copy * @param dst the destination key to copy to * @return true if the operation was successful, false otherwise */ private boolean copy(String src, String dst) { src = stripPrefixIfPresent(src); dst = stripPrefixIfPresent(dst); LOG.debug("Copying {} to {}", src, dst); // Retry copy for a few times, in case some AWS internal errors happened during copy. int retries = 3; for (int i = 0; i < retries; i++) { try { CopyObjectRequest request = new CopyObjectRequest(mBucketName, src, mBucketName, dst); if (Configuration.getBoolean(PropertyKey.UNDERFS_S3A_SERVER_SIDE_ENCRYPTION_ENABLED)) { ObjectMetadata meta = new ObjectMetadata(); meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); request.setNewObjectMetadata(meta); } mManager.copy(request).waitForCopyResult(); return true; } catch (AmazonClientException | InterruptedException e) { LOG.error("Failed to copy file {} to {}", src, dst, e); if (i != retries - 1) { LOG.error("Retrying copying file {} to {}", src, dst); } } } LOG.error("Failed to copy file {} to {}, after {} retries", src, dst, retries); return false; }
From source file:cloudExplorer.Put.java
License:Open Source License
public void run() { try {//from w w w. java 2s. c o m AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key); AmazonS3 s3Client = new AmazonS3Client(credentials, new ClientConfiguration().withSignerOverride("S3SignerType")); s3Client.setEndpoint(endpoint); TransferManager tx = new TransferManager(s3Client); File file = new File(what); PutObjectRequest putRequest; if (!rrs) { putRequest = new PutObjectRequest(bucket, ObjectKey, file); } else { putRequest = new PutObjectRequest(bucket, ObjectKey, file) .withStorageClass(StorageClass.ReducedRedundancy); } MimetypesFileTypeMap mimeTypesMap = new MimetypesFileTypeMap(); String mimeType = mimeTypesMap.getContentType(file); mimeType = mimeTypesMap.getContentType(file); ObjectMetadata objectMetadata = new ObjectMetadata(); if (encrypt) { objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); } if ((ObjectKey.contains(".html")) || ObjectKey.contains(".txt")) { objectMetadata.setContentType("text/html"); } else { objectMetadata.setContentType(mimeType); } long t1 = System.currentTimeMillis(); putRequest.setMetadata(objectMetadata); Upload myUpload = tx.upload(putRequest); myUpload.waitForCompletion(); tx.shutdownNow(); long t2 = System.currentTimeMillis(); long diff = t2 - t1; if (!mainFrame.perf) { if (terminal) { System.out.print("\nUploaded object: " + ObjectKey + " in " + diff / 1000 + " second(s).\n"); } else { mainFrame.jTextArea1 .append("\nUploaded object: " + ObjectKey + " in " + diff / 1000 + " second(s)."); } } } catch (AmazonServiceException ase) { if (NewJFrame.gui) { mainFrame.jTextArea1.append("\n\nError Message: " + ase.getMessage()); mainFrame.jTextArea1.append("\nHTTP Status Code: " + ase.getStatusCode()); mainFrame.jTextArea1.append("\nAWS Error Code: " + ase.getErrorCode()); mainFrame.jTextArea1.append("\nError Type: " + ase.getErrorType()); mainFrame.jTextArea1.append("\nRequest ID: " + ase.getRequestId()); calibrate(); } else { System.out.print("\n\nError Message: " + ase.getMessage()); System.out.print("\nHTTP Status Code: " + ase.getStatusCode()); System.out.print("\nAWS Error Code: " + ase.getErrorCode()); System.out.print("\nError Type: " + ase.getErrorType()); System.out.print("\nRequest ID: " + ase.getRequestId()); } } catch (Exception put) { } calibrate(); }
From source file:com.altoukhov.svsync.fileviews.S3FileSpace.java
License:Apache License
@Override public boolean createDirectory(String path) { path = trimPath(path);/*from w w w . j a v a 2 s .c o m*/ try { ObjectMetadata meta = new ObjectMetadata(); meta.setContentLength(0); meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); s3.putObject(bucketName, toAbsoluteDirPath(path), new ByteArrayInputStream(new byte[0]), meta); } catch (AmazonClientException ex) { return false; } return true; }
From source file:com.altoukhov.svsync.fileviews.S3FileSpace.java
License:Apache License
@Override public boolean writeFile(InputStream fileStream, FileSnapshot file) { if (fileStream == null) return false; if (file.isLargeFile()) { return writeLargeFile(fileStream, file); }/* w w w . j ava 2 s. com*/ try { ObjectMetadata meta = new ObjectMetadata(); meta.setContentLength(file.getFileSize()); meta.getUserMetadata().put("lmd", file.getModifiedTimestamp().toDate().getTime() + ""); meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); s3.putObject(bucketName, toAbsoluteFilePath(file.getRelativePath()), fileStream, meta); } catch (AmazonClientException ex) { return false; } return true; }
From source file:com.altoukhov.svsync.fileviews.S3FileSpace.java
License:Apache License
public boolean writeLargeFile(InputStream fileStream, FileSnapshot file) { if (fileStream == null) return false; try {//from w ww . j a v a 2s.c o m ObjectMetadata meta = new ObjectMetadata(); meta.setContentLength(file.getFileSize()); meta.getUserMetadata().put("lmd", file.getModifiedTimestamp().toDate().getTime() + ""); meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); List<PartETag> partTags = new ArrayList<>(); String fileKey = toAbsoluteFilePath(file.getRelativePath()); InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, fileKey, meta); InitiateMultipartUploadResult result = s3.initiateMultipartUpload(request); long contentLength = file.getFileSize(); long partSize = 256 * 1024 * 1024; try { // Uploading the file, part by part. long filePosition = 0; for (int i = 1; filePosition < contentLength; i++) { partSize = Math.min(partSize, (contentLength - filePosition)); // Creating the request for a part upload UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(bucketName) .withKey(fileKey).withUploadId(result.getUploadId()).withPartNumber(i) .withInputStream(fileStream).withPartSize(partSize); // Upload part and add response to the result list. partTags.add(s3.uploadPart(uploadRequest).getPartETag()); filePosition += partSize; System.out.println("Uploaded " + Utils.readableFileSize(filePosition) + " out of " + Utils.readableFileSize(contentLength)); } } catch (Exception e) { System.out.println("UploadPartRequest failed: " + e.getMessage()); s3.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, fileKey, result.getUploadId())); return false; } s3.completeMultipartUpload( new CompleteMultipartUploadRequest(bucketName, fileKey, result.getUploadId(), partTags)); } catch (AmazonClientException ex) { System.out.println("Upload failed: " + ex.getMessage()); return false; } return true; }
From source file:com.ge.predix.solsvc.blobstore.bootstrap.BlobstoreClientImpl.java
License:Apache License
/** * Adds a new Blob to the binded bucket in the Object Store * * @param obj S3Object to be added/*w ww .java 2 s. c o m*/ */ @Override public String saveBlob(S3Object obj) { if (obj == null) { this.log.error("put(): Empty file provided"); //$NON-NLS-1$ throw new RuntimeException("File is null"); //$NON-NLS-1$ } List<PartETag> partETags = new ArrayList<>(); String bucket = this.blobstoreConfig.getBucketName(); InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, obj.getKey()); InitiateMultipartUploadResult initResponse = this.s3Client.initiateMultipartUpload(initRequest); try (InputStream is = obj.getObjectContent();) { int i = 1; int currentPartSize = 0; ByteArrayOutputStream tempBuffer = new ByteArrayOutputStream(); int byteValue; while ((byteValue = is.read()) != -1) { tempBuffer.write(byteValue); currentPartSize = tempBuffer.size(); if (currentPartSize == (50 * 1024 * 1024)) //make this a const { byte[] b = tempBuffer.toByteArray(); ByteArrayInputStream byteStream = new ByteArrayInputStream(b); UploadPartRequest uploadPartRequest = new UploadPartRequest().withBucketName(bucket) .withKey(obj.getKey()).withUploadId(initResponse.getUploadId()).withPartNumber(i++) .withInputStream(byteStream).withPartSize(currentPartSize); partETags.add(this.s3Client.uploadPart(uploadPartRequest).getPartETag()); tempBuffer.reset(); } } this.log.info("currentPartSize: " + currentPartSize); //$NON-NLS-1$ ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(currentPartSize); if (this.enableSSE) { objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); } obj.setObjectMetadata(objectMetadata); if (i == 1 && currentPartSize < (5 * 1024 * 1024)) // make this a const { this.s3Client.abortMultipartUpload( new AbortMultipartUploadRequest(bucket, obj.getKey(), initResponse.getUploadId())); byte[] b = tempBuffer.toByteArray(); ByteArrayInputStream byteStream = new ByteArrayInputStream(b); objectMetadata.setContentType(getContentType(b)); if (this.enableSSE) { objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); } obj.setObjectMetadata(objectMetadata); PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, obj.getKey(), byteStream, obj.getObjectMetadata()); this.s3Client.putObject(putObjectRequest); ObjectMetadata meta = this.s3Client.getObjectMetadata(bucket, obj.getKey()); Map<String, Object> headers = meta.getRawMetadata(); for (Map.Entry<String, Object> entry : headers.entrySet()) { this.log.info("Object Metadata -- " + entry.getKey() + ": " + entry.getValue().toString()); //$NON-NLS-1$ //$NON-NLS-2$ } return initResponse.getUploadId(); } if (currentPartSize > 0 && currentPartSize <= (50 * 1024 * 1024)) // make this a const { byte[] b = tempBuffer.toByteArray(); ByteArrayInputStream byteStream = new ByteArrayInputStream(b); this.log.info("currentPartSize: " + currentPartSize); //$NON-NLS-1$ this.log.info("byteArray: " + b); //$NON-NLS-1$ UploadPartRequest uploadPartRequest = new UploadPartRequest().withBucketName(bucket) .withKey(obj.getKey()).withUploadId(initResponse.getUploadId()).withPartNumber(i) .withInputStream(byteStream).withPartSize(currentPartSize); partETags.add(this.s3Client.uploadPart(uploadPartRequest).getPartETag()); } CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest() .withBucketName(bucket).withPartETags(partETags).withUploadId(initResponse.getUploadId()) .withKey(obj.getKey()); this.s3Client.completeMultipartUpload(completeMultipartUploadRequest); return initResponse.getUploadId(); } catch (Exception e) { this.log.error("put(): Exception occurred in put(): " + e.getMessage()); //$NON-NLS-1$ this.s3Client.abortMultipartUpload( new AbortMultipartUploadRequest(bucket, obj.getKey(), initResponse.getUploadId())); throw new RuntimeException("put(): Exception occurred in put(): ", e); //$NON-NLS-1$ } }
From source file:com.pinterest.secor.uploader.S3UploadManager.java
License:Apache License
private void enableS3Encryption(PutObjectRequest uploadRequest) { ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); uploadRequest.setMetadata(objectMetadata); }
From source file:com.streamsets.pipeline.lib.aws.s3.S3Accessor.java
License:Apache License
public EncryptionMetadataBuilder createEncryptionMetadataBuilder() { return () -> { ObjectMetadata metadata = null; if (sseConfigs != null) { switch (sseConfigs.getEncryption()) { case NONE: metadata = null;// www.j ava 2 s . c om break; case S3: metadata = new ObjectMetadata(); metadata.setSSEAlgorithm(SSEAlgorithm.AES256.getAlgorithm()); break; case KMS: metadata = new ObjectMetadata(); metadata.setSSEAlgorithm(SSEAlgorithm.KMS.getAlgorithm()); metadata.setHeader(Headers.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEYID, sseConfigs.getKmsKeyId().get()); metadata.setHeader("x-amz-server-side-encryption-context", sseConfigs.getEncryptionContext().entrySet().stream().collect( Collectors.toMap(e -> e.getKey(), e -> Caller.call(() -> e.getValue().get())))); break; case CUSTOMER: metadata = new ObjectMetadata(); metadata.setSSECustomerAlgorithm(SSEAlgorithm.AES256.getAlgorithm()); metadata.setHeader(Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, sseConfigs.getCustomerKey().get()); metadata.setHeader(Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, sseConfigs.getCustomerKeyMd5().get()); break; default: throw new IllegalArgumentException( String.format("Invalid encryption option '%s'", sseConfigs.getEncryption())); } } return metadata; }; }