List of usage examples for com.amazonaws.services.s3.model CopyPartRequest CopyPartRequest
CopyPartRequest
From source file:com.proofpoint.event.collector.combiner.S3StorageSystem.java
License:Apache License
private StoredObject createCombinedObjectLarge(CombinedStoredObject combinedObject) { URI location = combinedObject.getLocation(); log.info("starting multipart upload: %s", location); String bucket = getS3Bucket(location); String key = getS3ObjectKey(location); String uploadId = s3Service.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key)) .getUploadId();// w ww.java2 s. c o m try { List<PartETag> parts = newArrayList(); int partNumber = 1; for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) { CopyPartResult part = s3Service.copyPart(new CopyPartRequest().withUploadId(uploadId) .withPartNumber(partNumber).withDestinationBucketName(bucket).withDestinationKey(key) .withSourceBucketName(getS3Bucket(newCombinedObjectPart.getLocation())) .withSourceKey(getS3ObjectKey(newCombinedObjectPart.getLocation()))); parts.add(new PartETag(partNumber, part.getETag())); partNumber++; } String etag = s3Service .completeMultipartUpload(new CompleteMultipartUploadRequest(bucket, key, uploadId, parts)) .getETag(); ObjectMetadata newObject = s3Service.getObjectMetadata(bucket, key); log.info("completed multipart upload: %s", location); if (!etag.equals(newObject.getETag())) { // this might happen in rare cases due to S3's eventual consistency throw new IllegalStateException("completed etag is different from combined object etag"); } return updateStoredObject(location, newObject); } catch (AmazonClientException e) { try { s3Service.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId)); } catch (AmazonClientException ignored) { } throw Throwables.propagate(e); } }
From source file:com.tango.BucketSyncer.KeyJobs.S32S3MultipartKeyCopyJob.java
License:Apache License
@Override boolean keyCopied(ObjectMetadata sourceMetadata, AccessControlList objectAcl) { long objectSize = summary.getSize(); MirrorOptions options = context.getOptions(); String sourceBucketName = options.getSourceBucket(); int maxPartRetries = options.getMaxRetries(); String targetBucketName = options.getDestinationBucket(); List<CopyPartResult> copyResponses = new ArrayList<CopyPartResult>(); if (options.isVerbose()) { log.info("Initiating multipart upload request for {}", summary.getKey()); }// w ww . j av a2s .c o m InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(targetBucketName, keydest).withObjectMetadata(sourceMetadata); if (options.isCrossAccountCopy()) { initiateRequest.withCannedACL(CannedAccessControlList.BucketOwnerFullControl); } else { initiateRequest.withAccessControlList(objectAcl); } InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initiateRequest); long partSize = options.getUploadPartSize(); long bytePosition = 0; for (int i = 1; bytePosition < objectSize; i++) { long lastByte = bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1; String infoMessage = String.format("Copying: %s to %s", bytePosition, lastByte); if (options.isVerbose()) { log.info(infoMessage); } CopyPartRequest copyRequest = new CopyPartRequest().withDestinationBucketName(targetBucketName) .withDestinationKey(keydest).withSourceBucketName(sourceBucketName) .withSourceKey(summary.getKey()).withUploadId(initResult.getUploadId()) .withFirstByte(bytePosition).withLastByte(lastByte).withPartNumber(i); for (int tries = 1; tries <= maxPartRetries; tries++) { try { if (options.isVerbose()) { log.info("try : {}", tries); } context.getStats().copyCount.incrementAndGet(); CopyPartResult copyPartResult = client.copyPart(copyRequest); copyResponses.add(copyPartResult); if (options.isVerbose()) { log.info("completed {} ", infoMessage); } break; } catch (Exception e) { if (tries == maxPartRetries) { client.abortMultipartUpload(new AbortMultipartUploadRequest(targetBucketName, keydest, initResult.getUploadId())); log.error("Exception while doing multipart copy: {}", e); return false; } } } bytePosition += partSize; } CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucketName, keydest, initResult.getUploadId(), getETags(copyResponses)); client.completeMultipartUpload(completeRequest); if (options.isVerbose()) { log.info("completed multipart request for : {}", summary.getKey()); } context.getStats().bytesCopied.addAndGet(objectSize); return true; }
From source file:com.upplication.s3fs.AmazonS3Client.java
License:Open Source License
public void multipartCopyObject(S3Path s3Source, S3Path s3Target, Long objectSize, S3MultipartOptions opts) { final String sourceBucketName = s3Source.getBucket(); final String sourceObjectKey = s3Source.getKey(); final String targetBucketName = s3Target.getBucket(); final String targetObjectKey = s3Target.getKey(); // Step 2: Initialize InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(targetBucketName, targetObjectKey);/*from ww w. ja v a 2s . c o m*/ InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initiateRequest); // Step 3: Save upload Id. String uploadId = initResult.getUploadId(); // Get object size. if (objectSize == null) { GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(sourceBucketName, sourceObjectKey); ObjectMetadata metadataResult = client.getObjectMetadata(metadataRequest); objectSize = metadataResult.getContentLength(); // in bytes } final int partSize = opts.getChunkSize(objectSize); ExecutorService executor = S3OutputStream.getOrCreateExecutor(opts.getMaxThreads()); List<Callable<CopyPartResult>> copyPartRequests = new ArrayList<>(); // Step 4. create copy part requests long bytePosition = 0; for (int i = 1; bytePosition < objectSize; i++) { long lastPosition = bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1; CopyPartRequest copyRequest = new CopyPartRequest().withDestinationBucketName(targetBucketName) .withDestinationKey(targetObjectKey).withSourceBucketName(sourceBucketName) .withSourceKey(sourceObjectKey).withUploadId(uploadId).withFirstByte(bytePosition) .withLastByte(lastPosition).withPartNumber(i); copyPartRequests.add(copyPart(client, copyRequest, opts)); bytePosition += partSize; } log.trace( "Starting multipart copy from: {} to {} -- uploadId={}; objectSize={}; chunkSize={}; numOfChunks={}", s3Source, s3Target, uploadId, objectSize, partSize, copyPartRequests.size()); List<PartETag> etags = new ArrayList<>(); List<Future<CopyPartResult>> responses; try { // Step 5. Start parallel parts copy responses = executor.invokeAll(copyPartRequests); // Step 6. Fetch all results for (Future<CopyPartResult> response : responses) { CopyPartResult result = response.get(); etags.add(new PartETag(result.getPartNumber(), result.getETag())); } } catch (Exception e) { throw new IllegalStateException("Multipart copy reported an unexpected error -- uploadId=" + uploadId, e); } // Step 7. Complete copy operation CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucketName, targetObjectKey, initResult.getUploadId(), etags); log.trace("Completing multipart copy uploadId={}", uploadId); client.completeMultipartUpload(completeRequest); }
From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java
License:Apache License
@VisibleForTesting
CompleteMultipartUploadResult multipartCopy(S3ResourceId sourcePath, S3ResourceId destinationPath,
ObjectMetadata sourceObjectMetadata) throws AmazonClientException {
InitiateMultipartUploadRequest initiateUploadRequest = new InitiateMultipartUploadRequest(
destinationPath.getBucket(), destinationPath.getKey()).withStorageClass(options.getS3StorageClass())
.withObjectMetadata(sourceObjectMetadata);
initiateUploadRequest.setSSECustomerKey(options.getSSECustomerKey());
InitiateMultipartUploadResult initiateUploadResult = amazonS3.get()
.initiateMultipartUpload(initiateUploadRequest);
final String uploadId = initiateUploadResult.getUploadId();
List<PartETag> eTags = new ArrayList<>();
final long objectSize = sourceObjectMetadata.getContentLength();
// extra validation in case a caller calls directly S3FileSystem.multipartCopy
// without using S3FileSystem.copy in the future
if (objectSize == 0) {
final CopyPartRequest copyPartRequest = new CopyPartRequest()
.withSourceBucketName(sourcePath.getBucket()).withSourceKey(sourcePath.getKey())
.withDestinationBucketName(destinationPath.getBucket())
.withDestinationKey(destinationPath.getKey()).withUploadId(uploadId).withPartNumber(1);
copyPartRequest.setSourceSSECustomerKey(options.getSSECustomerKey());
copyPartRequest.setDestinationSSECustomerKey(options.getSSECustomerKey());
CopyPartResult copyPartResult = amazonS3.get().copyPart(copyPartRequest);
eTags.add(copyPartResult.getPartETag());
} else {//from w w w .j a v a2s . co m
long bytePosition = 0;
Integer uploadBufferSizeBytes = options.getS3UploadBufferSizeBytes();
// Amazon parts are 1-indexed, not zero-indexed.
for (int partNumber = 1; bytePosition < objectSize; partNumber++) {
final CopyPartRequest copyPartRequest = new CopyPartRequest()
.withSourceBucketName(sourcePath.getBucket()).withSourceKey(sourcePath.getKey())
.withDestinationBucketName(destinationPath.getBucket())
.withDestinationKey(destinationPath.getKey()).withUploadId(uploadId)
.withPartNumber(partNumber).withFirstByte(bytePosition)
.withLastByte(Math.min(objectSize - 1, bytePosition + uploadBufferSizeBytes - 1));
copyPartRequest.setSourceSSECustomerKey(options.getSSECustomerKey());
copyPartRequest.setDestinationSSECustomerKey(options.getSSECustomerKey());
CopyPartResult copyPartResult = amazonS3.get().copyPart(copyPartRequest);
eTags.add(copyPartResult.getPartETag());
bytePosition += uploadBufferSizeBytes;
}
}
CompleteMultipartUploadRequest completeUploadRequest = new CompleteMultipartUploadRequest()
.withBucketName(destinationPath.getBucket()).withKey(destinationPath.getKey())
.withUploadId(uploadId).withPartETags(eTags);
return amazonS3.get().completeMultipartUpload(completeUploadRequest);
}