Example usage for com.amazonaws.services.s3.model CopyPartResult getETag

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model CopyPartResult getETag.

Prototype

public String getETag() 

Source Link

Document

Gets the ETag value for the new part that was created in the associated CopyPartRequest .

Usage

From source file:com.proofpoint.event.collector.combiner.S3StorageSystem.java

License:Apache License

private StoredObject createCombinedObjectLarge(CombinedStoredObject combinedObject) {
    URI location = combinedObject.getLocation();
    log.info("starting multipart upload: %s", location);

    String bucket = getS3Bucket(location);
    String key = getS3ObjectKey(location);

    String uploadId = s3Service.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key))
            .getUploadId();//from   w w w. j a va 2 s. co  m

    try {
        List<PartETag> parts = newArrayList();
        int partNumber = 1;
        for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
            CopyPartResult part = s3Service.copyPart(new CopyPartRequest().withUploadId(uploadId)
                    .withPartNumber(partNumber).withDestinationBucketName(bucket).withDestinationKey(key)
                    .withSourceBucketName(getS3Bucket(newCombinedObjectPart.getLocation()))
                    .withSourceKey(getS3ObjectKey(newCombinedObjectPart.getLocation())));
            parts.add(new PartETag(partNumber, part.getETag()));
            partNumber++;
        }

        String etag = s3Service
                .completeMultipartUpload(new CompleteMultipartUploadRequest(bucket, key, uploadId, parts))
                .getETag();

        ObjectMetadata newObject = s3Service.getObjectMetadata(bucket, key);
        log.info("completed multipart upload: %s", location);

        if (!etag.equals(newObject.getETag())) {
            // this might happen in rare cases due to S3's eventual consistency
            throw new IllegalStateException("completed etag is different from combined object etag");
        }

        return updateStoredObject(location, newObject);
    } catch (AmazonClientException e) {
        try {
            s3Service.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
        } catch (AmazonClientException ignored) {
        }
        throw Throwables.propagate(e);
    }
}

---

From source file:com.tango.BucketSyncer.KeyJobs.S32S3MultipartKeyCopyJob.java

License:Apache License

private List<PartETag> getETags(List<CopyPartResult> copyResponses) {
    List<PartETag> eTags = new ArrayList<PartETag>();
    for (CopyPartResult response : copyResponses) {
        eTags.add(new PartETag(response.getPartNumber(), response.getETag()));
    }/*from   w  w w. j av a2s  .  c o  m*/
    return eTags;
}

---

From source file:com.upplication.s3fs.AmazonS3Client.java

License:Open Source License

public void multipartCopyObject(S3Path s3Source, S3Path s3Target, Long objectSize, S3MultipartOptions opts) {

    final String sourceBucketName = s3Source.getBucket();
    final String sourceObjectKey = s3Source.getKey();
    final String targetBucketName = s3Target.getBucket();
    final String targetObjectKey = s3Target.getKey();

    // Step 2: Initialize
    InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(targetBucketName,
            targetObjectKey);//from   w ww.j a  v a  2  s .  c o m

    InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initiateRequest);

    // Step 3: Save upload Id.
    String uploadId = initResult.getUploadId();

    // Get object size.
    if (objectSize == null) {
        GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(sourceBucketName,
                sourceObjectKey);
        ObjectMetadata metadataResult = client.getObjectMetadata(metadataRequest);
        objectSize = metadataResult.getContentLength(); // in bytes
    }

    final int partSize = opts.getChunkSize(objectSize);
    ExecutorService executor = S3OutputStream.getOrCreateExecutor(opts.getMaxThreads());
    List<Callable<CopyPartResult>> copyPartRequests = new ArrayList<>();

    // Step 4. create copy part requests
    long bytePosition = 0;
    for (int i = 1; bytePosition < objectSize; i++) {
        long lastPosition = bytePosition + partSize - 1 >= objectSize ? objectSize - 1
                : bytePosition + partSize - 1;

        CopyPartRequest copyRequest = new CopyPartRequest().withDestinationBucketName(targetBucketName)
                .withDestinationKey(targetObjectKey).withSourceBucketName(sourceBucketName)
                .withSourceKey(sourceObjectKey).withUploadId(uploadId).withFirstByte(bytePosition)
                .withLastByte(lastPosition).withPartNumber(i);

        copyPartRequests.add(copyPart(client, copyRequest, opts));
        bytePosition += partSize;
    }

    log.trace(
            "Starting multipart copy from: {} to {} -- uploadId={}; objectSize={}; chunkSize={}; numOfChunks={}",
            s3Source, s3Target, uploadId, objectSize, partSize, copyPartRequests.size());

    List<PartETag> etags = new ArrayList<>();
    List<Future<CopyPartResult>> responses;
    try {
        // Step 5. Start parallel parts copy
        responses = executor.invokeAll(copyPartRequests);

        // Step 6. Fetch all results
        for (Future<CopyPartResult> response : responses) {
            CopyPartResult result = response.get();
            etags.add(new PartETag(result.getPartNumber(), result.getETag()));
        }
    } catch (Exception e) {
        throw new IllegalStateException("Multipart copy reported an unexpected error -- uploadId=" + uploadId,
                e);
    }

    // Step 7. Complete copy operation
    CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucketName,
            targetObjectKey, initResult.getUploadId(), etags);

    log.trace("Completing multipart copy uploadId={}", uploadId);
    client.completeMultipartUpload(completeRequest);
}

---