Example usage for com.amazonaws.services.s3.model PartETag PartETag

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model PartETag PartETag.

Prototype

public PartETag(int partNumber, String eTag) 

Source Link

Document

Constructs an instance of PartETag and sets the part number and ETag.

Usage

From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java

License:Open Source License

@Override
public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMultipartUploadType request)
        throws S3Exception {
    CompleteMultipartUploadResponseType reply = request.getReply();
    User requestUser = getRequestUser(request);
    OsgInternalS3Client internalS3Client = null;

    String bucketName = request.getBucket();
    String key = request.getKey();
    String uploadId = request.getUploadId();
    List<Part> parts = request.getParts();
    List<PartETag> partETags = new ArrayList<>();
    for (Part part : parts) {
        PartETag partETag = new PartETag(part.getPartNumber(), part.getEtag());
        partETags.add(partETag);//from   w  w  w.j av a  2s .c  om
    }
    CompleteMultipartUploadRequest multipartRequest = new CompleteMultipartUploadRequest(bucketName, key,
            uploadId, partETags);
    try {
        internalS3Client = getS3Client(requestUser);
        AmazonS3Client s3Client = internalS3Client.getS3Client();

        CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(multipartRequest);
        reply.setEtag(result.getETag());
        reply.setBucket(bucketName);
        reply.setKey(key);
        reply.setLocation(result.getLocation());
        reply.setLastModified(new Date());
    } catch (AmazonServiceException e) {
        LOG.debug("Error from backend", e);
        throw S3ExceptionMapper.fromAWSJavaSDK(e);
    }
    return reply;
}

---

From source file:com.example.jinux.mydemo.s3.Uploader.java

License:Apache License

private List<PartETag> getCachedPartEtags() {
    List<PartETag> result = new ArrayList<PartETag>();
    // get the cached etags
    ArrayList<String> etags = SharedPreferencesUtils.getStringArrayPref(prefs, s3key + PREFS_ETAGS);
    for (String etagString : etags) {
        String partNum = etagString.substring(0, etagString.indexOf(PREFS_ETAG_SEP));
        String partTag = etagString.substring(etagString.indexOf(PREFS_ETAG_SEP) + 2, etagString.length());

        PartETag etag = new PartETag(Integer.parseInt(partNum), partTag);
        result.add(etag);// w w  w  . ja  v  a  2  s . co  m
    }
    return result;
}

---

From source file:com.proofpoint.event.collector.combiner.S3StorageSystem.java

License:Apache License

private StoredObject createCombinedObjectLarge(CombinedStoredObject combinedObject) {
    URI location = combinedObject.getLocation();
    log.info("starting multipart upload: %s", location);

    String bucket = getS3Bucket(location);
    String key = getS3ObjectKey(location);

    String uploadId = s3Service.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key))
            .getUploadId();/*from   w w w.ja v  a  2 s  .  co  m*/

    try {
        List<PartETag> parts = newArrayList();
        int partNumber = 1;
        for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
            CopyPartResult part = s3Service.copyPart(new CopyPartRequest().withUploadId(uploadId)
                    .withPartNumber(partNumber).withDestinationBucketName(bucket).withDestinationKey(key)
                    .withSourceBucketName(getS3Bucket(newCombinedObjectPart.getLocation()))
                    .withSourceKey(getS3ObjectKey(newCombinedObjectPart.getLocation())));
            parts.add(new PartETag(partNumber, part.getETag()));
            partNumber++;
        }

        String etag = s3Service
                .completeMultipartUpload(new CompleteMultipartUploadRequest(bucket, key, uploadId, parts))
                .getETag();

        ObjectMetadata newObject = s3Service.getObjectMetadata(bucket, key);
        log.info("completed multipart upload: %s", location);

        if (!etag.equals(newObject.getETag())) {
            // this might happen in rare cases due to S3's eventual consistency
            throw new IllegalStateException("completed etag is different from combined object etag");
        }

        return updateStoredObject(location, newObject);
    } catch (AmazonClientException e) {
        try {
            s3Service.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
        } catch (AmazonClientException ignored) {
        }
        throw Throwables.propagate(e);
    }
}

---

From source file:com.tango.BucketSyncer.KeyJobs.S32S3MultipartKeyCopyJob.java

License:Apache License

private List<PartETag> getETags(List<CopyPartResult> copyResponses) {
    List<PartETag> eTags = new ArrayList<PartETag>();
    for (CopyPartResult response : copyResponses) {
        eTags.add(new PartETag(response.getPartNumber(), response.getETag()));
    }//from  w w  w .j  a  va  2 s  . c  om
    return eTags;
}

---

From source file:com.upplication.s3fs.AmazonS3Client.java

License:Open Source License

public void multipartCopyObject(S3Path s3Source, S3Path s3Target, Long objectSize, S3MultipartOptions opts) {

    final String sourceBucketName = s3Source.getBucket();
    final String sourceObjectKey = s3Source.getKey();
    final String targetBucketName = s3Target.getBucket();
    final String targetObjectKey = s3Target.getKey();

    // Step 2: Initialize
    InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(targetBucketName,
            targetObjectKey);//w  w w  .  j  a  v a  2  s .c  o  m

    InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initiateRequest);

    // Step 3: Save upload Id.
    String uploadId = initResult.getUploadId();

    // Get object size.
    if (objectSize == null) {
        GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(sourceBucketName,
                sourceObjectKey);
        ObjectMetadata metadataResult = client.getObjectMetadata(metadataRequest);
        objectSize = metadataResult.getContentLength(); // in bytes
    }

    final int partSize = opts.getChunkSize(objectSize);
    ExecutorService executor = S3OutputStream.getOrCreateExecutor(opts.getMaxThreads());
    List<Callable<CopyPartResult>> copyPartRequests = new ArrayList<>();

    // Step 4. create copy part requests
    long bytePosition = 0;
    for (int i = 1; bytePosition < objectSize; i++) {
        long lastPosition = bytePosition + partSize - 1 >= objectSize ? objectSize - 1
                : bytePosition + partSize - 1;

        CopyPartRequest copyRequest = new CopyPartRequest().withDestinationBucketName(targetBucketName)
                .withDestinationKey(targetObjectKey).withSourceBucketName(sourceBucketName)
                .withSourceKey(sourceObjectKey).withUploadId(uploadId).withFirstByte(bytePosition)
                .withLastByte(lastPosition).withPartNumber(i);

        copyPartRequests.add(copyPart(client, copyRequest, opts));
        bytePosition += partSize;
    }

    log.trace(
            "Starting multipart copy from: {} to {} -- uploadId={}; objectSize={}; chunkSize={}; numOfChunks={}",
            s3Source, s3Target, uploadId, objectSize, partSize, copyPartRequests.size());

    List<PartETag> etags = new ArrayList<>();
    List<Future<CopyPartResult>> responses;
    try {
        // Step 5. Start parallel parts copy
        responses = executor.invokeAll(copyPartRequests);

        // Step 6. Fetch all results
        for (Future<CopyPartResult> response : responses) {
            CopyPartResult result = response.get();
            etags.add(new PartETag(result.getPartNumber(), result.getETag()));
        }
    } catch (Exception e) {
        throw new IllegalStateException("Multipart copy reported an unexpected error -- uploadId=" + uploadId,
                e);
    }

    // Step 7. Complete copy operation
    CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucketName,
            targetObjectKey, initResult.getUploadId(), etags);

    log.trace("Completing multipart copy uploadId={}", uploadId);
    client.completeMultipartUpload(completeRequest);
}

---

From source file:org.apache.apex.malhar.lib.fs.s3.S3BlockUploadOperator.java

License:Apache License

/**
 * Upload the block into S3 bucket./*from w w  w.j  av  a 2  s  .  c o m*/
 * @param tuple block data
 */
protected void uploadBlockIntoS3(AbstractBlockReader.ReaderRecord<Slice> tuple) {
    if (currentWindowId <= windowDataManager.getLargestCompletedWindow()) {
        return;
    }
    // Check whether the block metadata is present for this block
    if (blockIdToFilePath.get(tuple.getBlockId()) == null) {
        if (!waitingTuples.contains(tuple)) {
            waitingTuples.add(tuple);
        }
        return;
    }
    String uniqueBlockId = getUniqueBlockIdFromFile(tuple.getBlockId(),
            blockIdToFilePath.get(tuple.getBlockId()));
    S3BlockMetaData metaData = blockInfo.get(uniqueBlockId);
    // Check whether the file metadata is received
    if (metaData == null) {
        if (!waitingTuples.contains(tuple)) {
            waitingTuples.add(tuple);
        }
        return;
    }
    long partSize = tuple.getRecord().length;
    PartETag partETag = null;
    ByteArrayInputStream bis = new ByteArrayInputStream(tuple.getRecord().buffer);
    // Check if it is a Single block of a file
    if (metaData.isLastBlock && metaData.partNo == 1) {
        ObjectMetadata omd = createObjectMetadata();
        omd.setContentLength(partSize);
        PutObjectResult result = s3Client
                .putObject(new PutObjectRequest(bucketName, metaData.getKeyName(), bis, omd));
        partETag = new PartETag(1, result.getETag());
    } else {
        // Else upload use multi-part feature
        try {
            // Create request to upload a part.
            UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(bucketName)
                    .withKey(metaData.getKeyName()).withUploadId(metaData.getUploadId())
                    .withPartNumber(metaData.getPartNo()).withInputStream(bis).withPartSize(partSize);
            partETag = s3Client.uploadPart(uploadRequest).getPartETag();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    UploadBlockMetadata uploadmetadata = new UploadBlockMetadata(partETag, metaData.getKeyName());
    output.emit(uploadmetadata);
    currentWindowRecoveryState.put(uniqueBlockId, uploadmetadata);
    try {
        bis.close();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:org.apache.flink.fs.s3.common.writer.S3RecoverableSerializer.java

License:Apache License

private static S3Recoverable deserializeV1(byte[] serialized) throws IOException {
    final ByteBuffer bb = ByteBuffer.wrap(serialized).order(ByteOrder.LITTLE_ENDIAN);

    if (bb.getInt() != MAGIC_NUMBER) {
        throw new IOException("Corrupt data: Unexpected magic number.");
    }/*  w  w  w .jav a2 s . c  o m*/

    final byte[] keyBytes = new byte[bb.getInt()];
    bb.get(keyBytes);

    final byte[] uploadIdBytes = new byte[bb.getInt()];
    bb.get(uploadIdBytes);

    final int numParts = bb.getInt();
    final ArrayList<PartETag> parts = new ArrayList<>(numParts);
    for (int i = 0; i < numParts; i++) {
        final int partNum = bb.getInt();
        final byte[] buffer = new byte[bb.getInt()];
        bb.get(buffer);
        parts.add(new PartETag(partNum, new String(buffer, CHARSET)));
    }

    final long numBytes = bb.getLong();

    final String lastPart;
    final int lastObjectArraySize = bb.getInt();
    if (lastObjectArraySize == 0) {
        lastPart = null;
    } else {
        byte[] lastPartBytes = new byte[lastObjectArraySize];
        bb.get(lastPartBytes);
        lastPart = new String(lastPartBytes, CHARSET);
    }

    final long lastPartLength = bb.getLong();

    return new S3Recoverable(new String(keyBytes, CHARSET), new String(uploadIdBytes, CHARSET), parts, numBytes,
            lastPart, lastPartLength);
}

---

From source file:org.elasticsearch.cloud.aws.blobstore.MockDefaultS3OutputStream.java

License:Apache License

@Override
protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId,
        InputStream is, int length, boolean lastPart) throws AmazonS3Exception {
    try {//w  w  w  . j  a  v a2  s .co  m
        long copied = Streams.copy(is, out);
        if (copied != length) {
            throw new AmazonS3Exception("Not all the bytes were copied");
        }
        return new PartETag(numberOfUploadRequests++, RandomizedTest.randomAsciiOfLength(50));
    } catch (IOException e) {
        throw new AmazonS3Exception(e.getMessage());
    }
}

---

From source file:org.icgc.dcc.storage.server.repository.s3.S3UploadStateStore.java

License:Open Source License

@Override
@SneakyThrows//from  w  ww  .j  av a  2s. co  m
public Map<Integer, UploadPartDetail> getUploadStatePartDetails(String objectId, String uploadId) {
    val uploadStateKey = getUploadStateKey(objectId, uploadId, PART);
    val details = Maps.<Integer, UploadPartDetail>newHashMap();

    eachObjectSummary(objectId, uploadStateKey, (objectSummary) -> {
        CompletedPart part = readCompletedPart(objectId, uploadId, objectSummary);

        PartETag etag = new PartETag(part.getPartNumber(), part.getEtag());
        UploadPartDetailBuilder detailBuilder = UploadPartDetail.builder().etag(etag)
                .partNumber(part.getPartNumber()).md5(part.getMd5());
        details.put(part.getPartNumber(), detailBuilder.build());
    });

    return details;
}

---