Example usage for com.amazonaws.services.s3.transfer TransferProgress setTotalBytesToTransfer

List of usage examples for com.amazonaws.services.s3.transfer TransferProgress setTotalBytesToTransfer

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.transfer.*
  5. TransferProgress
  6. setTotalBytesToTransfer

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.transfer TransferProgress setTotalBytesToTransfer.

Prototype

public void setTotalBytesToTransfer(long totalBytesToTransfer)

Source Link

Usage

From source file:org.finra.dm.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * <p>/*from   w  ww .  j av a2s  .c  o m*/
 * Simulates a copyFile operation.
 * </p>
 * <p>
 * This method does not actually copy files in-memory, but always returns a pre-defined result.
 * </p>
 * <p>
 * The result {@link Copy} has the following properties:
 * <dl>
 * <p/>
 * <dt>description</dt>
 * <dd>"MockTransfer"</dd>
 * <p/>
 * <dt>state</dt>
 * <dd>{@link TransferState#Completed}</dd>
 * <p/>
 * <dt>transferProgress.totalBytesToTransfer</dt>
 * <dd>1024</dd>
 * <p/>
 * <dt>transferProgress.updateProgress</dt>
 * <dd>1024</dd>
 * <p/>
 * </dl>
 * <p/>
 * All other properties are set as default.
 * </p>
 * <p>
 * This operation takes the following hints when suffixed in copyObjectRequest.sourceKey:
 * <dl>
 * <p/>
 * <dt>MOCK_S3_FILE_NAME_NOT_FOUND</dt>
 * <dd>Throws a AmazonServiceException</dd>
 * <p/>
 * </dl>
 * </p>
 */
@Override
public Copy copyFile(final CopyObjectRequest copyObjectRequest, TransferManager transferManager) {
    LOGGER.debug("copyFile(): copyObjectRequest.getSourceBucketName() = "
            + copyObjectRequest.getSourceBucketName() + ", copyObjectRequest.getSourceKey() = "
            + copyObjectRequest.getSourceKey() + ", copyObjectRequest.getDestinationBucketName() = "
            + copyObjectRequest.getDestinationBucketName() + ", copyObjectRequest.getDestinationKey() = "
            + copyObjectRequest.getDestinationKey());

    if (copyObjectRequest.getSourceKey().endsWith(MOCK_S3_FILE_NAME_NOT_FOUND)) {
        throw new AmazonServiceException(null);
    }
    /*
     * Does not actually copy files in memory. There is a test case S3ServiceTest.testCopyFile() which is expecting a result without staging any data.
     * Below is implementation when needed.
     */
    /*
     * String sourceBucketName = copyObjectRequest.getSourceBucketName();
     * String sourceKey = copyObjectRequest.getSourceKey();
     * MockS3Bucket mockSourceS3Bucket = mockS3Buckets.get(sourceBucketName);
     * MockS3Object mockSourceS3Object = mockSourceS3Bucket.getObjects().get(sourceKey);
     * String destinationBucketName = copyObjectRequest.getDestinationBucketName();
     * String destinationKey = copyObjectRequest.getDestinationKey();
     * ObjectMetadata objectMetadata = copyObjectRequest.getNewObjectMetadata();
     * MockS3Object mockDestinationS3Object = new MockS3Object();
     * mockDestinationS3Object.setKey(destinationKey);
     * mockDestinationS3Object.setData(Arrays.copyOf(mockSourceS3Object.getData(), mockSourceS3Object.getData().length));
     * mockDestinationS3Object.setObjectMetadata(objectMetadata);
     * MockS3Bucket mockDestinationS3Bucket = getOrCreateBucket(destinationBucketName);
     * mockDestinationS3Bucket.getObjects().put(destinationKey, mockDestinationS3Object);
     */

    // Set the result CopyImpl and TransferProgress.
    TransferProgress transferProgress = new TransferProgress();
    transferProgress.setTotalBytesToTransfer(AbstractCoreTest.FILE_SIZE_1_KB);
    transferProgress.updateProgress(AbstractCoreTest.FILE_SIZE_1_KB);
    CopyImpl copy = new CopyImpl(MOCK_TRANSFER_DESCRIPTION, transferProgress, null, null);
    copy.setState(TransferState.Completed);

    // If an invalid KMS Id was passed in, mark the transfer as failed and return an exception via the transfer monitor.
    final String kmsId = copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId();
    if (kmsId.startsWith(MOCK_KMS_ID_FAILED_TRANSFER)) {
        copy.setState(TransferState.Failed);
        copy.setMonitor(new TransferMonitor() {
            @Override
            public Future<?> getFuture() {
                if (!kmsId.equals(MOCK_KMS_ID_FAILED_TRANSFER_NO_EXCEPTION)) {
                    throw new AmazonServiceException("Key '"
                            + copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId()
                            + "' does not exist (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: 1234567890123456)");
                }

                // We don't want an exception to be thrown so return a basic future that won't throw an exception.
                BasicFuture<?> future = new BasicFuture<Void>(null);
                future.completed(null);
                return future;
            }

            @Override
            public boolean isDone() {
                return true;
            }
        });
    } else if (kmsId.startsWith(MOCK_KMS_ID_CANCELED_TRANSFER)) {
        // If the KMS indicates a cancelled transfer, just update the state to canceled.
        copy.setState(TransferState.Canceled);
    }

    return copy;
}

From source file:org.finra.dm.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * Uploads a list of files./*from w w w.j  av a  2 s.c o m*/
 * <p/>
 * Delegates to {@link #putObject(PutObjectRequest, AmazonS3Client)} for each file.
 */
@Override
public MultipleFileUpload uploadFileList(String bucketName, String virtualDirectoryKeyPrefix, File directory,
        List<File> files, ObjectMetadataProvider metadataProvider, TransferManager transferManager) {
    LOGGER.debug("uploadFileList(): bucketName = " + bucketName + ", virtualDirectoryKeyPrefix = "
            + virtualDirectoryKeyPrefix + ", directory = " + directory + ", files = " + files);

    String directoryPath = directory.getAbsolutePath();

    long totalFileLength = 0;
    List<Upload> subTransfers = new ArrayList<>();
    for (File file : files) {
        // Get path to file relative to the specified directory
        String relativeFilePath = file.getAbsolutePath().substring(directoryPath.length());

        // Replace any backslashes (i.e. Windows separator) with a forward slash.
        relativeFilePath = relativeFilePath.replace("\\", "/");

        // Remove any leading slashes
        relativeFilePath = relativeFilePath.replaceAll("^/+", "");

        long fileLength = file.length();

        // Remove any trailing slashes
        virtualDirectoryKeyPrefix = virtualDirectoryKeyPrefix.replaceAll("/+$", "");

        String s3ObjectKey = virtualDirectoryKeyPrefix + "/" + relativeFilePath;
        totalFileLength += fileLength;

        PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, s3ObjectKey, file);

        putObject(putObjectRequest, (AmazonS3Client) transferManager.getAmazonS3Client());

        subTransfers.add(new UploadImpl(null, null, null, null));
    }

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(totalFileLength);
    progress.updateProgress(totalFileLength);

    MultipleFileUploadImpl multipleFileUpload = new MultipleFileUploadImpl(null, progress, null,
            virtualDirectoryKeyPrefix, bucketName, subTransfers);
    multipleFileUpload.setState(TransferState.Completed);
    return multipleFileUpload;
}

From source file:org.finra.dm.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * Downloads objects with the given prefix into a destination directory.
 * <p/>/*from  w  ww  .j  a  va 2s  .  c  om*/
 * Creates any directory that does not exist in the path to the destination directory.
 */
@Override
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory,
        TransferManager transferManager) {
    LOGGER.debug("downloadDirectory(): bucketName = " + bucketName + ", keyPrefix = " + keyPrefix
            + ", destinationDirectory = " + destinationDirectory);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);

    List<Download> downloads = new ArrayList<>();
    long totalBytes = 0;

    if (mockS3Bucket != null) {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values()) {
            if (mockS3Object.getKey().startsWith(keyPrefix)) {
                String filePath = destinationDirectory.getAbsolutePath() + "/" + mockS3Object.getKey();
                File file = new File(filePath);
                file.getParentFile().mkdirs(); // Create any directory in the path that does not exist.
                try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
                    LOGGER.debug("downloadDirectory(): Writing file " + file);
                    fileOutputStream.write(mockS3Object.getData());
                    totalBytes += mockS3Object.getData().length;
                    downloads.add(new DownloadImpl(null, null, null, null, null,
                            new GetObjectRequest(bucketName, mockS3Object.getKey()), file));
                } catch (IOException e) {
                    throw new RuntimeException("Error writing to file " + file, e);
                }
            }
        }
    }

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(totalBytes);
    progress.updateProgress(totalBytes);

    MultipleFileDownloadImpl multipleFileDownload = new MultipleFileDownloadImpl(null, progress, null,
            keyPrefix, bucketName, downloads);
    multipleFileDownload.setState(TransferState.Completed);
    return multipleFileDownload;
}

From source file:org.finra.dm.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * Puts an object./*from   ww w .  ja  v a2s . c  o  m*/
 */
@Override
public Upload upload(PutObjectRequest putObjectRequest, TransferManager transferManager)
        throws AmazonServiceException, AmazonClientException {
    LOGGER.debug("upload(): putObjectRequest.getBucketName() = " + putObjectRequest.getBucketName()
            + ", putObjectRequest.getKey() = " + putObjectRequest.getKey());

    putObject(putObjectRequest, (AmazonS3Client) transferManager.getAmazonS3Client());

    long contentLength = putObjectRequest.getFile().length();
    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(contentLength);
    progress.updateProgress(contentLength);

    UploadImpl upload = new UploadImpl(null, progress, null, null);
    upload.setState(TransferState.Completed);

    return upload;
}

From source file:org.finra.dm.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * Downloads an object./*from ww w.  j av a2 s.com*/
 */
@Override
public Download download(String bucket, String key, File file, TransferManager transferManager) {
    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucket);
    MockS3Object mockS3Object = mockS3Bucket.getObjects().get(key);
    try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
        fileOutputStream.write(mockS3Object.getData());
    } catch (IOException e) {
        throw new RuntimeException("Error writing to file " + file, e);
    }

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(mockS3Object.getData().length);
    progress.updateProgress(mockS3Object.getData().length);

    DownloadImpl download = new DownloadImpl(null, progress, null, null, null,
            new GetObjectRequest(bucket, key), file);
    download.setState(TransferState.Completed);

    return download;
}

From source file:org.finra.herd.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * {@inheritDoc} <p/> <p> This implementation simulates a copyFile operation. </p> <p> This method copies files in-memory. </p> <p> The result {@link Copy}
 * has the following properties: <dl> <p/> <dt>description</dt> <dd>"MockTransfer"</dd> <p/> <dt>state</dt> <dd>{@link TransferState#Completed}</dd> <p/>
 * <dt>transferProgress.totalBytesToTransfer</dt> <dd>1024</dd> <p/> <dt>transferProgress.updateProgress</dt> <dd>1024</dd> <p/> </dl> <p/> All other
 * properties are set as default. </p> <p> This operation takes the following hints when suffixed in copyObjectRequest.sourceKey: <dl> <p/>
 * <dt>MOCK_S3_FILE_NAME_SERVICE_EXCEPTION</dt> <dd>Throws a AmazonServiceException</dd> <p/> </dl> </p>
 */// ww w  .ja  v  a 2 s.co  m
@Override
public Copy copyFile(final CopyObjectRequest copyObjectRequest, TransferManager transferManager) {
    LOGGER.debug("copyFile(): copyObjectRequest.getSourceBucketName() = "
            + copyObjectRequest.getSourceBucketName() + ", copyObjectRequest.getSourceKey() = "
            + copyObjectRequest.getSourceKey() + ", copyObjectRequest.getDestinationBucketName() = "
            + copyObjectRequest.getDestinationBucketName() + ", copyObjectRequest.getDestinationKey() = "
            + copyObjectRequest.getDestinationKey());

    if (copyObjectRequest.getSourceKey().endsWith(MOCK_S3_FILE_NAME_SERVICE_EXCEPTION)) {
        throw new AmazonServiceException(null);
    }

    String sourceBucketName = copyObjectRequest.getSourceBucketName();
    String sourceKey = copyObjectRequest.getSourceKey();

    MockS3Bucket mockSourceS3Bucket = getOrCreateBucket(sourceBucketName);
    MockS3Object mockSourceS3Object = mockSourceS3Bucket.getObjects().get(sourceKey);

    if (mockSourceS3Object == null) {
        AmazonServiceException amazonServiceException = new AmazonServiceException(
                S3Operations.ERROR_CODE_NO_SUCH_KEY);
        amazonServiceException.setErrorCode(S3Operations.ERROR_CODE_NO_SUCH_KEY);
        throw amazonServiceException;
    }

    // Set the result CopyImpl and TransferProgress.
    TransferProgress transferProgress = new TransferProgress();
    transferProgress.setTotalBytesToTransfer(mockSourceS3Object.getObjectMetadata().getContentLength());
    transferProgress.updateProgress(mockSourceS3Object.getObjectMetadata().getContentLength());
    CopyImpl copy = new CopyImpl(MOCK_TRANSFER_DESCRIPTION, transferProgress, null, null);
    copy.setState(TransferState.Completed);

    // If an invalid KMS Id was passed in, mark the transfer as failed and return an exception via the transfer monitor.
    if (copyObjectRequest.getSSEAwsKeyManagementParams() != null) {
        final String kmsId = copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId();
        if (kmsId.startsWith(MOCK_KMS_ID_FAILED_TRANSFER)) {
            copy.setState(TransferState.Failed);
            copy.setMonitor(new TransferMonitor() {
                @Override
                public Future<?> getFuture() {
                    if (!kmsId.equals(MOCK_KMS_ID_FAILED_TRANSFER_NO_EXCEPTION)) {
                        throw new AmazonServiceException("Key '"
                                + copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId()
                                + "' does not exist (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: 1234567890123456)");
                    }

                    // We don't want an exception to be thrown so return a basic future that won't throw an exception.
                    BasicFuture<?> future = new BasicFuture<Void>(null);
                    future.completed(null);
                    return future;
                }

                @Override
                public boolean isDone() {
                    return true;
                }
            });
        } else if (kmsId.startsWith(MOCK_KMS_ID_CANCELED_TRANSFER)) {
            // If the KMS indicates a cancelled transfer, just update the state to canceled.
            copy.setState(TransferState.Canceled);
        }
    }

    // If copy operation is marked as completed, perform the actual file copy in memory.
    if (copy.getState().equals(TransferState.Completed)) {
        String destinationBucketName = copyObjectRequest.getDestinationBucketName();
        String destinationObjectKey = copyObjectRequest.getDestinationKey();
        String destinationObjectVersion = MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED.equals(destinationBucketName)
                ? UUID.randomUUID().toString()
                : null;
        String destinationObjectKeyVersion = destinationObjectKey
                + (destinationObjectVersion != null ? destinationObjectVersion : "");

        ObjectMetadata objectMetadata = copyObjectRequest.getNewObjectMetadata();
        MockS3Object mockDestinationS3Object = new MockS3Object();
        mockDestinationS3Object.setKey(destinationObjectKey);
        mockDestinationS3Object.setVersion(destinationObjectVersion);
        mockDestinationS3Object
                .setData(Arrays.copyOf(mockSourceS3Object.getData(), mockSourceS3Object.getData().length));
        mockDestinationS3Object.setObjectMetadata(objectMetadata);

        MockS3Bucket mockDestinationS3Bucket = getOrCreateBucket(destinationBucketName);
        mockDestinationS3Bucket.getObjects().put(destinationObjectKey, mockDestinationS3Object);
        mockDestinationS3Bucket.getVersions().put(destinationObjectKeyVersion, mockDestinationS3Object);
    }

    return copy;
}

From source file:org.finra.herd.dao.impl.MockS3OperationsImpl.java

License:Apache License

@Override
public Download download(String bucket, String key, File file, TransferManager transferManager) {
    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucket);
    MockS3Object mockS3Object = mockS3Bucket.getObjects().get(key);
    try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
        fileOutputStream.write(mockS3Object.getData());
    } catch (IOException e) {
        throw new RuntimeException("Error writing to file " + file, e);
    }//from   w ww .java 2 s.  c  om

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(mockS3Object.getData().length);
    progress.updateProgress(mockS3Object.getData().length);

    DownloadImpl download = new DownloadImpl(null, progress, null, null, null,
            new GetObjectRequest(bucket, key), file, mockS3Object.getObjectMetadata(), false);
    download.setState(TransferState.Completed);

    return download;
}

From source file:org.finra.herd.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * {@inheritDoc}//w  w  w. j  a  v a  2s  . co  m
 * <p/>
 * This implementation creates any directory that does not exist in the path to the destination directory.
 */
@Override
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory,
        TransferManager transferManager) {
    LOGGER.debug("downloadDirectory(): bucketName = " + bucketName + ", keyPrefix = " + keyPrefix
            + ", destinationDirectory = " + destinationDirectory);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);

    List<Download> downloads = new ArrayList<>();
    long totalBytes = 0;

    if (mockS3Bucket != null) {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values()) {
            if (mockS3Object.getKey().startsWith(keyPrefix)) {
                String filePath = destinationDirectory.getAbsolutePath() + "/" + mockS3Object.getKey();
                File file = new File(filePath);
                file.getParentFile().mkdirs(); // Create any directory in the path that does not exist.
                try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
                    LOGGER.debug("downloadDirectory(): Writing file " + file);
                    fileOutputStream.write(mockS3Object.getData());
                    totalBytes += mockS3Object.getData().length;
                    downloads.add(new DownloadImpl(null, null, null, null, null,
                            new GetObjectRequest(bucketName, mockS3Object.getKey()), file,
                            mockS3Object.getObjectMetadata(), false));
                } catch (IOException e) {
                    throw new RuntimeException("Error writing to file " + file, e);
                }
            }
        }
    }

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(totalBytes);
    progress.updateProgress(totalBytes);

    MultipleFileDownloadImpl multipleFileDownload = new MultipleFileDownloadImpl(null, progress, null,
            keyPrefix, bucketName, downloads);
    multipleFileDownload.setState(TransferState.Completed);
    return multipleFileDownload;
}

From source file:org.finra.herd.dao.impl.MockS3OperationsImpl.java

License:Apache License

@Override
public Upload upload(PutObjectRequest putObjectRequest, TransferManager transferManager) {
    LOGGER.debug("upload(): putObjectRequest.getBucketName() = " + putObjectRequest.getBucketName()
            + ", putObjectRequest.getKey() = " + putObjectRequest.getKey());

    putObject(putObjectRequest, transferManager.getAmazonS3Client());

    long contentLength = putObjectRequest.getFile().length();
    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(contentLength);
    progress.updateProgress(contentLength);

    UploadImpl upload = new UploadImpl(null, progress, null, null);
    upload.setState(TransferState.Completed);

    return upload;
}

From source file:org.finra.herd.dao.impl.MockS3OperationsImpl.java

License:Apache License

@Override
public MultipleFileUpload uploadFileList(String bucketName, String virtualDirectoryKeyPrefix, File directory,
        List<File> files, ObjectMetadataProvider metadataProvider, TransferManager transferManager) {
    LOGGER.debug("uploadFileList(): bucketName = " + bucketName + ", virtualDirectoryKeyPrefix = "
            + virtualDirectoryKeyPrefix + ", directory = " + directory + ", files = " + files);

    String directoryPath = directory.getAbsolutePath();

    long totalFileLength = 0;
    List<Upload> subTransfers = new ArrayList<>();
    for (File file : files) {
        // Get path to file relative to the specified directory
        String relativeFilePath = file.getAbsolutePath().substring(directoryPath.length());

        // Replace any backslashes (i.e. Windows separator) with a forward slash.
        relativeFilePath = relativeFilePath.replace("\\", "/");

        // Remove any leading slashes
        relativeFilePath = relativeFilePath.replaceAll("^/+", "");

        long fileLength = file.length();

        // Remove any trailing slashes
        virtualDirectoryKeyPrefix = virtualDirectoryKeyPrefix.replaceAll("/+$", "");

        String s3ObjectKey = virtualDirectoryKeyPrefix + "/" + relativeFilePath;
        totalFileLength += fileLength;//from  ww  w  . j  a v a  2  s  .c  o  m

        PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, s3ObjectKey, file);

        ObjectMetadata objectMetadata = new ObjectMetadata();
        metadataProvider.provideObjectMetadata(null, objectMetadata);
        putObjectRequest.setMetadata(objectMetadata);

        putObject(putObjectRequest, transferManager.getAmazonS3Client());

        subTransfers.add(new UploadImpl(null, null, null, null));
    }

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(totalFileLength);
    progress.updateProgress(totalFileLength);

    MultipleFileUploadImpl multipleFileUpload = new MultipleFileUploadImpl(null, progress, null,
            virtualDirectoryKeyPrefix, bucketName, subTransfers);
    multipleFileUpload.setState(TransferState.Completed);
    return multipleFileUpload;
}