Example usage for com.amazonaws.services.s3.transfer Transfer isDone

List of usage examples for com.amazonaws.services.s3.transfer Transfer isDone

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.transfer.*
  5. Transfer
  6. isDone

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.transfer Transfer isDone.

Prototype

public boolean isDone();

Source Link

Document

Returns whether or not the transfer is finished (i.e.

Usage

From source file:aws.example.s3.XferMgrProgress.java

License:Open Source License

public static void showTransferProgress(Transfer xfer) {
    // print the transfer's human-readable description
    System.out.println(xfer.getDescription());
    // print an empty progress bar...
    printProgressBar(0.0);//from  w w w . ja  v a 2 s  .c  o  m
    // update the progress bar while the xfer is ongoing.
    do {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            return;
        }
        // Note: so_far and total aren't used, they're just for
        // documentation purposes.
        TransferProgress progress = xfer.getProgress();
        long so_far = progress.getBytesTransferred();
        long total = progress.getTotalBytesToTransfer();
        double pct = progress.getPercentTransferred();
        eraseProgressBar();
        printProgressBar(pct);
    } while (xfer.isDone() == false);
    // print the final state of the transfer.
    TransferState xfer_state = xfer.getState();
    System.out.println(": " + xfer_state);
}

From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java

License:Open Source License

@Override
public boolean uploadDirectoryOrFileAndListenProgress(final String bucketName, final File source,
        final String virtualDirectoryKeyPrefix)
        throws AmazonClientException, AmazonServiceException, FileNotFoundException {
    LOGGER.info("uploadDirectoryOrFileAndWaitForCompletion invoked, bucketName: {} , Source: {}", bucketName,
            source.getAbsolutePath());// w ww .  ja  va  2  s. com
    Transfer transfer = null;
    final TransferManager transferMgr = new TransferManager(s3client);
    if (source.isFile()) {
        transfer = transferMgr.upload(bucketName, source.getPath(), source);
    } else if (source.isDirectory()) {
        //upload recursively
        transfer = transferMgr.uploadDirectory(bucketName, virtualDirectoryKeyPrefix, source, true);
    } else {
        throw new FileNotFoundException("File is neither a regular file nor a directory " + source);
    }

    // You can poll your transfer's status to check its progress
    if (transfer.isDone()) {
        LOGGER.info("Start: {}  , State: {} and Progress (%): {}", transfer.getDescription(),
                transfer.getState(), transfer.getProgress().getPercentTransferred());

    }

    // Add progressListener to listen asynchronous notifications about your transfer's progress
    // Uncomment below code snippet during development
    /*transfer.addProgressListener(new ProgressListener() {
       public void progressChanged(ProgressEvent event) {
    if (LOGGER.isDebugEnabled()) {
       LOGGER.debug("Transferred bytes: " + (long) event.getBytesTransferred());
    }
     }
    });*/

    try {
        //Block the current thread and wait for completion
        //If the transfer fails AmazonClientException will be thrown
        transfer.waitForCompletion();
    } catch (AmazonClientException | InterruptedException excp) {
        LOGGER.error("Exception occured while waiting for transfer: ", excp);
    }

    LOGGER.info("End: {} , State: {} , Progress (%): {}", transfer.getDescription(), transfer.getState(),
            transfer.getProgress().getPercentTransferred());
    return transfer.isDone();
}

From source file:org.finra.dm.dao.impl.S3DaoImpl.java

License:Apache License

/**
 * Performs a file/directory transfer./* ww w  . j  ava  2 s .  co m*/
 *
 * @param params the parameters.
 * @param transferer a transferer that knows how to perform the transfer.
 *
 * @return the results.
 * @throws InterruptedException if a problem is encountered.
 */
private S3FileTransferResultsDto performTransfer(final S3FileTransferRequestParamsDto params,
        Transferer transferer) throws InterruptedException {
    // Create a transfer manager.
    TransferManager transferManager = null;

    try {
        // Create a transfer manager.
        transferManager = getTransferManager(params);

        // Start a stop watch to keep track of how long the transfer takes.
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();

        // Perform the transfer.
        Transfer transfer = transferer.performTransfer(transferManager);
        TransferProgress transferProgress = transfer.getProgress();

        LOGGER.info(
                String.format("%d bytes transferred out of %d (%.1f%%)", transferProgress.getBytesTransferred(),
                        transferProgress.getTotalBytesToTransfer(), transferProgress.getPercentTransferred()));

        long stepCount = 0;

        // Loop until the transfer is complete.
        do {
            Thread.sleep(SLEEP_INTERVAL_MILLIS);
            stepCount++;

            // Log progress status every 30 seconds and when transfer is complete.
            if (transfer.isDone() || stepCount % 300 == 0) {
                LOGGER.info(String.format("%d bytes transferred out of %d (%.1f%%)",
                        transferProgress.getBytesTransferred(), transferProgress.getTotalBytesToTransfer(),
                        transferProgress.getPercentTransferred()));
            }
        } while (!transfer.isDone());

        // Stop the stop watch and create a results object.
        stopWatch.stop();

        // If the transfer failed, throw the underlying AWS exception if we can determine one. Otherwise, throw our own exception.
        TransferState transferState = transfer.getState();
        if (transferState == TransferState.Failed) {
            // The waitForException method should return the underlying AWS exception since the state is "Failed". It should not block since the
            // transfer is already "done" per previous code checking "isDone".
            AmazonClientException amazonClientException = transfer.waitForException();

            // If the returned exception is null, we weren't able to get the underlying AWS exception so just throw our own exception.
            // This is unlikely since the transfer failed, but it's better to handle the possibility just in case.
            if (amazonClientException == null) {
                throw new IllegalStateException("The transfer operation \"" + transfer.getDescription()
                        + "\" failed for an unknown reason.");
            }

            // Throw the Amazon underlying exception.
            throw amazonClientException;
        }
        // Ensure the transfer completed. If not, throw an exception.
        else if (transferState != TransferState.Completed) {
            throw new IllegalStateException("The transfer operation \"" + transfer.getDescription()
                    + "\" did not complete successfully. Current state: \"" + transferState + "\".");
        }

        // TransferProgress.getBytesTransferred() are not populated for S3 Copy objects.
        if (!(transfer instanceof Copy)) {
            // Sanity check for the number of bytes transferred.
            Assert.isTrue(transferProgress.getBytesTransferred() >= transferProgress.getTotalBytesToTransfer(),
                    String.format(
                            "Actual number of bytes transferred is less than expected (actual: %d bytes; expected: %d bytes).",
                            transferProgress.getBytesTransferred(),
                            transferProgress.getTotalBytesToTransfer()));
        }

        // Create the results object and populate it with the standard data.
        S3FileTransferResultsDto results = new S3FileTransferResultsDto();
        results.setDurationMillis(stopWatch.getTime());
        results.setTotalBytesTransferred(transfer.getProgress().getBytesTransferred());
        results.setTotalFilesTransferred(1L);

        if (transfer instanceof MultipleFileUpload) {
            // For upload directory, we need to calculate the total number of files transferred differently.
            results.setTotalFilesTransferred((long) ((MultipleFileUpload) transfer).getSubTransfers().size());
        } else if (transfer instanceof MultipleFileDownload) {
            // For download directory, we need to calculate the total number of files differently.
            results.setTotalFilesTransferred((long) listDirectory(params).size());
        }

        // Return the results.
        return results;
    } finally {
        // Shutdown the transfer manager to release resources. If this isn't done, the JVM may delay upon exiting.
        if (transferManager != null) {
            transferManager.shutdownNow();
        }
    }
}

From source file:org.finra.herd.dao.impl.S3DaoImpl.java

License:Apache License

/**
 * Performs a file/directory transfer.//from w  w  w .  j a va2  s  .c o m
 *
 * @param params the parameters.
 * @param transferer a transferer that knows how to perform the transfer.
 *
 * @return the results.
 * @throws InterruptedException if a problem is encountered.
 */
private S3FileTransferResultsDto performTransfer(final S3FileTransferRequestParamsDto params,
        Transferer transferer) throws InterruptedException {
    // Create a transfer manager.
    TransferManager transferManager = getTransferManager(params);

    try {
        // Start a stop watch to keep track of how long the transfer takes.
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();

        // Perform the transfer.
        Transfer transfer = transferer.performTransfer(transferManager);
        TransferProgress transferProgress = transfer.getProgress();

        logTransferProgress(transferProgress);

        long stepCount = 0;

        // Loop until the transfer is complete.
        do {
            Thread.sleep(sleepIntervalsMillis);
            stepCount++;

            // Log progress status every 30 seconds and when transfer is complete.
            if (transfer.isDone() || stepCount % 300 == 0) {
                logTransferProgress(transferProgress);
            }
        } while (!transfer.isDone());

        // Stop the stop watch and create a results object.
        stopWatch.stop();

        // If the transfer failed, throw the underlying AWS exception if we can determine one. Otherwise, throw our own exception.
        TransferState transferState = transfer.getState();
        if (transferState == TransferState.Failed) {
            // The waitForException method should return the underlying AWS exception since the state is "Failed". It should not block since the
            // transfer is already "done" per previous code checking "isDone".
            AmazonClientException amazonClientException = transfer.waitForException();

            // If the returned exception is null, we weren't able to get the underlying AWS exception so just throw our own exception.
            // This is unlikely since the transfer failed, but it's better to handle the possibility just in case.
            if (amazonClientException == null) {
                throw new IllegalStateException("The transfer operation \"" + transfer.getDescription()
                        + "\" failed for an unknown reason.");
            }

            // Throw the Amazon underlying exception.
            throw amazonClientException;
        }
        // Ensure the transfer completed. If not, throw an exception.
        else if (transferState != TransferState.Completed) {
            throw new IllegalStateException("The transfer operation \"" + transfer.getDescription()
                    + "\" did not complete successfully. Current state: \"" + transferState + "\".");
        }

        // TransferProgress.getBytesTransferred() are not populated for S3 Copy objects.
        if (!(transfer instanceof Copy)) {
            // Sanity check for the number of bytes transferred.
            Assert.isTrue(transferProgress.getBytesTransferred() >= transferProgress.getTotalBytesToTransfer(),
                    String.format(
                            "Actual number of bytes transferred is less than expected (actual: %d bytes; expected: %d bytes).",
                            transferProgress.getBytesTransferred(),
                            transferProgress.getTotalBytesToTransfer()));
        }

        // Create the results object and populate it with the standard data.
        S3FileTransferResultsDto results = new S3FileTransferResultsDto();
        results.setDurationMillis(stopWatch.getTime());
        results.setTotalBytesTransferred(transfer.getProgress().getBytesTransferred());
        results.setTotalFilesTransferred(1L);

        if (transfer instanceof MultipleFileUpload) {
            // For upload directory, we need to calculate the total number of files transferred differently.
            results.setTotalFilesTransferred((long) ((MultipleFileUpload) transfer).getSubTransfers().size());
        } else if (transfer instanceof MultipleFileDownload) {
            // For download directory, we need to calculate the total number of files differently.
            results.setTotalFilesTransferred((long) listDirectory(params).size());
        }

        // Return the results.
        return results;
    } finally {
        // Shutdown the transfer manager to release resources. If this isn't done, the JVM may delay upon exiting.
        transferManager.shutdownNow();
    }
}

From source file:org.ow2.proactive.scheduler.examples.SchedulerExamplesUtils.java

License:Open Source License

/**
 * Prints progress while waiting for the transfer to finish.
 * @param xfer// w ww.ja va 2  s.  c  om
 */
static void showTransferProgress(Transfer xfer) {
    // print the transfer's human-readable description
    logger.info(xfer.getDescription());
    // print an empty progress bar...
    printProgressBar(0.0);
    // update the progress bar while the xfer is ongoing.
    do {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            return;
        }
        // Note: so_far and total aren't used, they're just for
        // documentation purposes.
        TransferProgress progress = xfer.getProgress();
        double pct = progress.getPercentTransferred();
        eraseProgressBar();
        printProgressBar(pct);
    } while (!xfer.isDone());
    // print the final state of the transfer.
    Transfer.TransferState xferState = xfer.getState();
    logger.info(": " + xferState);
}

From source file:squash.deployment.lambdas.utils.TransferUtils.java

License:Apache License

/**
 * Waits for S3 transfers to complete./* w  w  w  .  j a  v  a2  s . c om*/
 * 
 * <p>S3 transfers via the S3 TransferManager are asynchronous. This can be used
 *    by all transfers (uploads, copies, and downloads) to wait until they have
 *    completed.
 * 
 *    @param transfer returned by the TransferManager when transfer is initiated.
 *    @param logger a CloudwatchLogs logger.
 *    @throws AmazonServiceException if any errors occurred in S3 during the wait.
 *    @throws InterruptedException if the wait is interrupted.
 */
public static void waitForS3Transfer(Transfer transfer, LambdaLogger logger) throws InterruptedException {
    while (transfer.isDone() == false) {
        logger.log("Transfer progress: " + transfer.getProgress().getPercentTransferred() + "%");
        try {
            Thread.sleep(100); // milliseconds
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            logger.log("Transfer thread interrupted");
            throw e;
        }
    }
    logger.log("Transfer is done - now wait for completion to see if transfer succeeded.");
    try {
        transfer.waitForCompletion(); // Will throw if transfer failed
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        logger.log("Transfer thread interrupted");
        throw e;
    }
}