Example usage for com.amazonaws.services.s3.transfer TransferProgress getTotalBytesToTransfer

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.transfer TransferProgress getTotalBytesToTransfer.

Prototype

public long getTotalBytesToTransfer() 

Source Link

Document

Returns the total size in bytes of the associated transfer, or -1 if the total size isn't known.

Usage

From source file:aws.example.s3.XferMgrProgress.java

License:Open Source License

public static void showTransferProgress(Transfer xfer) {
    // print the transfer's human-readable description
    System.out.println(xfer.getDescription());
    // print an empty progress bar...
    printProgressBar(0.0);/*from ww w. j av a  2  s  . co m*/
    // update the progress bar while the xfer is ongoing.
    do {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            return;
        }
        // Note: so_far and total aren't used, they're just for
        // documentation purposes.
        TransferProgress progress = xfer.getProgress();
        long so_far = progress.getBytesTransferred();
        long total = progress.getTotalBytesToTransfer();
        double pct = progress.getPercentTransferred();
        eraseProgressBar();
        printProgressBar(pct);
    } while (xfer.isDone() == false);
    // print the final state of the transfer.
    TransferState xfer_state = xfer.getState();
    System.out.println(": " + xfer_state);
}

---

From source file:org.finra.dm.dao.impl.S3DaoImpl.java

License:Apache License

/**
 * Performs a file/directory transfer./*from www.  ja v  a2  s  .c o m*/
 *
 * @param params the parameters.
 * @param transferer a transferer that knows how to perform the transfer.
 *
 * @return the results.
 * @throws InterruptedException if a problem is encountered.
 */
private S3FileTransferResultsDto performTransfer(final S3FileTransferRequestParamsDto params,
        Transferer transferer) throws InterruptedException {
    // Create a transfer manager.
    TransferManager transferManager = null;

    try {
        // Create a transfer manager.
        transferManager = getTransferManager(params);

        // Start a stop watch to keep track of how long the transfer takes.
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();

        // Perform the transfer.
        Transfer transfer = transferer.performTransfer(transferManager);
        TransferProgress transferProgress = transfer.getProgress();

        LOGGER.info(
                String.format("%d bytes transferred out of %d (%.1f%%)", transferProgress.getBytesTransferred(),
                        transferProgress.getTotalBytesToTransfer(), transferProgress.getPercentTransferred()));

        long stepCount = 0;

        // Loop until the transfer is complete.
        do {
            Thread.sleep(SLEEP_INTERVAL_MILLIS);
            stepCount++;

            // Log progress status every 30 seconds and when transfer is complete.
            if (transfer.isDone() || stepCount % 300 == 0) {
                LOGGER.info(String.format("%d bytes transferred out of %d (%.1f%%)",
                        transferProgress.getBytesTransferred(), transferProgress.getTotalBytesToTransfer(),
                        transferProgress.getPercentTransferred()));
            }
        } while (!transfer.isDone());

        // Stop the stop watch and create a results object.
        stopWatch.stop();

        // If the transfer failed, throw the underlying AWS exception if we can determine one. Otherwise, throw our own exception.
        TransferState transferState = transfer.getState();
        if (transferState == TransferState.Failed) {
            // The waitForException method should return the underlying AWS exception since the state is "Failed". It should not block since the
            // transfer is already "done" per previous code checking "isDone".
            AmazonClientException amazonClientException = transfer.waitForException();

            // If the returned exception is null, we weren't able to get the underlying AWS exception so just throw our own exception.
            // This is unlikely since the transfer failed, but it's better to handle the possibility just in case.
            if (amazonClientException == null) {
                throw new IllegalStateException("The transfer operation \"" + transfer.getDescription()
                        + "\" failed for an unknown reason.");
            }

            // Throw the Amazon underlying exception.
            throw amazonClientException;
        }
        // Ensure the transfer completed. If not, throw an exception.
        else if (transferState != TransferState.Completed) {
            throw new IllegalStateException("The transfer operation \"" + transfer.getDescription()
                    + "\" did not complete successfully. Current state: \"" + transferState + "\".");
        }

        // TransferProgress.getBytesTransferred() are not populated for S3 Copy objects.
        if (!(transfer instanceof Copy)) {
            // Sanity check for the number of bytes transferred.
            Assert.isTrue(transferProgress.getBytesTransferred() >= transferProgress.getTotalBytesToTransfer(),
                    String.format(
                            "Actual number of bytes transferred is less than expected (actual: %d bytes; expected: %d bytes).",
                            transferProgress.getBytesTransferred(),
                            transferProgress.getTotalBytesToTransfer()));
        }

        // Create the results object and populate it with the standard data.
        S3FileTransferResultsDto results = new S3FileTransferResultsDto();
        results.setDurationMillis(stopWatch.getTime());
        results.setTotalBytesTransferred(transfer.getProgress().getBytesTransferred());
        results.setTotalFilesTransferred(1L);

        if (transfer instanceof MultipleFileUpload) {
            // For upload directory, we need to calculate the total number of files transferred differently.
            results.setTotalFilesTransferred((long) ((MultipleFileUpload) transfer).getSubTransfers().size());
        } else if (transfer instanceof MultipleFileDownload) {
            // For download directory, we need to calculate the total number of files differently.
            results.setTotalFilesTransferred((long) listDirectory(params).size());
        }

        // Return the results.
        return results;
    } finally {
        // Shutdown the transfer manager to release resources. If this isn't done, the JVM may delay upon exiting.
        if (transferManager != null) {
            transferManager.shutdownNow();
        }
    }
}

---

From source file:org.finra.herd.dao.impl.S3DaoImpl.java

License:Apache License

/**
 * Logs transfer progress for an S3 file transfer operation.
 *
 * @param transferProgress the progress of an S3 transfer operation
 *//*from  www .j ava  2s .c o m*/
private void logTransferProgress(TransferProgress transferProgress) {
    // If the total bytes to transfer is set to 0, we do not log the transfer progress.
    if (LOGGER.isInfoEnabled() && transferProgress.getTotalBytesToTransfer() > 0) {
        NumberFormat formatter = new DecimalFormat("#0.0");

        LOGGER.info("progressBytesTransferred={} totalBytesToTransfer={} progressPercentTransferred={}",
                transferProgress.getBytesTransferred(), transferProgress.getTotalBytesToTransfer(),
                formatter.format(transferProgress.getPercentTransferred()));
    }
}

---

From source file:org.finra.herd.dao.impl.S3DaoImpl.java

License:Apache License

/**
 * Performs a file/directory transfer./*from   ww w.j  a v  a2  s  .c om*/
 *
 * @param params the parameters.
 * @param transferer a transferer that knows how to perform the transfer.
 *
 * @return the results.
 * @throws InterruptedException if a problem is encountered.
 */
private S3FileTransferResultsDto performTransfer(final S3FileTransferRequestParamsDto params,
        Transferer transferer) throws InterruptedException {
    // Create a transfer manager.
    TransferManager transferManager = getTransferManager(params);

    try {
        // Start a stop watch to keep track of how long the transfer takes.
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();

        // Perform the transfer.
        Transfer transfer = transferer.performTransfer(transferManager);
        TransferProgress transferProgress = transfer.getProgress();

        logTransferProgress(transferProgress);

        long stepCount = 0;

        // Loop until the transfer is complete.
        do {
            Thread.sleep(sleepIntervalsMillis);
            stepCount++;

            // Log progress status every 30 seconds and when transfer is complete.
            if (transfer.isDone() || stepCount % 300 == 0) {
                logTransferProgress(transferProgress);
            }
        } while (!transfer.isDone());

        // Stop the stop watch and create a results object.
        stopWatch.stop();

        // If the transfer failed, throw the underlying AWS exception if we can determine one. Otherwise, throw our own exception.
        TransferState transferState = transfer.getState();
        if (transferState == TransferState.Failed) {
            // The waitForException method should return the underlying AWS exception since the state is "Failed". It should not block since the
            // transfer is already "done" per previous code checking "isDone".
            AmazonClientException amazonClientException = transfer.waitForException();

            // If the returned exception is null, we weren't able to get the underlying AWS exception so just throw our own exception.
            // This is unlikely since the transfer failed, but it's better to handle the possibility just in case.
            if (amazonClientException == null) {
                throw new IllegalStateException("The transfer operation \"" + transfer.getDescription()
                        + "\" failed for an unknown reason.");
            }

            // Throw the Amazon underlying exception.
            throw amazonClientException;
        }
        // Ensure the transfer completed. If not, throw an exception.
        else if (transferState != TransferState.Completed) {
            throw new IllegalStateException("The transfer operation \"" + transfer.getDescription()
                    + "\" did not complete successfully. Current state: \"" + transferState + "\".");
        }

        // TransferProgress.getBytesTransferred() are not populated for S3 Copy objects.
        if (!(transfer instanceof Copy)) {
            // Sanity check for the number of bytes transferred.
            Assert.isTrue(transferProgress.getBytesTransferred() >= transferProgress.getTotalBytesToTransfer(),
                    String.format(
                            "Actual number of bytes transferred is less than expected (actual: %d bytes; expected: %d bytes).",
                            transferProgress.getBytesTransferred(),
                            transferProgress.getTotalBytesToTransfer()));
        }

        // Create the results object and populate it with the standard data.
        S3FileTransferResultsDto results = new S3FileTransferResultsDto();
        results.setDurationMillis(stopWatch.getTime());
        results.setTotalBytesTransferred(transfer.getProgress().getBytesTransferred());
        results.setTotalFilesTransferred(1L);

        if (transfer instanceof MultipleFileUpload) {
            // For upload directory, we need to calculate the total number of files transferred differently.
            results.setTotalFilesTransferred((long) ((MultipleFileUpload) transfer).getSubTransfers().size());
        } else if (transfer instanceof MultipleFileDownload) {
            // For download directory, we need to calculate the total number of files differently.
            results.setTotalFilesTransferred((long) listDirectory(params).size());
        }

        // Return the results.
        return results;
    } finally {
        // Shutdown the transfer manager to release resources. If this isn't done, the JVM may delay upon exiting.
        transferManager.shutdownNow();
    }
}

---