Example usage for com.amazonaws.services.s3.transfer TransferManager download

List of usage examples for com.amazonaws.services.s3.transfer TransferManager download

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.transfer.*
  5. TransferManager
  6. download

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.transfer TransferManager download.

Prototype

public PresignedUrlDownload download(final PresignedUrlDownloadRequest request, final File destFile,
        final PresignedUrlDownloadConfig downloadContext)

Source Link

Document

Schedules a new transfer to download data from Amazon S3 using presigned url and save it to the specified file.

Usage

From source file:aws.example.s3.XferMgrDownload.java

License:Open Source License

public static void downloadFile(String bucket_name, String key_name, String file_path, boolean pause) {
    System.out.println("Downloading to file: " + file_path + (pause ? " (pause)" : ""));

    File f = new File(file_path);
    TransferManager xfer_mgr = new TransferManager();
    try {//from w  w  w. j a va  2s . c  om
        Download xfer = xfer_mgr.download(bucket_name, key_name, f);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}

From source file:com.emc.vipr.s3.sample._10_ReadLargeObjectTM.java

License:Open Source License

public static void main(String[] args) throws Exception {
    // create the AWS S3 Client
    AmazonS3 s3 = AWSS3Factory.getS3Client();

    // retrieve the key value from user
    System.out.println("Enter the object key:");
    String key = new BufferedReader(new InputStreamReader(System.in)).readLine();

    // print start time
    Date start_date = new Date();
    System.out.println(start_date.toString());

    // file will be placed in temp dir with .tmp extension
    File file = File.createTempFile("read-large-object-tm", null);

    TransferManager tm = TransferManagerBuilder.standard().withS3Client(s3).build();

    // download the object to file
    Download download = tm.download(AWSS3Factory.S3_BUCKET, key, file);

    // block until download finished
    download.waitForCompletion();//from  ww  w.j a  v a2s  .  c o  m

    tm.shutdownNow();

    // print end time
    Date end_date = new Date();
    System.out.println(end_date.toString());
}

From source file:com.mesosphere.dcos.cassandra.executor.backup.S3StorageDriver.java

License:Apache License

private void downloadFile(TransferManager tx, String bucketName, String sourcePrefixKey, String destinationFile)
        throws Exception {
    try {/*from  w  ww  .  jav  a 2 s  .c  o m*/
        final File snapshotFile = new File(destinationFile);
        // Only create parent directory once, if it doesn't exist.
        final File parentDir = new File(snapshotFile.getParent());
        if (!parentDir.isDirectory()) {
            final boolean parentDirCreated = parentDir.mkdirs();
            if (!parentDirCreated) {
                LOGGER.error("Error creating parent directory for file: {}. Skipping to next", destinationFile);
                return;
            }
        }
        snapshotFile.createNewFile();
        final Download download = tx.download(bucketName, sourcePrefixKey, snapshotFile);
        download.waitForCompletion();
    } catch (Exception e) {
        LOGGER.error("Error downloading the file {} : {}", destinationFile, e);
        throw new Exception(e);
    }
}

From source file:jenkins.plugins.itemstorage.s3.Downloads.java

License:Open Source License

public void startDownload(TransferManager manager, File base, String pathPrefix, S3ObjectSummary summary)
        throws AmazonServiceException, IOException {
    // calculate target file name
    File targetFile = FileUtils.getFile(base, summary.getKey().substring(pathPrefix.length() + 1));

    // if target file exists, only download it if newer
    if (targetFile.lastModified() < summary.getLastModified().getTime()) {
        // ensure directory above file exists
        FileUtils.forceMkdir(targetFile.getParentFile());

        // Start the download
        Download download = manager.download(summary.getBucketName(), summary.getKey(), targetFile);

        // Keep for later
        startedDownloads.add(new Memo(download, targetFile, summary.getLastModified().getTime()));
    }/*  w  w  w .jav a  2  s  . c o m*/
}

From source file:jetbrains.buildServer.codepipeline.CodePipelineBuildListener.java

License:Apache License

private void processJobInput(@NotNull final AgentRunningBuild build) {
    if (myJobInputProcessed)
        return;//from  www. j a  va2s.com
    myJobInputProcessed = true;

    final Map<String, String> params = build.getSharedConfigParameters();
    myJobID = getJobId(params);
    if (myJobID == null) {
        LOG.debug(msgForBuild("No AWS CodePipeline job found for the build", build));
        return;
    }

    AWSCommonParams.withAWSClients(params, new AWSCommonParams.WithAWSClients<Void, RuntimeException>() {
        @Nullable
        @Override
        public Void run(@NotNull AWSClients clients) throws RuntimeException {
            AWSCodePipelineClient codePipelineClient = null;
            try {
                codePipelineClient = clients.createCodePipeLineClient();
                final JobData jobData = getJobData(codePipelineClient, params);

                final PipelineContext pipelineContext = jobData.getPipelineContext();
                build.getBuildLogger().message("This build is a part of an AWS CodePipeline pipeline: "
                        + pipelineContext.getPipelineName()
                        + "\nLink: https://console.aws.amazon.com/codepipeline/home?region="
                        + params.get(AWSCommonParams.REGION_NAME_PARAM) + "#/view/"
                        + pipelineContext.getPipelineName() + "\nStage: " + pipelineContext.getStage().getName()
                        + "\nAction: " + pipelineContext.getAction().getName() + "\nJob ID: " + myJobID);

                final List<Artifact> inputArtifacts = jobData.getInputArtifacts();
                if (inputArtifacts.isEmpty()) {
                    LOG.debug(
                            msgForBuild("No input artifacts provided for the job with ID: " + myJobID, build));
                } else {

                    final File inputFolder = new File(params.get(ARTIFACT_INPUT_FOLDER_CONFIG_PARAM));
                    FileUtil.createDir(inputFolder);

                    final Collection<Download> downloads = S3Util.withTransferManager(
                            getArtifactS3Client(jobData.getArtifactCredentials(), params),
                            new S3Util.WithTransferManager<Download>() {
                                @NotNull
                                @Override
                                public Collection<Download> run(@NotNull final TransferManager manager)
                                        throws Throwable {
                                    return CollectionsUtil.convertCollection(inputArtifacts,
                                            new Converter<Download, Artifact>() {
                                                @Override
                                                public Download createFrom(@NotNull Artifact artifact) {
                                                    final S3ArtifactLocation s3Location = artifact.getLocation()
                                                            .getS3Location();
                                                    final File destinationFile = getInputArtifactFile(
                                                            inputFolder, s3Location.getObjectKey());

                                                    build.getBuildLogger()
                                                            .message("Downloading job input artifact "
                                                                    + s3Location.getObjectKey() + " to "
                                                                    + destinationFile.getAbsolutePath());
                                                    return manager.download(s3Location.getBucketName(),
                                                            s3Location.getObjectKey(), destinationFile);
                                                }
                                            });
                                }
                            });
                    // for backward compatibility, TW-47902
                    for (Download d : downloads) {
                        makeArtifactCopy(inputFolder, getInputArtifactFile(inputFolder, d.getKey()), d.getKey(),
                                build);
                    }
                    if (!jobData.getOutputArtifacts().isEmpty()) {
                        FileUtil.createDir(new File(params.get(ARTIFACT_OUTPUT_FOLDER_CONFIG_PARAM)));
                    }
                }
            } catch (Throwable e) {
                failOnException(codePipelineClient, build, e);
            }
            return null;
        }
    });
}

From source file:org.apache.flink.streaming.tests.util.s3.S3UtilProgram.java

License:Apache License

private static void downloadFile(ParameterTool params) {
    final String bucket = params.getRequired("bucket");
    final String s3file = params.getRequired("s3file");
    final String localFile = params.getRequired("localFile");
    TransferManager tx = TransferManagerBuilder.defaultTransferManager();
    try {// w ww  . jav a 2s  .co  m
        tx.download(bucket, s3file, new File(localFile)).waitForCompletion();
    } catch (InterruptedException e) {
        System.out.println("Transfer interrupted");
    } finally {
        tx.shutdownNow();
    }
}

From source file:org.finra.dm.dao.impl.S3OperationsImpl.java

License:Apache License

/**
 * Implementation delegates to {@link TransferManager#download(String, String, File)}
 *//*ww  w .ja v a2  s .com*/
@Override
public Download download(String bucket, String key, File file, TransferManager transferManager) {
    return transferManager.download(bucket, key, file);
}

From source file:org.finra.herd.dao.impl.S3OperationsImpl.java

License:Apache License

@Override
public Download download(String s3BucketName, String s3Key, File file, TransferManager transferManager) {
    return transferManager.download(s3BucketName, s3Key, file);
}

From source file:org.opendedup.sdfs.filestore.cloud.BatchAwsS3ChunkStore.java

License:Open Source License

private void multiPartDownload(String keyName, File f)
        throws AmazonServiceException, AmazonClientException, InterruptedException {
    TransferManager tx = null;
    try {//from   w  w w  .j  a v  a2 s .  c  o  m
        if (awsCredentials != null)
            tx = new TransferManager(awsCredentials);
        else
            tx = new TransferManager(new InstanceProfileCredentialsProvider());
        Download myDownload = tx.download(this.name, keyName, f);
        myDownload.waitForCompletion();
    } finally {
        if (tx != null)
            tx.shutdownNow();
    }
}

From source file:org.ow2.proactive.scheduler.examples.S3ConnectorDownloader.java

License:Open Source License

/**
 * Download a file from S3. <br>/* ww w . j  a  v  a  2  s .c o  m*/
 * Requires a bucket name. <br>
 * Requires a key prefix. <br>
 *
 * @param bucketName
 * @param keyName
 * @param filePath
 * @param pause
 * @param s3Client
 */
private void downloadFile(String bucketName, String keyName, String filePath, boolean pause,
        AmazonS3 s3Client) {
    getOut().println("Downloading to file: " + filePath + (pause ? " (pause)" : ""));

    File f = new File(filePath);
    TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3Client).build();
    try {
        Download xfer = xferMgr.download(bucketName, keyName, f);
        // loop with Transfer.isDone()
        SchedulerExamplesUtils.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        SchedulerExamplesUtils.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        getErr().println(e.getMessage());
        System.exit(1);
    } finally {
        xferMgr.shutdownNow();
    }
}