List of usage examples for com.amazonaws.services.s3.transfer TransferManager TransferManager
protected TransferManager(TransferManagerBuilder builder)
From source file:com.forerunnergames.peril.client.assets.S3AssetUpdater.java
License:Open Source License
S3AssetUpdater(final String bucketPath) { Arguments.checkIsNotNull(bucketPath, "bucketPath"); Preconditions.checkIsTrue(AssetSettings.isValidS3BucketPath(bucketPath), AssetSettings.VALID_S3_BUCKET_PATH_DESCRIPTION); bucketName = AssetSettings.getS3BucketName(bucketPath); executorService = Executors.newSingleThreadExecutor(); final ClientConfiguration clientConfig = new ClientConfiguration().withMaxErrorRetry(10) .withConnectionTimeout(10_000).withSocketTimeout(10_000).withTcpKeepAlive(true); final AmazonS3 s3 = new AmazonS3Client(new ProfileCredentialsProvider(), clientConfig); transferManager = new TransferManager(s3); }
From source file:com.gendevs.bedrock.appengine.service.storage.StorageProvider.java
License:Apache License
private void initalizeS3() { AWSCredentials credentials = new BasicAWSCredentials(StorageConstants.ACCESS_KEY, StorageConstants.SECRET_KEY); if (s3Client == null) { s3Client = new AmazonS3Client(credentials); s3Client.setRegion(region);/*from w ww. jav a 2 s . co m*/ s3Client.setEndpoint("s3-us-west-2.amazonaws.com"); } manager = new TransferManager(credentials); }
From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java
License:Open Source License
@Override public boolean uploadObjectAndListenProgress(final String bucketName, final String fileName, final InputStream inputStream, final CannedAccessControlList cannedAcl) throws AmazonClientException, AmazonServiceException, IOException { LOGGER.info(//from w w w . j a v a 2 s.co m "uploadObjectAndListenProgress invoked, bucketName: {} , fileName: {} and cannedAccessControlList: {}", bucketName, fileName, cannedAcl); File tempFile = null; PutObjectRequest putObjectRequest = null; Upload upload = null; try { // Create temporary file from stream to avoid 'out of memory' exception tempFile = AWSUtil.createTempFileFromStream(inputStream); putObjectRequest = new PutObjectRequest(bucketName, fileName, tempFile).withCannedAcl(cannedAcl); final TransferManager transferMgr = new TransferManager(s3client); upload = transferMgr.upload(putObjectRequest); // You can poll your transfer's status to check its progress if (upload.isDone()) { LOGGER.info("Start: {} , State: {} and Progress (%): {}", upload.getDescription(), upload.getState(), upload.getProgress().getPercentTransferred()); } // Add progressListener to listen asynchronous notifications about your transfer's progress // Uncomment below code snippet during development /*upload.addProgressListener(new ProgressListener() { public void progressChanged(ProgressEvent event) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Transferred bytes: " + (long) event.getBytesTransferred()); } } });*/ try { //Block the current thread and wait for completion //If the transfer fails AmazonClientException will be thrown upload.waitForCompletion(); } catch (AmazonClientException | InterruptedException excp) { LOGGER.error("Exception occured while waiting for transfer: ", excp); } } finally { AWSUtil.deleteTempFile(tempFile); // Delete the temporary file once uploaded } LOGGER.info("End: {} , State: {} , Progress (%): {}", upload.getDescription(), upload.getState(), upload.getProgress().getPercentTransferred()); return upload.isDone(); }
From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java
License:Open Source License
@Override public boolean uploadObjectAndListenProgress(final String bucketName, final String fileName, final InputStream inputStream, final boolean isPublicAccessible) throws AmazonClientException, AmazonServiceException, IOException { LOGGER.info(/*from w w w. j a v a 2 s . co m*/ "uploadObjectAndListenProgress invoked, bucketName: {} , fileName: {} and isPublicAccessible: {}", bucketName, fileName, isPublicAccessible); File tempFile = null; PutObjectRequest putObjectRequest = null; Upload upload = null; try { // Create temporary file from stream to avoid 'out of memory' exception tempFile = AWSUtil.createTempFileFromStream(inputStream); putObjectRequest = new PutObjectRequest(bucketName, fileName, tempFile); if (isPublicAccessible) { putObjectRequest.setCannedAcl(CannedAccessControlList.PublicRead); } final TransferManager transferMgr = new TransferManager(s3client); upload = transferMgr.upload(putObjectRequest); // You can poll your transfer's status to check its progress if (upload.isDone()) { LOGGER.info("Start: {} , State: {} and Progress (%): {}", upload.getDescription(), upload.getState(), upload.getProgress().getPercentTransferred()); } // Add progressListener to listen asynchronous notifications about your transfer's progress // Uncomment below code snippet during development /*upload.addProgressListener(new ProgressListener() { public void progressChanged(ProgressEvent event) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Transferred bytes: " + (long) event.getBytesTransferred()); } } });*/ try { //Block the current thread and wait for completion //If the transfer fails AmazonClientException will be thrown upload.waitForCompletion(); } catch (AmazonClientException | InterruptedException excp) { LOGGER.error("Exception occured while waiting for transfer: ", excp); } } finally { AWSUtil.deleteTempFile(tempFile); // Delete the temporary file once uploaded } LOGGER.info("End: {} , State: {} , Progress (%): {}", upload.getDescription(), upload.getState(), upload.getProgress().getPercentTransferred()); return upload.isDone(); }
From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java
License:Open Source License
@Override public Upload uploadFileAsync(final String bucketName, final String fileName, final File fileObj, final CannedAccessControlList cannedAcl) throws AmazonClientException, AmazonServiceException, IOException { LOGGER.info("uploadObjectAsync invoked, bucketName: {} , fileName: {} and cannedAccessControlList: {}", bucketName, fileName, cannedAcl); final PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileName, fileObj) .withCannedAcl(cannedAcl);//w w w. ja v a 2 s . c o m final TransferManager transferMgr = new TransferManager(s3client); return transferMgr.upload(putObjectRequest); }
From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java
License:Open Source License
@Override public Upload uploadFileAsync(final String bucketName, final String fileName, final File fileObj, final boolean isPublicAccessible) throws AmazonClientException, AmazonServiceException, IOException { LOGGER.info("uploadObjectAsync invoked, bucketName: {} , fileName: {} and isPublicAccessible: {}", bucketName, fileName, isPublicAccessible); final PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileName, fileObj); if (isPublicAccessible) { putObjectRequest.setCannedAcl(CannedAccessControlList.PublicRead); }//from w w w . j a v a 2 s.c o m final TransferManager transferMgr = new TransferManager(s3client); return transferMgr.upload(putObjectRequest); }
From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java
License:Open Source License
@Override public boolean uploadDirectoryOrFileAndListenProgress(final String bucketName, final File source, final String virtualDirectoryKeyPrefix) throws AmazonClientException, AmazonServiceException, FileNotFoundException { LOGGER.info("uploadDirectoryOrFileAndWaitForCompletion invoked, bucketName: {} , Source: {}", bucketName, source.getAbsolutePath());/*w w w . j a v a 2 s. c om*/ Transfer transfer = null; final TransferManager transferMgr = new TransferManager(s3client); if (source.isFile()) { transfer = transferMgr.upload(bucketName, source.getPath(), source); } else if (source.isDirectory()) { //upload recursively transfer = transferMgr.uploadDirectory(bucketName, virtualDirectoryKeyPrefix, source, true); } else { throw new FileNotFoundException("File is neither a regular file nor a directory " + source); } // You can poll your transfer's status to check its progress if (transfer.isDone()) { LOGGER.info("Start: {} , State: {} and Progress (%): {}", transfer.getDescription(), transfer.getState(), transfer.getProgress().getPercentTransferred()); } // Add progressListener to listen asynchronous notifications about your transfer's progress // Uncomment below code snippet during development /*transfer.addProgressListener(new ProgressListener() { public void progressChanged(ProgressEvent event) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Transferred bytes: " + (long) event.getBytesTransferred()); } } });*/ try { //Block the current thread and wait for completion //If the transfer fails AmazonClientException will be thrown transfer.waitForCompletion(); } catch (AmazonClientException | InterruptedException excp) { LOGGER.error("Exception occured while waiting for transfer: ", excp); } LOGGER.info("End: {} , State: {} , Progress (%): {}", transfer.getDescription(), transfer.getState(), transfer.getProgress().getPercentTransferred()); return transfer.isDone(); }
From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java
License:Open Source License
@Override public Transfer uploadDirectoryOrFile(final String bucketName, final File source, final String virtualDirectoryKeyPrefix) throws AmazonClientException, AmazonServiceException, IOException { LOGGER.info("uploadDirectoryOrFile invoked, bucketName: {} , Source: {}", bucketName, source.getAbsolutePath());//from ww w . j a v a 2s . co m Transfer transfer = null; final TransferManager trMgr = new TransferManager(s3client); if (source.isFile()) { transfer = trMgr.upload(bucketName, source.getPath(), source); } else if (source.isDirectory()) { //Upload recursively //virtualDirectoryKeyPrefix could be virtual directory name inside the bucket transfer = trMgr.uploadDirectory(bucketName, virtualDirectoryKeyPrefix, source, true); } else { throw new FileNotFoundException("Source is neither a regular file nor a directory " + source); } return transfer; }
From source file:com.github.rholder.esthree.command.Put.java
License:Apache License
@Override public Integer call() throws Exception { TransferManager t = new TransferManager(amazonS3Client); ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setUserMetadata(metadata); Upload u = t.upload(new PutObjectRequest(bucket, key, inputFile).withMetadata(objectMetadata)); // TODO this listener spews out garbage >100% on a retry, add a test to verify if (progressListener != null) { progressListener.withTransferProgress(new TransferProgressWrapper(u.getProgress())); u.addProgressListener(progressListener); }/*from w ww . ja v a 2 s .com*/ try { u.waitForCompletion(); } finally { t.shutdownNow(); } return 0; }
From source file:com.hpe.caf.worker.datastore.s3.S3DataStore.java
License:Apache License
private String store(InputStream inputStream, String partialReference, Long length) throws DataStoreException { try {//from w w w . j a v a 2 s.c om String fullReference = partialReference + UUID.randomUUID().toString(); ObjectMetadata objectMetadata = new ObjectMetadata(); if (length != null) { objectMetadata.setContentLength(length); } TransferManager transferManager = new TransferManager(amazonS3Client); Upload upload = transferManager.upload(bucketName, fullReference, inputStream, objectMetadata); upload.waitForCompletion(); // amazonS3Client.putObject(bucketName, fullReference, inputStream, objectMetadata); transferManager.shutdownNow(false); return fullReference; } catch (Exception ex) { errors.incrementAndGet(); throw new DataStoreException("Could not store input stream.", ex); } }