List of usage examples for com.amazonaws.services.s3.transfer TransferManager uploadDirectory
public MultipleFileUpload uploadDirectory(String bucketName, String virtualDirectoryKeyPrefix, File directory, boolean includeSubdirectories)
From source file:aws.example.s3.XferMgrProgress.java
License:Open Source License
public static void uploadDirWithSubprogress(String dir_path, String bucket_name, String key_prefix, boolean recursive, boolean pause) { System.out//from w w w . j a v a2 s. co m .println("directory: " + dir_path + (recursive ? " (recursive)" : "") + (pause ? " (pause)" : "")); TransferManager xfer_mgr = new TransferManager(); try { MultipleFileUpload multi_upload = xfer_mgr.uploadDirectory(bucket_name, key_prefix, new File(dir_path), recursive); // loop with Transfer.isDone() XferMgrProgress.showMultiUploadProgress(multi_upload); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(multi_upload); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
From source file:aws.example.s3.XferMgrUpload.java
License:Open Source License
public static void uploadDir(String dir_path, String bucket_name, String key_prefix, boolean recursive, boolean pause) { System.out/*from w w w .ja v a 2s . com*/ .println("directory: " + dir_path + (recursive ? " (recursive)" : "") + (pause ? " (pause)" : "")); TransferManager xfer_mgr = new TransferManager(); try { MultipleFileUpload xfer = xfer_mgr.uploadDirectory(bucket_name, key_prefix, new File(dir_path), recursive); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java
License:Open Source License
@Override public boolean uploadDirectoryOrFileAndListenProgress(final String bucketName, final File source, final String virtualDirectoryKeyPrefix) throws AmazonClientException, AmazonServiceException, FileNotFoundException { LOGGER.info("uploadDirectoryOrFileAndWaitForCompletion invoked, bucketName: {} , Source: {}", bucketName, source.getAbsolutePath());/*from w ww . j ava2 s . c o m*/ Transfer transfer = null; final TransferManager transferMgr = new TransferManager(s3client); if (source.isFile()) { transfer = transferMgr.upload(bucketName, source.getPath(), source); } else if (source.isDirectory()) { //upload recursively transfer = transferMgr.uploadDirectory(bucketName, virtualDirectoryKeyPrefix, source, true); } else { throw new FileNotFoundException("File is neither a regular file nor a directory " + source); } // You can poll your transfer's status to check its progress if (transfer.isDone()) { LOGGER.info("Start: {} , State: {} and Progress (%): {}", transfer.getDescription(), transfer.getState(), transfer.getProgress().getPercentTransferred()); } // Add progressListener to listen asynchronous notifications about your transfer's progress // Uncomment below code snippet during development /*transfer.addProgressListener(new ProgressListener() { public void progressChanged(ProgressEvent event) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Transferred bytes: " + (long) event.getBytesTransferred()); } } });*/ try { //Block the current thread and wait for completion //If the transfer fails AmazonClientException will be thrown transfer.waitForCompletion(); } catch (AmazonClientException | InterruptedException excp) { LOGGER.error("Exception occured while waiting for transfer: ", excp); } LOGGER.info("End: {} , State: {} , Progress (%): {}", transfer.getDescription(), transfer.getState(), transfer.getProgress().getPercentTransferred()); return transfer.isDone(); }
From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java
License:Open Source License
@Override public Transfer uploadDirectoryOrFile(final String bucketName, final File source, final String virtualDirectoryKeyPrefix) throws AmazonClientException, AmazonServiceException, IOException { LOGGER.info("uploadDirectoryOrFile invoked, bucketName: {} , Source: {}", bucketName, source.getAbsolutePath());// w ww. j a v a2s . c om Transfer transfer = null; final TransferManager trMgr = new TransferManager(s3client); if (source.isFile()) { transfer = trMgr.upload(bucketName, source.getPath(), source); } else if (source.isDirectory()) { //Upload recursively //virtualDirectoryKeyPrefix could be virtual directory name inside the bucket transfer = trMgr.uploadDirectory(bucketName, virtualDirectoryKeyPrefix, source, true); } else { throw new FileNotFoundException("Source is neither a regular file nor a directory " + source); } return transfer; }
From source file:com.mesosphere.dcos.cassandra.executor.backup.S3StorageDriver.java
License:Apache License
private void uploadDirectory(TransferManager tx, String bucketName, String key, String keyspaceName, String cfName, File snapshotDirectory) throws Exception { try {//from w w w. j av a 2 s .c om final String fileKey = key + "/" + keyspaceName + "/" + cfName + "/"; final MultipleFileUpload myUpload = tx.uploadDirectory(bucketName, fileKey, snapshotDirectory, true); myUpload.waitForCompletion(); } catch (Exception e) { LOGGER.error("Error occurred on uploading directory {} : {}", snapshotDirectory.getName(), e); throw new Exception(e); } }
From source file:com.netflix.genie.common.internal.services.impl.S3JobArchiverImpl.java
License:Apache License
/** * {@inheritDoc}/*from ww w. j av a 2 s . c o m*/ */ @Override public boolean archiveDirectory(@NotNull final Path directory, @NotNull final URI target) throws JobArchiveException { final String uriString = target.toString(); final AmazonS3URI s3URI; try { s3URI = new AmazonS3URI(target); } catch (final IllegalArgumentException iae) { log.debug("{} is not a valid S3 URI", uriString); return false; } final String directoryString = directory.toString(); log.debug("{} is a valid S3 location. Proceeding to archive {} to location: {}", uriString, directoryString, uriString); try { final TransferManager transferManager = this.s3ClientFactory.getTransferManager(s3URI); final MultipleFileUpload upload = transferManager.uploadDirectory(s3URI.getBucket(), s3URI.getKey(), directory.toFile(), true); upload.waitForCompletion(); return true; } catch (final Exception e) { log.error("Error archiving to S3 location: {} ", uriString, e); throw new JobArchiveException("Error archiving " + directoryString, e); } }
From source file:doug.iotdemo.mojo.deployer.Deployer.java
License:Open Source License
@Override public void execute() throws MojoExecutionException, MojoFailureException { AmazonS3 s3 = new AmazonS3Client(); if (!s3.doesBucketExist(bucketName)) { throw new MojoExecutionException("bucket " + bucketName + " does not exist"); }//w w w .ja va 2s .c o m getLog().info("Uploading " + source.getName() + " to s3://" + bucketName + "/" + bucketKey); TransferManager manager = new TransferManager(s3); Transfer transfer; if (source.isFile()) { transfer = manager.upload(bucketName, bucketKey, source); } else if (source.isDirectory()) { transfer = manager.uploadDirectory(bucketName, bucketKey, source, true); } else { throw new MojoExecutionException("Unknown file type " + source.getAbsolutePath()); } try { transfer.waitForCompletion(); } catch (InterruptedException e) { throw new MojoExecutionException("Upload to S3 failed", e); } }
From source file:ecplugins.s3.S3Util.java
License:Apache License
public static boolean UploadFolder(String bucketName, String key) throws AmazonClientException, AmazonServiceException, Exception { Properties props = TestUtils.getProperties(); File file = new File(createFolder()); BasicAWSCredentials credentials = new BasicAWSCredentials(props.getProperty(StringConstants.ACCESS_ID), props.getProperty(StringConstants.SECRET_ACCESS_ID)); // Create TransferManager TransferManager tx = new TransferManager(credentials); // Get S3 Client AmazonS3 s3 = tx.getAmazonS3Client(); MultipleFileUpload objectUpload = tx.uploadDirectory(bucketName, key, file, true); while (!objectUpload.isDone()) { Thread.sleep(1000);// w ww. j av a 2 s . c o m } return true; }
From source file:org.ow2.proactive.scheduler.examples.S3ConnectorUploader.java
License:Open Source License
/** * Upload a local directory to S3. <br> * Requires a bucket name. <br>/*from www . ja va 2 s .c om*/ * If recursive is set to true, upload all subdirectories recursively. * * @param dirPath local directory to upload * @param bucketName * @param keyPrefix * @param recursive * @param pause * @param s3Client */ private void uploadDir(String dirPath, String bucketName, String keyPrefix, boolean recursive, boolean pause, AmazonS3 s3Client) { getOut().println( "directory: " + dirPath + (recursive ? " (recursive)" : "") + (pause ? " (" + PAUSE + ")" : "")); File folder = new File(dirPath); String keyName = (keyPrefix != null) ? Paths.get(keyPrefix, folder.getName()).toString() : folder.getName(); TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(s3Client).build(); try { MultipleFileUpload uploader = transferManager.uploadDirectory(bucketName, keyName, folder, recursive); // loop with Transfer.isDone() SchedulerExamplesUtils.showTransferProgress(uploader); // or block with Transfer.waitForCompletion() SchedulerExamplesUtils.waitForCompletion(uploader); } catch (AmazonServiceException e) { getErr().println(e.getErrorMessage()); System.exit(1); } transferManager.shutdownNow(); }