Example usage for com.amazonaws.services.s3.transfer TransferManager getAmazonS3Client

List of usage examples for com.amazonaws.services.s3.transfer TransferManager getAmazonS3Client

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.transfer.*
  5. TransferManager
  6. getAmazonS3Client

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.transfer TransferManager getAmazonS3Client.

Prototype

public AmazonS3 getAmazonS3Client()

Source Link

Document

Returns the underlying Amazon S3 client used to make requests to Amazon S3.

Usage

From source file:com.davidsoergel.s3napback.S3ops.java

License:Apache License

public static void delete(TransferManager tx, String bucket, String fileprefix) throws InterruptedException {
    logger.info("Deleting " + fileprefix);

    List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<DeleteObjectsRequest.KeyVersion>();

    ObjectListing objectListing = tx.getAmazonS3Client()
            .listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(fileprefix));
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        keys.add(new DeleteObjectsRequest.KeyVersion(objectSummary.getKey()));
    }//from w  w  w.  j a  va2  s .  c  om

    DeleteObjectsRequest req = new DeleteObjectsRequest(bucket);
    req.setKeys(keys);
    DeleteObjectsResult result = tx.getAmazonS3Client().deleteObjects(req);
}

From source file:ecplugins.s3.S3Util.java

License:Apache License

/**
 * This procedure deletes the bucket along with its contents
 * @param bucketName//from   ww w . j  av a  2  s . c om
 * @return
 * @throws Exception
 */
public static boolean DeleteBucket(String bucketName) throws Exception {

    Properties props = TestUtils.getProperties();

    BasicAWSCredentials credentials = new BasicAWSCredentials(props.getProperty(StringConstants.ACCESS_ID),
            props.getProperty(StringConstants.SECRET_ACCESS_ID));

    // Create TransferManager
    TransferManager tx = new TransferManager(credentials);

    // Get S3 Client
    AmazonS3 s3 = tx.getAmazonS3Client();

    if (s3.doesBucketExist(bucketName)) {
        // Multi-object delete by specifying only keys (no version ID).
        DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucketName).withQuiet(false);

        //get keys
        List<String> keys = new ArrayList<String>();
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName));
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            keys.add(objectSummary.getKey());
        }

        // Create request that include only object key names.
        List<DeleteObjectsRequest.KeyVersion> justKeys = new ArrayList<DeleteObjectsRequest.KeyVersion>();
        for (String key : keys) {
            justKeys.add(new DeleteObjectsRequest.KeyVersion(key));
        }

        if (justKeys.size() == 0) {
            return false;
        }

        multiObjectDeleteRequest.setKeys(justKeys);
        // Execute DeleteObjects - Amazon S3 add delete marker for each object
        // deletion. The objects no disappear from your bucket (verify).
        DeleteObjectsResult delObjRes = null;

        delObjRes = s3.deleteObjects(multiObjectDeleteRequest);

        s3.deleteBucket(bucketName);
        return true;
    } else {
        System.out.println("Error: Bucket with name " + bucketName + " does not exists.");
        return false;
    }
}

From source file:ecplugins.s3.S3Util.java

License:Apache License

public static void CreateBucket(String bucketName) throws Exception {

    Properties props = TestUtils.getProperties();

    BasicAWSCredentials credentials = new BasicAWSCredentials(props.getProperty(StringConstants.ACCESS_ID),
            props.getProperty(StringConstants.SECRET_ACCESS_ID));

    // Create TransferManager
    TransferManager tx = new TransferManager(credentials);

    // Get S3 Client
    AmazonS3 s3 = tx.getAmazonS3Client();

    s3.createBucket(bucketName);//from  w  w w . ja v a 2s  .  c  o m

}

From source file:ecplugins.s3.S3Util.java

License:Apache License

public static boolean CheckIsBucketAvailable(String bucketName) throws Exception {

    Properties props = TestUtils.getProperties();

    BasicAWSCredentials credentials = new BasicAWSCredentials(props.getProperty(StringConstants.ACCESS_ID),
            props.getProperty(StringConstants.SECRET_ACCESS_ID));

    // Create TransferManager
    TransferManager tx = new TransferManager(credentials);

    // Get S3 Client
    AmazonS3 s3 = tx.getAmazonS3Client();

    return s3.doesBucketExist(bucketName);
}

From source file:ecplugins.s3.S3Util.java

License:Apache License

public static int ListBuckets() throws Exception {
    Properties props = TestUtils.getProperties();

    BasicAWSCredentials credentials = new BasicAWSCredentials(props.getProperty(StringConstants.ACCESS_ID),
            props.getProperty(StringConstants.SECRET_ACCESS_ID));

    // Create TransferManager
    TransferManager tx = new TransferManager(credentials);

    // Get S3 Client
    AmazonS3 s3 = tx.getAmazonS3Client();

    List<Bucket> buckets = s3.listBuckets();

    return buckets.size();
}

From source file:ecplugins.s3.S3Util.java

License:Apache License

public static void UploadObject(String bucketName, String key)
        throws AmazonClientException, AmazonServiceException, Exception {
    Properties props = TestUtils.getProperties();
    File file = new File(createFile());
    BasicAWSCredentials credentials = new BasicAWSCredentials(props.getProperty(StringConstants.ACCESS_ID),
            props.getProperty(StringConstants.SECRET_ACCESS_ID));

    // Create TransferManager
    TransferManager tx = new TransferManager(credentials);

    // Get S3 Client
    AmazonS3 s3 = tx.getAmazonS3Client();

    try {/* w  w w. j a  v a 2 s.  co m*/
        System.out.println("Uploading a new object to S3 from a file\n");

        s3.putObject(new PutObjectRequest(bucketName, key, file));

    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException");
        System.out.println("Error Message:    " + ase.getMessage());
        System.out.println("HTTP Status Code: " + ase.getStatusCode());
        System.out.println("AWS Error Code:   " + ase.getErrorCode());
        System.out.println("Error Type:       " + ase.getErrorType());
        System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println(
                "Caught an AmazonClientException, which  means the client encountered  an internal error while trying to such as not being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    }

}

From source file:ecplugins.s3.S3Util.java

License:Apache License

public static boolean UploadFolder(String bucketName, String key)
        throws AmazonClientException, AmazonServiceException, Exception {
    Properties props = TestUtils.getProperties();
    File file = new File(createFolder());
    BasicAWSCredentials credentials = new BasicAWSCredentials(props.getProperty(StringConstants.ACCESS_ID),
            props.getProperty(StringConstants.SECRET_ACCESS_ID));

    // Create TransferManager
    TransferManager tx = new TransferManager(credentials);

    // Get S3 Client
    AmazonS3 s3 = tx.getAmazonS3Client();
    MultipleFileUpload objectUpload = tx.uploadDirectory(bucketName, key, file, true);

    while (!objectUpload.isDone()) {
        Thread.sleep(1000);/*w ww  . java2 s.co  m*/
    }

    return true;
}

From source file:ecplugins.s3.S3Util.java

License:Apache License

public static boolean isValidFile(String bucketName, String path)
        throws AmazonClientException, AmazonServiceException, Exception {
    boolean isValidFile = true;
    Properties props = TestUtils.getProperties();

    BasicAWSCredentials credentials = new BasicAWSCredentials(props.getProperty(StringConstants.ACCESS_ID),
            props.getProperty(StringConstants.SECRET_ACCESS_ID));

    // Create TransferManager
    TransferManager tx = new TransferManager(credentials);

    // Get S3 Client
    AmazonS3 s3 = tx.getAmazonS3Client();

    try {//  w w  w.j  ava  2s .c  om
        ObjectMetadata objectMetadata = s3.getObjectMetadata(bucketName, path);
    } catch (AmazonS3Exception s3e) {
        if (s3e.getStatusCode() == 404) {
            // i.e. 404: NoSuchKey - The specified key does not exist
            isValidFile = false;
        } else {
            throw s3e; // rethrow all S3 exceptions other than 404
        }
    }

    return isValidFile;
}

From source file:jenkins.plugins.itemstorage.s3.S3DownloadAllCallable.java

License:Open Source License

/**
 * Download to executor//from  www . jav a  2 s. c  om
 */
@Override
public Integer invoke(TransferManager transferManager, File base, VirtualChannel channel)
        throws IOException, InterruptedException {
    if (!base.exists()) {
        if (!base.mkdirs()) {
            throw new IOException("Failed to create directory : " + base);
        }
    }

    int totalCount;
    Downloads downloads = new Downloads();
    ObjectListing objectListing = null;

    do {
        objectListing = transferManager.getAmazonS3Client()
                .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(pathPrefix)
                        .withMarker(objectListing != null ? objectListing.getNextMarker() : null));

        for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
            downloads.startDownload(transferManager, base, pathPrefix, summary);
        }

    } while (objectListing.getNextMarker() != null);

    // Grab # of files copied
    totalCount = downloads.count();

    // Finish the asynchronous downloading process
    downloads.finishDownloading();

    return totalCount;
}

From source file:jenkins.plugins.itemstorage.s3.S3UploadAllCallable.java

License:Open Source License

/**
 * Upload from slave/*from   w  w  w  . j  a v  a 2  s.co m*/
 */
@Override
public Integer invoke(final TransferManager transferManager, File base, VirtualChannel channel)
        throws IOException, InterruptedException {
    if (!base.exists())
        return 0;

    final AtomicInteger count = new AtomicInteger(0);
    final Uploads uploads = new Uploads();

    final Map<String, S3ObjectSummary> summaries = lookupExistingCacheEntries(
            transferManager.getAmazonS3Client());

    // Find files to upload that match scan
    scanner.scan(base, new FileVisitor() {
        @Override
        public void visit(File f, String relativePath) throws IOException {
            if (f.isFile()) {
                String key = pathPrefix + "/" + relativePath;

                S3ObjectSummary summary = summaries.get(key);
                if (summary == null || f.lastModified() > summary.getLastModified().getTime()) {
                    final ObjectMetadata metadata = buildMetadata(f);

                    uploads.startUploading(transferManager, f,
                            IOUtils.toBufferedInputStream(FileUtils.openInputStream(f)),
                            new Destination(bucketName, key), metadata);

                    if (uploads.count() > 20) {
                        waitForUploads(count, uploads);
                    }
                }
            }
        }
    });

    // Wait for each file to complete before returning
    waitForUploads(count, uploads);

    return uploads.count();
}