Example usage for com.amazonaws.services.s3 AmazonS3 listNextBatchOfObjects

List of usage examples for com.amazonaws.services.s3 AmazonS3 listNextBatchOfObjects

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.*
  5. AmazonS3
  6. listNextBatchOfObjects

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 AmazonS3 listNextBatchOfObjects.

Prototype

public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest)
        throws SdkClientException, AmazonServiceException;

Source Link

Document

Provides an easy way to continue a truncated object listing and retrieve the next page of results.

Usage

From source file:S3ClientSideEncryptionWithSymmetricMasterKey.java

License:Apache License

private static void deleteBucketAndAllContents(AmazonS3 client) {
    System.out.println("Deleting S3 bucket: " + bucketName);
    ObjectListing objectListing = client.listObjects(bucketName);

    while (true) {
        for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext();) {
            S3ObjectSummary objectSummary = (S3ObjectSummary) iterator.next();
            client.deleteObject(bucketName, objectSummary.getKey());
        }//from   ww w  . j a va  2s . co m

        if (objectListing.isTruncated()) {
            objectListing = client.listNextBatchOfObjects(objectListing);
        } else {
            break;
        }
    }
    ;
    client.deleteBucket(bucketName);
}

From source file:S3ClientSideEncryptionAsymmetricMasterKey.java

License:Apache License

private static void deleteBucketAndAllContents(AmazonS3 client) {
    System.out.println("Deleting S3 bucket: " + bucketName);
    ObjectListing objectListing = client.listObjects(bucketName);

    while (true) {
        for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext();) {
            S3ObjectSummary objectSummary = (S3ObjectSummary) iterator.next();
            client.deleteObject(bucketName, objectSummary.getKey());
        }//from   w  w w .j a  v  a2 s.c  o  m

        if (objectListing.isTruncated()) {
            objectListing = client.listNextBatchOfObjects(objectListing);
        } else {
            break;
        }
    }
    client.deleteBucket(bucketName);
}

From source file:aws.example.s3.DeleteBucket.java

License:Open Source License

public static void main(String[] args) {
    final String USAGE = "\n" + "To run this example, supply the name of an S3 bucket\n" + "\n"
            + "Ex: DeleteBucket <bucketname>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);//from   w  ww  . j a  va 2  s  . c o  m
    }

    String bucket_name = args[0];

    System.out.println("Deleting S3 bucket: " + bucket_name);
    final AmazonS3 s3 = new AmazonS3Client();

    try {
        System.out.println(" - removing objects from bucket");
        ObjectListing object_listing = s3.listObjects(bucket_name);
        while (true) {
            for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) {
                S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
                s3.deleteObject(bucket_name, summary.getKey());
            }

            // more object_listing to retrieve?
            if (object_listing.isTruncated()) {
                object_listing = s3.listNextBatchOfObjects(object_listing);
            } else {
                break;
            }
        }
        ;

        System.out.println(" - removing versions from bucket");
        VersionListing version_listing = s3.listVersions(new ListVersionsRequest().withBucketName(bucket_name));
        while (true) {
            for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext();) {
                S3VersionSummary vs = (S3VersionSummary) iterator.next();
                s3.deleteVersion(bucket_name, vs.getKey(), vs.getVersionId());
            }

            if (version_listing.isTruncated()) {
                version_listing = s3.listNextBatchOfVersions(version_listing);
            } else {
                break;
            }
        }

        System.out.println(" OK, bucket ready to delete!");
        s3.deleteBucket(bucket_name);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    System.out.println("Done!");
}

From source file:com.conductor.s3.S3InputFormatUtils.java

License:Apache License

/**
 * Efficiently gets the Hadoop {@link org.apache.hadoop.fs.FileStatus} for all S3 files under the provided
 * {@code dirs}//from   ww w  .  j a  va2  s .  c om
 * 
 * @param s3Client
 *            s3 client
 * @param blockSize
 *            the block size
 * @param dirs
 *            the dirs to search through
 * @return the {@link org.apache.hadoop.fs.FileStatus} version of all S3 files under {@code dirs}
 */
static List<FileStatus> getFileStatuses(final AmazonS3 s3Client, final long blockSize, final Path... dirs) {
    final List<FileStatus> result = Lists.newArrayList();
    for (final Path dir : dirs) {
        // get bucket and prefix from path
        final String bucket = S3HadoopUtils.getBucketFromPath(dir.toString());
        final String prefix = S3HadoopUtils.getKeyFromPath(dir.toString());
        // list request
        final ListObjectsRequest req = new ListObjectsRequest().withMaxKeys(Integer.MAX_VALUE)
                .withBucketName(bucket).withPrefix(prefix);
        // recursively page through all objects under the path
        for (ObjectListing listing = s3Client.listObjects(req); listing.getObjectSummaries()
                .size() > 0; listing = s3Client.listNextBatchOfObjects(listing)) {
            for (final S3ObjectSummary summary : listing.getObjectSummaries()) {
                final Path path = new Path(
                        String.format("s3n://%s/%s", summary.getBucketName(), summary.getKey()));
                if (S3_PATH_FILTER.accept(path)) {
                    result.add(new FileStatus(summary.getSize(), false, 1, blockSize,
                            summary.getLastModified().getTime(), path));
                }
            }
            // don't need to check the next listing if this one is not truncated
            if (!listing.isTruncated()) {
                break;
            }
        }
    }
    return result;
}

From source file:com.haskins.cloudtrailviewer.dialog.s3filechooser.S3FileList.java

License:Open Source License

private void addFolderFiles(String path) {

    AmazonS3 s3Client = getS3Client();

    ObjectListing current = s3Client.listObjects(currentAccount.getBucket(), path);
    List<S3ObjectSummary> objectSummaries = current.getObjectSummaries();

    for (final S3ObjectSummary objectSummary : objectSummaries) {
        String file = objectSummary.getKey();
        selected_keys.add(file);/*from  w  ww  .j  a v a2s . co m*/
    }

    while (current.isTruncated()) {

        current = s3Client.listNextBatchOfObjects(current);
        objectSummaries = current.getObjectSummaries();

        for (final S3ObjectSummary objectSummary : objectSummaries) {
            String file = objectSummary.getKey();
            selected_keys.add(file);
        }
    }
}

From source file:com.images3.data.impl.ImageContentAccessImplS3.java

License:Apache License

private void deleteAllImageContent(AmazonS3 client, AmazonS3Bucket bucket, ObjectListing objList) {
    boolean isFinished = false;
    while (!isFinished) {
        DeleteObjectsRequest request = new DeleteObjectsRequest(bucket.getName());
        List<KeyVersion> keys = new ArrayList<KeyVersion>(objList.getMaxKeys());
        for (S3ObjectSummary sum : objList.getObjectSummaries()) {
            keys.add(new KeyVersion(sum.getKey()));
        }//from  w  w w .ja v  a 2s .c om
        request.setKeys(keys);
        client.deleteObjects(request);
        if (objList.isTruncated()) {
            objList = client.listNextBatchOfObjects(objList);
        } else {
            isFinished = true;
        }
    }
}

From source file:com.yahoo.athenz.zts.store.impl.S3ChangeLogStore.java

License:Apache License

/**
 * list the objects in the zts bucket. If the mod time is specified as 0
 * then we want to list all objects otherwise, we only list objects
 * that are newer than the specified timestamp
 * @param s3 AWS S3 client object/* w w  w  .j a va  2 s . c o  m*/
 * @param domains collection to be updated to include domain names
 * @param modTime only include domains newer than this timestamp
 */
void listObjects(AmazonS3 s3, Collection<String> domains, long modTime) {

    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("listObjects: Retrieving domains from {} with mod time > {}", s3BucketName, modTime);
    }

    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(s3BucketName));

    String objectName;
    while (objectListing != null) {

        // process each entry in our result set and add the domain
        // name to our return list

        final List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
        boolean listTruncated = objectListing.isTruncated();

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("listObjects: retrieved {} objects, more objects available - {}",
                    objectSummaries.size(), listTruncated);
        }

        for (S3ObjectSummary objectSummary : objectSummaries) {

            // if mod time is specified then make sure we automatically skip
            // any domains older than the specified value

            if (modTime > 0 && objectSummary.getLastModified().getTime() <= modTime) {
                continue;
            }

            // for now skip any folders/objects that start with '.'

            objectName = objectSummary.getKey();
            if (objectName.charAt(0) == '.') {
                continue;
            }
            domains.add(objectName);
        }

        // check if the object listing is truncated or not (break out in this case)
        // technically we can skip this call and just call listNextBatchOfResults
        // since that returns null if the object listing is not truncated but 
        // this direct check here makes the logic easier to follow

        if (!listTruncated) {
            break;
        }

        objectListing = s3.listNextBatchOfObjects(objectListing);
    }
}

From source file:com.yahoo.athenz.zts.store.s3.S3ChangeLogStore.java

License:Apache License

/**
 * list the objects in the zts bucket. If te mod time is specified as 0
 * then we want to list all objects otherwise, we only list objects
 * that are newer than the specified timestamp
 * @param s3 AWS S3 client object//from ww  w.  j  a  v  a2 s  .  c  om
 * @param domains collection to be updated to include domain names
 * @param modTime only include domains newer than this timestamp
 */
void listObjects(AmazonS3 s3, Collection<String> domains, long modTime) {

    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(s3BucketName));

    String objectName = null;
    while (objectListing != null) {

        // process each entry in our result set and add the domain
        // name to our return list

        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {

            // if mod time is specified then make sure we automatically skip
            // any domains older than the specified value

            if (modTime > 0 && objectSummary.getLastModified().getTime() <= modTime) {
                continue;
            }

            // for now skip any folders/objects that start with '.'

            objectName = objectSummary.getKey();
            if (objectName.charAt(0) == '.') {
                continue;
            }
            domains.add(objectName);
        }

        // check if the object listing is truncated or not (break out in this case)
        // technically we can skip this call and just call listNextBatchOfResults
        // since that returns null if the object listing is not truncated but 
        // this direct check here makes the logic easier to follow

        if (!objectListing.isTruncated()) {
            break;
        }

        objectListing = s3.listNextBatchOfObjects(objectListing);
    }
}

From source file:jenkins.plugins.itemstorage.s3.S3UploadAllCallable.java

License:Open Source License

private Map<String, S3ObjectSummary> lookupExistingCacheEntries(AmazonS3 s3) {
    Map<String, S3ObjectSummary> summaries = new HashMap<>();

    ObjectListing listing = s3.listObjects(bucketName, pathPrefix);
    do {/*from   w  w w  . ja  v a  2s.  com*/
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            summaries.put(summary.getKey(), summary);
        }
        listing = listing.isTruncated() ? s3.listNextBatchOfObjects(listing) : null;
    } while (listing != null);

    return summaries;
}

From source file:jp.classmethod.aws.gradle.s3.DeleteBucketTask.java

License:Apache License

@TaskAction
public void deleteBucket() {
    // to enable conventionMappings feature
    String bucketName = getBucketName();
    boolean ifExists = isIfExists();

    if (bucketName == null) {
        throw new GradleException("bucketName is not specified");
    }//  ww w  . j  a  v  a2 s  . co  m

    AmazonS3PluginExtension ext = getProject().getExtensions().getByType(AmazonS3PluginExtension.class);
    AmazonS3 s3 = ext.getClient();

    if (ifExists == false || exists(s3)) {
        if (deleteObjects) {
            getLogger().info("Delete all S3 objects in bucket [{}]", bucketName);
            ObjectListing objectListing = s3.listObjects(bucketName);
            while (objectListing.getObjectSummaries().isEmpty() == false) {
                objectListing.getObjectSummaries().forEach(summary -> {
                    getLogger().info(" => delete s3://{}/{}", bucketName, summary.getKey());
                    s3.deleteObject(bucketName, summary.getKey());
                });
                objectListing = s3.listNextBatchOfObjects(objectListing);
            }
        }
        s3.deleteBucket(bucketName);
        getLogger().info("S3 bucket {} is deleted", bucketName);
    } else {
        getLogger().debug("S3 bucket {} does not exist", bucketName);
    }
}