Example usage for com.amazonaws.services.s3.model VersionListing isTruncated

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model VersionListing isTruncated.

Prototype

boolean isTruncated

To view the source code for com.amazonaws.services.s3.model VersionListing isTruncated.

Click Source Link

Document

Indicates if this is a complete listing, or if the caller needs to make additional requests to Amazon S3 to see the full object listing for an S3 bucket

Usage

From source file:aws.example.s3.DeleteBucket.java

License:Open Source License

public static void main(String[] args) {
    final String USAGE = "\n" + "To run this example, supply the name of an S3 bucket\n" + "\n"
            + "Ex: DeleteBucket <bucketname>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);/*from w ww . ja  v a 2  s.  co  m*/
    }

    String bucket_name = args[0];

    System.out.println("Deleting S3 bucket: " + bucket_name);
    final AmazonS3 s3 = new AmazonS3Client();

    try {
        System.out.println(" - removing objects from bucket");
        ObjectListing object_listing = s3.listObjects(bucket_name);
        while (true) {
            for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) {
                S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
                s3.deleteObject(bucket_name, summary.getKey());
            }

            // more object_listing to retrieve?
            if (object_listing.isTruncated()) {
                object_listing = s3.listNextBatchOfObjects(object_listing);
            } else {
                break;
            }
        }
        ;

        System.out.println(" - removing versions from bucket");
        VersionListing version_listing = s3.listVersions(new ListVersionsRequest().withBucketName(bucket_name));
        while (true) {
            for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext();) {
                S3VersionSummary vs = (S3VersionSummary) iterator.next();
                s3.deleteVersion(bucket_name, vs.getKey(), vs.getVersionId());
            }

            if (version_listing.isTruncated()) {
                version_listing = s3.listNextBatchOfVersions(version_listing);
            } else {
                break;
            }
        }

        System.out.println(" OK, bucket ready to delete!");
        s3.deleteBucket(bucket_name);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    System.out.println("Done!");
}

---

From source file:com.emc.ecs.sync.util.AwsS3Util.java

License:Open Source License

public static ListIterator<S3ObjectVersion> listVersions(SyncPlugin parentPlugin, AmazonS3 s3, String bucket,
        String key, String relativePath) {
    List<S3ObjectVersion> versions = new ArrayList<>();

    VersionListing listing = null;
    do {/*from w w w  . j av a  2s  .  c o  m*/
        if (listing == null)
            listing = s3.listVersions(bucket, key, null, null, "/", null);
        else
            listing = s3.listNextBatchOfVersions(listing);

        for (S3VersionSummary summary : listing.getVersionSummaries()) {

            if (summary.getKey().equals(key)) {
                versions.add(new S3ObjectVersion(parentPlugin, s3, bucket, key, summary.getVersionId(),
                        summary.isLatest(), summary.isDeleteMarker(), summary.getLastModified(),
                        summary.getETag(), relativePath, summary.getSize()));
            }
        }
    } while (listing.isTruncated());

    // sort chronologically
    Collections.sort(versions, new VersionComparator());
    return versions.listIterator();
}

---

From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java

License:Open Source License

@Override
public ListVersionsResponseType listVersions(ListVersionsType request) throws S3Exception {
    ListVersionsResponseType reply = request.getReply();
    User requestUser = getRequestUser(request);
    OsgInternalS3Client internalS3Client = null;

    try {/*from w w  w  .  ja va  2s .  co  m*/
        internalS3Client = getS3Client(requestUser);
        AmazonS3Client s3Client = internalS3Client.getS3Client();
        ListVersionsRequest listVersionsRequest = new ListVersionsRequest(request.getBucket(),
                request.getPrefix(), request.getKeyMarker(), request.getVersionIdMarker(),
                request.getDelimiter(), Integer.parseInt(request.getMaxKeys()));
        VersionListing result = s3Client.listVersions(listVersionsRequest);

        CanonicalUser owner;
        try {
            owner = AclUtils.buildCanonicalUser(requestUser.getAccount());
        } catch (AuthException e) {
            LOG.error("Error getting request user's account during bucket version listing", e);
            owner = null;
            throw new AccountProblemException("Account for user " + requestUser.getUserId());
        }

        //Populate result to euca
        reply.setBucket(request.getBucket());
        reply.setMaxKeys(result.getMaxKeys());
        reply.setDelimiter(result.getDelimiter());
        reply.setNextKeyMarker(result.getNextKeyMarker());
        reply.setNextVersionIdMarker(result.getNextVersionIdMarker());
        reply.setIsTruncated(result.isTruncated());
        reply.setVersionIdMarker(result.getVersionIdMarker());
        reply.setKeyMarker(result.getKeyMarker());

        if (result.getCommonPrefixes() != null && result.getCommonPrefixes().size() > 0) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());

            for (String s : result.getCommonPrefixes()) {
                reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s));
            }
        }

        ArrayList<KeyEntry> versions = new ArrayList<>();
        VersionEntry v;
        DeleteMarkerEntry d;
        for (S3VersionSummary summary : result.getVersionSummaries()) {
            if (!summary.isDeleteMarker()) {
                v = new VersionEntry();
                v.setKey(summary.getKey());
                v.setVersionId(summary.getVersionId());
                v.setLastModified(DateFormatter.dateToHeaderFormattedString(summary.getLastModified()));
                v.setEtag(summary.getETag());
                v.setIsLatest(summary.isLatest());
                v.setOwner(owner);
                v.setSize(summary.getSize());
                versions.add(v);
            } else {
                d = new DeleteMarkerEntry();
                d.setIsLatest(summary.isLatest());
                d.setKey(summary.getKey());
                d.setLastModified(DateFormatter.dateToHeaderFormattedString(summary.getLastModified()));
                d.setOwner(owner);
                d.setVersionId(summary.getVersionId());
                versions.add(d);
            }
        }
        //Again, this is wrong, should be a single listing
        reply.setKeyEntries(versions);
    } catch (AmazonServiceException e) {
        LOG.debug("Error from backend", e);
        throw S3ExceptionMapper.fromAWSJavaSDK(e);
    }

    return reply;
}

---

From source file:com.universal.storage.UniversalS3Storage.java

License:Open Source License

/**
 * This method wipes the root folder of a storage, basically, will remove all files and folder in it.  
 * Be careful with this method because in too many cases this action won't provide a rollback action.
 * /* w w w.  j ava 2s.  co  m*/
 * This method loops over the versions for deletion, the bucket will be empty and without any version of its objects.
 */
public void wipe() throws UniversalIOException {
    ObjectListing object_listing = this.s3client.listObjects(this.settings.getRoot());
    while (true) {
        for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) {
            S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
            this.s3client.deleteObject(this.settings.getRoot(), summary.getKey());
        }

        if (object_listing.isTruncated()) {
            object_listing = this.s3client.listNextBatchOfObjects(object_listing);
        } else {
            break;
        }
    }
    ;

    VersionListing version_listing = this.s3client
            .listVersions(new ListVersionsRequest().withBucketName(this.settings.getRoot()));
    while (true) {
        for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext();) {
            S3VersionSummary vs = (S3VersionSummary) iterator.next();
            this.s3client.deleteVersion(

                    this.settings.getRoot(), vs.getKey(), vs.getVersionId());
        }

        if (version_listing.isTruncated()) {
            version_listing = this.s3client.listNextBatchOfVersions(version_listing);
        } else {
            break;
        }
    }
}

---

From source file:org.finra.herd.dao.impl.S3DaoImpl.java

License:Apache License

@Override
public List<S3VersionSummary> listVersions(final S3FileTransferRequestParamsDto params) {
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()),
            "Listing of S3 versions from root directory is not allowed.");

    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3VersionSummary> s3VersionSummaries = new ArrayList<>();

    try {/* w w w.ja va2  s  .co  m*/
        ListVersionsRequest listVersionsRequest = new ListVersionsRequest()
                .withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        VersionListing versionListing;

        do {
            versionListing = s3Operations.listVersions(listVersionsRequest, s3Client);
            s3VersionSummaries.addAll(versionListing.getVersionSummaries());
            listVersionsRequest.setKeyMarker(versionListing.getNextKeyMarker());
            listVersionsRequest.setVersionIdMarker(versionListing.getNextVersionIdMarker());
        } while (versionListing.isTruncated());
    } catch (AmazonS3Exception amazonS3Exception) {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode())) {
            throw new IllegalArgumentException(
                    "The specified bucket '" + params.getS3BucketName() + "' does not exist.",
                    amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    } catch (AmazonClientException e) {
        throw new IllegalStateException(
                String.format("Failed to list S3 versions with prefix \"%s\" from bucket \"%s\". Reason: %s",
                        params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()),
                e);
    } finally {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }

    return s3VersionSummaries;
}

---

From source file:squash.deployment.lambdas.AngularjsAppCustomResourceLambda.java

License:Apache License

void deleteAngularjsApp(String websiteBucket, LambdaLogger logger) {
    logger.log("Removing AngularjsApp content from website versioned S3 bucket");

    // We need to delete every version of every key
    ListVersionsRequest listVersionsRequest = new ListVersionsRequest().withBucketName(websiteBucket);
    VersionListing versionListing;

    AmazonS3 client = TransferManagerBuilder.defaultTransferManager().getAmazonS3Client();
    do {/*from w  ww. j a  v  a  2s .  c  o m*/
        versionListing = client.listVersions(listVersionsRequest);
        versionListing.getVersionSummaries().stream().filter(k -> (k.getKey().startsWith("app"))).forEach(k -> {
            logger.log("About to delete version: " + k.getVersionId() + " of AngularjsApp page: " + k.getKey());
            DeleteVersionRequest deleteVersionRequest = new DeleteVersionRequest(websiteBucket, k.getKey(),
                    k.getVersionId());
            client.deleteVersion(deleteVersionRequest);
            logger.log("Successfully deleted version: " + k.getVersionId() + " of AngularjsApp page: "
                    + k.getKey());
        });

        listVersionsRequest.setKeyMarker(versionListing.getNextKeyMarker());
    } while (versionListing.isTruncated());
    logger.log("Finished removing AngularjsApp content from website S3 bucket");
}

---

From source file:squash.deployment.lambdas.ApiGatewayCustomResourceLambda.java

License:Apache License

void removeSdkFromS3(LambdaLogger logger) {
    logger.log("About to remove apigateway sdk from website versioned S3 bucket");
    // We need to delete every version of every key
    ListVersionsRequest listVersionsRequest = new ListVersionsRequest().withBucketName(squashWebsiteBucket);
    VersionListing versionListing;
    IS3TransferManager transferManager = getS3TransferManager();
    AmazonS3 client = transferManager.getAmazonS3Client();
    do {//from w  w  w  .  j  a  v  a2 s.  c om
        versionListing = client.listVersions(listVersionsRequest);
        versionListing
                .getVersionSummaries().stream().filter(k -> !(k.getKey().startsWith("20")
                        || k.getKey().equals("today.html") || k.getKey().equals("bookings.html")))
                .forEach(k -> {
                    logger.log("About to delete version: " + k.getVersionId() + " of API SDK: " + k.getKey());
                    DeleteVersionRequest deleteVersionRequest = new DeleteVersionRequest(squashWebsiteBucket,
                            k.getKey(), k.getVersionId());
                    client.deleteVersion(deleteVersionRequest);
                    logger.log("Successfully deleted version: " + k.getVersionId() + " of API SDK key: "
                            + k.getKey());
                });

        listVersionsRequest.setKeyMarker(versionListing.getNextKeyMarker());
    } while (versionListing.isTruncated());

    logger.log("Finished remove apigateway sdk from website S3 bucket");
}

---

From source file:squash.deployment.lambdas.NoScriptAppCustomResourceLambda.java

License:Apache License

/**
 * Implementation for the AWS Lambda function backing the NoScriptApp custom resource.
 * /*from w w  w. j  av a  2s.c  o m*/
 * <p>This lambda requires the following environment variables:
 * <ul>
 *    <li>SimpleDBDomainName - name of the simpleDB domain for bookings and booking rules.</li>
 *    <li>WebsiteBucket - name of S3 bucket serving the booking website.</li>
 *    <li>ApiGatewayBaseUrl - base Url of the ApiGateway Api.</li>
 *    <li>Region - the AWS region in which the Cloudformation stack is created.</li>
 *    <li>Revision - integer incremented to force stack updates to update this resource.</li>
 * </ul>
 * 
 * <p>On success, it returns the following output to Cloudformation:
 * <ul>
 *    <li>WebsiteURL - Url of the website's first booking page.</li>
 * </ul>
 *
 * @param request request parameters as provided by the CloudFormation service
 * @param context context as provided by the CloudFormation service
 */
@Override
public Object handleRequest(Map<String, Object> request, Context context) {

    LambdaLogger logger = context.getLogger();
    logger.log("Starting NoScriptApp custom resource handleRequest");

    // Handle standard request parameters
    Map<String, String> standardRequestParameters = LambdaInputLogger.logStandardRequestParameters(request,
            logger);
    String requestType = standardRequestParameters.get("RequestType");

    // Handle required environment variables
    logger.log("Logging environment variables required by custom resource request");

    String simpleDBDomainName = System.getenv("SimpleDBDomainName");
    String websiteBucket = System.getenv("WebsiteBucket");
    String apiGatewayBaseUrl = System.getenv("ApiGatewayBaseUrl");
    String region = System.getenv("AWS_REGION");
    String revision = System.getenv("Revision");

    // Log out our required environment variables
    logger.log("SimpleDBDomainName: " + simpleDBDomainName);
    logger.log("WebsiteBucket: " + websiteBucket);
    logger.log("ApiGatewayBaseUrl: " + apiGatewayBaseUrl);
    logger.log("Region: " + region);
    logger.log("Revision: " + revision);

    // API calls below can sometimes give access denied errors during stack
    // creation which I think is bc required new roles have not yet propagated
    // across AWS. We sleep here to allow time for this propagation.
    try {
        Thread.sleep(10000);
    } catch (InterruptedException e) {
        logger.log("Sleep to allow new roles to propagate has been interrupted.");
    }

    // Prepare our response to be sent in the finally block
    CloudFormationResponder cloudFormationResponder = new CloudFormationResponder(standardRequestParameters,
            "DummyPhysicalResourceId");
    // Initialise failure response, which will be changed on success
    String responseStatus = "FAILED";

    String websiteURL = null;
    try {
        cloudFormationResponder.initialise();

        if (requestType.equals("Create") || requestType.equals("Update")) {

            // Upload 21 initial bookings pages and index page to the S3 bucket
            UpdateBookingsLambdaRequest updateBookingsLambdaRequest = new UpdateBookingsLambdaRequest();
            UpdateBookingsLambda updateBookingsLambda = new UpdateBookingsLambda();
            UpdateBookingsLambdaResponse updateBookingsLambdaResponse = updateBookingsLambda
                    .updateBookings(updateBookingsLambdaRequest, context);
            String firstDate = updateBookingsLambdaResponse.getCurrentDate();

            websiteURL = "http://" + websiteBucket + ".s3-website-" + region + ".amazonaws.com?selectedDate="
                    + firstDate + ".html";

        } else if (requestType.equals("Delete")) {
            logger.log("Delete request - so removing bookings pages from website versioned S3 bucket");

            // We need to delete every version of every key before the bucket itself
            // can be deleted
            ListVersionsRequest listVersionsRequest = new ListVersionsRequest().withBucketName(websiteBucket);
            VersionListing versionListing;

            AmazonS3 client = TransferManagerBuilder.defaultTransferManager().getAmazonS3Client();
            do {
                versionListing = client.listVersions(listVersionsRequest);
                versionListing.getVersionSummaries().stream().filter(
                        // Maybe a bit slack, but '20' is to include e.g. 2015-10-04.html
                        k -> (k.getKey().startsWith("20") || k.getKey().equals("today.html"))).forEach(k -> {
                            logger.log("About to delete version: " + k.getVersionId() + " of booking page: "
                                    + k.getKey());
                            DeleteVersionRequest deleteVersionRequest = new DeleteVersionRequest(websiteBucket,
                                    k.getKey(), k.getVersionId());
                            client.deleteVersion(deleteVersionRequest);
                            logger.log("Successfully deleted version: " + k.getVersionId()
                                    + " of booking page: " + k.getKey());
                        });

                listVersionsRequest.setKeyMarker(versionListing.getNextKeyMarker());
            } while (versionListing.isTruncated());
            logger.log("Finished removing bookings pages from website S3 bucket");
        }

        responseStatus = "SUCCESS";
        return null;
    } catch (AmazonServiceException ase) {
        ExceptionUtils.logAmazonServiceException(ase, logger);
        return null;
    } catch (AmazonClientException ace) {
        ExceptionUtils.logAmazonClientException(ace, logger);
        return null;
    } catch (Exception e) {
        logger.log("Exception caught in NoScriptApp Lambda: " + e.getMessage());
        return null;
    } finally {
        // Send response to CloudFormation
        cloudFormationResponder.addKeyValueOutputsPair("WebsiteURL", websiteURL);
        cloudFormationResponder.sendResponse(responseStatus, logger);
    }
}

---