Example usage for com.amazonaws.services.s3.model VersionListing getVersionSummaries

List of usage examples for com.amazonaws.services.s3.model VersionListing getVersionSummaries

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. VersionListing
  6. getVersionSummaries

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model VersionListing getVersionSummaries.

Prototype

public List<S3VersionSummary> getVersionSummaries()

Source Link

Document

Gets the list of version summaries describing the versions stored in the associated S3 bucket.

Usage

From source file:aws.example.s3.DeleteBucket.java

License:Open Source License

public static void main(String[] args) {
    final String USAGE = "\n" + "To run this example, supply the name of an S3 bucket\n" + "\n"
            + "Ex: DeleteBucket <bucketname>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);/*  w w w. ja va  2s . co  m*/
    }

    String bucket_name = args[0];

    System.out.println("Deleting S3 bucket: " + bucket_name);
    final AmazonS3 s3 = new AmazonS3Client();

    try {
        System.out.println(" - removing objects from bucket");
        ObjectListing object_listing = s3.listObjects(bucket_name);
        while (true) {
            for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) {
                S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
                s3.deleteObject(bucket_name, summary.getKey());
            }

            // more object_listing to retrieve?
            if (object_listing.isTruncated()) {
                object_listing = s3.listNextBatchOfObjects(object_listing);
            } else {
                break;
            }
        }
        ;

        System.out.println(" - removing versions from bucket");
        VersionListing version_listing = s3.listVersions(new ListVersionsRequest().withBucketName(bucket_name));
        while (true) {
            for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext();) {
                S3VersionSummary vs = (S3VersionSummary) iterator.next();
                s3.deleteVersion(bucket_name, vs.getKey(), vs.getVersionId());
            }

            if (version_listing.isTruncated()) {
                version_listing = s3.listNextBatchOfVersions(version_listing);
            } else {
                break;
            }
        }

        System.out.println(" OK, bucket ready to delete!");
        s3.deleteBucket(bucket_name);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    System.out.println("Done!");
}

From source file:cloudExplorer.Versioning.java

License:Open Source License

void getVersions(String key, String access_key, String secret_key, String bucket, String endpoint) {
    String results = null;//from w w  w  .ja  va  2 s .c o  m
    Boolean check_finished = false;

    try {
        mainFrame.jTextArea1.append("\nPlease wait, loading versions.");
        mainFrame.calibrateTextArea();
        AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
        AmazonS3 s3Client = new AmazonS3Client(credentials,
                new ClientConfiguration().withSignerOverride("S3SignerType"));
        s3Client.setEndpoint(endpoint);

        VersionListing vListing;
        if (key == null) {
            vListing = s3Client.listVersions(bucket, null);
        } else {
            vListing = s3Client.listVersions(bucket, key);
        }

        List<S3VersionSummary> summary = vListing.getVersionSummaries();

        if (summary.size() == 0) {
            check_finished = true;
        } else {
            check_finished = false;
        }

        for (S3VersionSummary object : summary) {
            if (!Versioning.delete) {
                mainFrame.versioning_date.add(object.getLastModified().toString());
            }
            mainFrame.versioning_id.add(object.getVersionId());
            mainFrame.versioning_name.add(object.getKey());
            System.gc();
        }

        if (Versioning.delete) {
            int i = 0;
            for (String delVer : mainFrame.versioning_name) {
                del = new Delete(delVer, mainFrame.cred.getAccess_key(), mainFrame.cred.getSecret_key(),
                        mainFrame.cred.getBucket(), mainFrame.cred.getEndpoint(),
                        mainFrame.versioning_id.get(i));
                del.startc(delVer, mainFrame.cred.getAccess_key(), mainFrame.cred.getSecret_key(),
                        mainFrame.cred.getBucket(), mainFrame.cred.getEndpoint(),
                        mainFrame.versioning_id.get(i));
                i++;
            }
            if (!check_finished) {
                getVersions(key, access_key, secret_key, bucket, endpoint);
            }
        }

        if (Versioning.delete) {
            mainFrame.jTextArea1.append(
                    "\nCompleted deleting every object. Please observe this window for any tasks that are still running.");
        } else {
            mainFrame.jTextArea1
                    .append("\nDone gathering Versions. If files are found, the first 1000 will be displayed.");
        }
        mainFrame.calibrateTextArea();
    } catch (Exception getVersions) {

    }
    if (Versioning.delete) {
        Versioning.delete = false;
        mainFrame.reloadObjects(false);
    }

}

From source file:com.emc.ecs.sync.util.AwsS3Util.java

License:Open Source License

public static ListIterator<S3ObjectVersion> listVersions(SyncPlugin parentPlugin, AmazonS3 s3, String bucket,
        String key, String relativePath) {
    List<S3ObjectVersion> versions = new ArrayList<>();

    VersionListing listing = null;
    do {/* ww  w  . jav  a 2 s . com*/
        if (listing == null)
            listing = s3.listVersions(bucket, key, null, null, "/", null);
        else
            listing = s3.listNextBatchOfVersions(listing);

        for (S3VersionSummary summary : listing.getVersionSummaries()) {

            if (summary.getKey().equals(key)) {
                versions.add(new S3ObjectVersion(parentPlugin, s3, bucket, key, summary.getVersionId(),
                        summary.isLatest(), summary.isDeleteMarker(), summary.getLastModified(),
                        summary.getETag(), relativePath, summary.getSize()));
            }
        }
    } while (listing.isTruncated());

    // sort chronologically
    Collections.sort(versions, new VersionComparator());
    return versions.listIterator();
}

From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java

License:Open Source License

@Override
public ListVersionsResponseType listVersions(ListVersionsType request) throws S3Exception {
    ListVersionsResponseType reply = request.getReply();
    User requestUser = getRequestUser(request);
    OsgInternalS3Client internalS3Client = null;

    try {//  w  w w .  ja v  a  2 s .  c o  m
        internalS3Client = getS3Client(requestUser);
        AmazonS3Client s3Client = internalS3Client.getS3Client();
        ListVersionsRequest listVersionsRequest = new ListVersionsRequest(request.getBucket(),
                request.getPrefix(), request.getKeyMarker(), request.getVersionIdMarker(),
                request.getDelimiter(), Integer.parseInt(request.getMaxKeys()));
        VersionListing result = s3Client.listVersions(listVersionsRequest);

        CanonicalUser owner;
        try {
            owner = AclUtils.buildCanonicalUser(requestUser.getAccount());
        } catch (AuthException e) {
            LOG.error("Error getting request user's account during bucket version listing", e);
            owner = null;
            throw new AccountProblemException("Account for user " + requestUser.getUserId());
        }

        //Populate result to euca
        reply.setBucket(request.getBucket());
        reply.setMaxKeys(result.getMaxKeys());
        reply.setDelimiter(result.getDelimiter());
        reply.setNextKeyMarker(result.getNextKeyMarker());
        reply.setNextVersionIdMarker(result.getNextVersionIdMarker());
        reply.setIsTruncated(result.isTruncated());
        reply.setVersionIdMarker(result.getVersionIdMarker());
        reply.setKeyMarker(result.getKeyMarker());

        if (result.getCommonPrefixes() != null && result.getCommonPrefixes().size() > 0) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());

            for (String s : result.getCommonPrefixes()) {
                reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s));
            }
        }

        ArrayList<KeyEntry> versions = new ArrayList<>();
        VersionEntry v;
        DeleteMarkerEntry d;
        for (S3VersionSummary summary : result.getVersionSummaries()) {
            if (!summary.isDeleteMarker()) {
                v = new VersionEntry();
                v.setKey(summary.getKey());
                v.setVersionId(summary.getVersionId());
                v.setLastModified(DateFormatter.dateToHeaderFormattedString(summary.getLastModified()));
                v.setEtag(summary.getETag());
                v.setIsLatest(summary.isLatest());
                v.setOwner(owner);
                v.setSize(summary.getSize());
                versions.add(v);
            } else {
                d = new DeleteMarkerEntry();
                d.setIsLatest(summary.isLatest());
                d.setKey(summary.getKey());
                d.setLastModified(DateFormatter.dateToHeaderFormattedString(summary.getLastModified()));
                d.setOwner(owner);
                d.setVersionId(summary.getVersionId());
                versions.add(d);
            }
        }
        //Again, this is wrong, should be a single listing
        reply.setKeyEntries(versions);
    } catch (AmazonServiceException e) {
        LOG.debug("Error from backend", e);
        throw S3ExceptionMapper.fromAWSJavaSDK(e);
    }

    return reply;
}

From source file:com.netflix.spinnaker.front50.model.S3StorageService.java

License:Apache License

@Override
public <T extends Timestamped> Collection<T> listObjectVersions(ObjectType objectType, String objectKey,
        int maxResults) throws NotFoundException {
    try {//from  w  w w.  ja  va2  s  .  com
        VersionListing versionListing = amazonS3.listVersions(new ListVersionsRequest(bucket,
                buildS3Key(objectType.group, objectKey, objectType.defaultMetadataFilename), null, null, null,
                maxResults));
        return versionListing.getVersionSummaries().stream().map(s3VersionSummary -> {
            try {
                S3Object s3Object = amazonS3.getObject(new GetObjectRequest(bucket,
                        buildS3Key(objectType.group, objectKey, objectType.defaultMetadataFilename),
                        s3VersionSummary.getVersionId()));
                T item = deserialize(s3Object, (Class<T>) objectType.clazz);
                item.setLastModified(s3Object.getObjectMetadata().getLastModified().getTime());
                return item;
            } catch (IOException e) {
                throw new IllegalStateException(e);
            }
        }).collect(Collectors.toList());
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == 404) {
            throw new NotFoundException(String.format("No item found with id of %s", objectKey.toLowerCase()));
        }

        throw e;
    }
}

From source file:com.netflix.spinnaker.front50.model.S3Support.java

License:Apache License

public Collection<T> allVersionsOf(String id, int limit) throws NotFoundException {
    try {//  w  ww . j  av  a  2  s .  c  o m
        VersionListing versionListing = amazonS3
                .listVersions(new ListVersionsRequest(bucket, buildS3Key(id), null, null, null, limit));
        return versionListing.getVersionSummaries().stream().map(s3VersionSummary -> {
            try {
                S3Object s3Object = amazonS3.getObject(
                        new GetObjectRequest(bucket, buildS3Key(id), s3VersionSummary.getVersionId()));
                T item = deserialize(s3Object);
                item.setLastModified(s3Object.getObjectMetadata().getLastModified().getTime());
                return item;
            } catch (IOException e) {
                throw new IllegalStateException(e);
            }
        }).collect(Collectors.toList());
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == 404) {
            throw new NotFoundException(String.format("No item found with id of %s", id.toLowerCase()));
        }

        throw e;
    }
}

From source file:com.universal.storage.UniversalS3Storage.java

License:Open Source License

/**
 * This method wipes the root folder of a storage, basically, will remove all files and folder in it.  
 * Be careful with this method because in too many cases this action won't provide a rollback action.
 * //from   w  w  w  . j a  v  a 2  s. c  o m
 * This method loops over the versions for deletion, the bucket will be empty and without any version of its objects.
 */
public void wipe() throws UniversalIOException {
    ObjectListing object_listing = this.s3client.listObjects(this.settings.getRoot());
    while (true) {
        for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) {
            S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
            this.s3client.deleteObject(this.settings.getRoot(), summary.getKey());
        }

        if (object_listing.isTruncated()) {
            object_listing = this.s3client.listNextBatchOfObjects(object_listing);
        } else {
            break;
        }
    }
    ;

    VersionListing version_listing = this.s3client
            .listVersions(new ListVersionsRequest().withBucketName(this.settings.getRoot()));
    while (true) {
        for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext();) {
            S3VersionSummary vs = (S3VersionSummary) iterator.next();
            this.s3client.deleteVersion(

                    this.settings.getRoot(), vs.getKey(), vs.getVersionId());
        }

        if (version_listing.isTruncated()) {
            version_listing = this.s3client.listNextBatchOfVersions(version_listing);
        } else {
            break;
        }
    }
}

From source file:org.apache.nifi.processors.aws.s3.ListS3.java

License:Apache License

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    try {/*w  ww .  ja  va 2s. c o m*/
        restoreState(context);
    } catch (IOException ioe) {
        getLogger().error("Failed to restore processor state; yielding", ioe);
        context.yield();
        return;
    }

    final long startNanos = System.nanoTime();
    final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions().getValue();

    final AmazonS3 client = getClient();
    int listCount = 0;
    long maxTimestamp = 0L;
    String delimiter = context.getProperty(DELIMITER).getValue();
    String prefix = context.getProperty(PREFIX).evaluateAttributeExpressions().getValue();

    boolean useVersions = context.getProperty(USE_VERSIONS).asBoolean();
    int listType = context.getProperty(LIST_TYPE).asInteger();
    S3BucketLister bucketLister = useVersions ? new S3VersionBucketLister(client)
            : listType == 2 ? new S3ObjectBucketListerVersion2(client) : new S3ObjectBucketLister(client);

    bucketLister.setBucketName(bucket);

    if (delimiter != null && !delimiter.isEmpty()) {
        bucketLister.setDelimiter(delimiter);
    }
    if (prefix != null && !prefix.isEmpty()) {
        bucketLister.setPrefix(prefix);
    }

    VersionListing versionListing;
    do {
        versionListing = bucketLister.listVersions();
        for (S3VersionSummary versionSummary : versionListing.getVersionSummaries()) {
            long lastModified = versionSummary.getLastModified().getTime();
            if (lastModified < currentTimestamp
                    || lastModified == currentTimestamp && currentKeys.contains(versionSummary.getKey())) {
                continue;
            }

            // Create the attributes
            final Map<String, String> attributes = new HashMap<>();
            attributes.put(CoreAttributes.FILENAME.key(), versionSummary.getKey());
            attributes.put("s3.bucket", versionSummary.getBucketName());
            if (versionSummary.getOwner() != null) { // We may not have permission to read the owner
                attributes.put("s3.owner", versionSummary.getOwner().getId());
            }
            attributes.put("s3.etag", versionSummary.getETag());
            attributes.put("s3.lastModified", String.valueOf(lastModified));
            attributes.put("s3.length", String.valueOf(versionSummary.getSize()));
            attributes.put("s3.storeClass", versionSummary.getStorageClass());
            attributes.put("s3.isLatest", String.valueOf(versionSummary.isLatest()));
            if (versionSummary.getVersionId() != null) {
                attributes.put("s3.version", versionSummary.getVersionId());
            }

            // Create the flowfile
            FlowFile flowFile = session.create();
            flowFile = session.putAllAttributes(flowFile, attributes);
            session.transfer(flowFile, REL_SUCCESS);

            // Update state
            if (lastModified > maxTimestamp) {
                maxTimestamp = lastModified;
                currentKeys.clear();
            }
            if (lastModified == maxTimestamp) {
                currentKeys.add(versionSummary.getKey());
            }
            listCount++;
        }
        bucketLister.setNextMarker();

        commit(context, session, listCount);
        listCount = 0;
    } while (bucketLister.isTruncated());
    currentTimestamp = maxTimestamp;

    final long listMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully listed S3 bucket {} in {} millis", new Object[] { bucket, listMillis });

    if (!commit(context, session, listCount)) {
        if (currentTimestamp > 0) {
            persistState(context);
        }
        getLogger().debug("No new objects in S3 bucket {} to list. Yielding.", new Object[] { bucket });
        context.yield();
    }
}

From source file:org.finra.herd.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * {@inheritDoc}//from w w  w.  j  av a 2 s  . c o  m
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listVersionsRequest, only versions starting with the
 * prefix will be returned.
 */
@Override
public VersionListing listVersions(ListVersionsRequest listVersionsRequest, AmazonS3 s3Client) {
    LOGGER.debug(
            "listVersions(): listVersionsRequest.getBucketName() = " + listVersionsRequest.getBucketName());

    String bucketName = listVersionsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName)) {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(
                MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    } else if (MOCK_S3_BUCKET_NAME_INTERNAL_ERROR.equals(bucketName)) {
        throw new AmazonServiceException(S3Operations.ERROR_CODE_INTERNAL_ERROR);
    }

    VersionListing versionListing = new VersionListing();
    versionListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null) {
        for (MockS3Object mockS3Object : mockS3Bucket.getVersions().values()) {
            String s3ObjectKey = mockS3Object.getKey();
            if (listVersionsRequest.getPrefix() == null
                    || s3ObjectKey.startsWith(listVersionsRequest.getPrefix())) {
                S3VersionSummary s3VersionSummary = new S3VersionSummary();
                s3VersionSummary.setBucketName(bucketName);
                s3VersionSummary.setKey(s3ObjectKey);
                s3VersionSummary.setVersionId(mockS3Object.getVersion());
                s3VersionSummary.setSize(mockS3Object.getData().length);
                s3VersionSummary.setStorageClass(mockS3Object.getObjectMetadata() != null
                        ? mockS3Object.getObjectMetadata().getStorageClass()
                        : null);

                versionListing.getVersionSummaries().add(s3VersionSummary);
            }
        }
    }

    return versionListing;
}

From source file:org.finra.herd.dao.impl.S3DaoImpl.java

License:Apache License

@Override
public List<S3VersionSummary> listVersions(final S3FileTransferRequestParamsDto params) {
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()),
            "Listing of S3 versions from root directory is not allowed.");

    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3VersionSummary> s3VersionSummaries = new ArrayList<>();

    try {//from w  w w . ja  v  a 2  s  . com
        ListVersionsRequest listVersionsRequest = new ListVersionsRequest()
                .withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        VersionListing versionListing;

        do {
            versionListing = s3Operations.listVersions(listVersionsRequest, s3Client);
            s3VersionSummaries.addAll(versionListing.getVersionSummaries());
            listVersionsRequest.setKeyMarker(versionListing.getNextKeyMarker());
            listVersionsRequest.setVersionIdMarker(versionListing.getNextVersionIdMarker());
        } while (versionListing.isTruncated());
    } catch (AmazonS3Exception amazonS3Exception) {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode())) {
            throw new IllegalArgumentException(
                    "The specified bucket '" + params.getS3BucketName() + "' does not exist.",
                    amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    } catch (AmazonClientException e) {
        throw new IllegalStateException(
                String.format("Failed to list S3 versions with prefix \"%s\" from bucket \"%s\". Reason: %s",
                        params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()),
                e);
    } finally {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }

    return s3VersionSummaries;
}