Example usage for com.amazonaws.services.s3.model S3VersionSummary getETag

List of usage examples for com.amazonaws.services.s3.model S3VersionSummary getETag

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. S3VersionSummary
  6. getETag

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3VersionSummary getETag.

Prototype

public String getETag()

Source Link

Document

Gets the hex encoded 128-bit MD5 hash of this version's contents as computed by Amazon S3.

Usage

From source file:com.emc.ecs.sync.util.AwsS3Util.java

License:Open Source License

public static ListIterator<S3ObjectVersion> listVersions(SyncPlugin parentPlugin, AmazonS3 s3, String bucket,
        String key, String relativePath) {
    List<S3ObjectVersion> versions = new ArrayList<>();

    VersionListing listing = null;/*  w ww  .  ja  v a 2  s  .com*/
    do {
        if (listing == null)
            listing = s3.listVersions(bucket, key, null, null, "/", null);
        else
            listing = s3.listNextBatchOfVersions(listing);

        for (S3VersionSummary summary : listing.getVersionSummaries()) {

            if (summary.getKey().equals(key)) {
                versions.add(new S3ObjectVersion(parentPlugin, s3, bucket, key, summary.getVersionId(),
                        summary.isLatest(), summary.isDeleteMarker(), summary.getLastModified(),
                        summary.getETag(), relativePath, summary.getSize()));
            }
        }
    } while (listing.isTruncated());

    // sort chronologically
    Collections.sort(versions, new VersionComparator());
    return versions.listIterator();
}

From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java

License:Open Source License

@Override
public ListVersionsResponseType listVersions(ListVersionsType request) throws S3Exception {
    ListVersionsResponseType reply = request.getReply();
    User requestUser = getRequestUser(request);
    OsgInternalS3Client internalS3Client = null;

    try {//from  www . j av a2s. c  o m
        internalS3Client = getS3Client(requestUser);
        AmazonS3Client s3Client = internalS3Client.getS3Client();
        ListVersionsRequest listVersionsRequest = new ListVersionsRequest(request.getBucket(),
                request.getPrefix(), request.getKeyMarker(), request.getVersionIdMarker(),
                request.getDelimiter(), Integer.parseInt(request.getMaxKeys()));
        VersionListing result = s3Client.listVersions(listVersionsRequest);

        CanonicalUser owner;
        try {
            owner = AclUtils.buildCanonicalUser(requestUser.getAccount());
        } catch (AuthException e) {
            LOG.error("Error getting request user's account during bucket version listing", e);
            owner = null;
            throw new AccountProblemException("Account for user " + requestUser.getUserId());
        }

        //Populate result to euca
        reply.setBucket(request.getBucket());
        reply.setMaxKeys(result.getMaxKeys());
        reply.setDelimiter(result.getDelimiter());
        reply.setNextKeyMarker(result.getNextKeyMarker());
        reply.setNextVersionIdMarker(result.getNextVersionIdMarker());
        reply.setIsTruncated(result.isTruncated());
        reply.setVersionIdMarker(result.getVersionIdMarker());
        reply.setKeyMarker(result.getKeyMarker());

        if (result.getCommonPrefixes() != null && result.getCommonPrefixes().size() > 0) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());

            for (String s : result.getCommonPrefixes()) {
                reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s));
            }
        }

        ArrayList<KeyEntry> versions = new ArrayList<>();
        VersionEntry v;
        DeleteMarkerEntry d;
        for (S3VersionSummary summary : result.getVersionSummaries()) {
            if (!summary.isDeleteMarker()) {
                v = new VersionEntry();
                v.setKey(summary.getKey());
                v.setVersionId(summary.getVersionId());
                v.setLastModified(DateFormatter.dateToHeaderFormattedString(summary.getLastModified()));
                v.setEtag(summary.getETag());
                v.setIsLatest(summary.isLatest());
                v.setOwner(owner);
                v.setSize(summary.getSize());
                versions.add(v);
            } else {
                d = new DeleteMarkerEntry();
                d.setIsLatest(summary.isLatest());
                d.setKey(summary.getKey());
                d.setLastModified(DateFormatter.dateToHeaderFormattedString(summary.getLastModified()));
                d.setOwner(owner);
                d.setVersionId(summary.getVersionId());
                versions.add(d);
            }
        }
        //Again, this is wrong, should be a single listing
        reply.setKeyEntries(versions);
    } catch (AmazonServiceException e) {
        LOG.debug("Error from backend", e);
        throw S3ExceptionMapper.fromAWSJavaSDK(e);
    }

    return reply;
}

From source file:org.apache.nifi.processors.aws.s3.ListS3.java

License:Apache License

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    try {/*from w  w  w  .  j  a va  2 s  . c  o m*/
        restoreState(context);
    } catch (IOException ioe) {
        getLogger().error("Failed to restore processor state; yielding", ioe);
        context.yield();
        return;
    }

    final long startNanos = System.nanoTime();
    final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions().getValue();

    final AmazonS3 client = getClient();
    int listCount = 0;
    long maxTimestamp = 0L;
    String delimiter = context.getProperty(DELIMITER).getValue();
    String prefix = context.getProperty(PREFIX).evaluateAttributeExpressions().getValue();

    boolean useVersions = context.getProperty(USE_VERSIONS).asBoolean();
    int listType = context.getProperty(LIST_TYPE).asInteger();
    S3BucketLister bucketLister = useVersions ? new S3VersionBucketLister(client)
            : listType == 2 ? new S3ObjectBucketListerVersion2(client) : new S3ObjectBucketLister(client);

    bucketLister.setBucketName(bucket);

    if (delimiter != null && !delimiter.isEmpty()) {
        bucketLister.setDelimiter(delimiter);
    }
    if (prefix != null && !prefix.isEmpty()) {
        bucketLister.setPrefix(prefix);
    }

    VersionListing versionListing;
    do {
        versionListing = bucketLister.listVersions();
        for (S3VersionSummary versionSummary : versionListing.getVersionSummaries()) {
            long lastModified = versionSummary.getLastModified().getTime();
            if (lastModified < currentTimestamp
                    || lastModified == currentTimestamp && currentKeys.contains(versionSummary.getKey())) {
                continue;
            }

            // Create the attributes
            final Map<String, String> attributes = new HashMap<>();
            attributes.put(CoreAttributes.FILENAME.key(), versionSummary.getKey());
            attributes.put("s3.bucket", versionSummary.getBucketName());
            if (versionSummary.getOwner() != null) { // We may not have permission to read the owner
                attributes.put("s3.owner", versionSummary.getOwner().getId());
            }
            attributes.put("s3.etag", versionSummary.getETag());
            attributes.put("s3.lastModified", String.valueOf(lastModified));
            attributes.put("s3.length", String.valueOf(versionSummary.getSize()));
            attributes.put("s3.storeClass", versionSummary.getStorageClass());
            attributes.put("s3.isLatest", String.valueOf(versionSummary.isLatest()));
            if (versionSummary.getVersionId() != null) {
                attributes.put("s3.version", versionSummary.getVersionId());
            }

            // Create the flowfile
            FlowFile flowFile = session.create();
            flowFile = session.putAllAttributes(flowFile, attributes);
            session.transfer(flowFile, REL_SUCCESS);

            // Update state
            if (lastModified > maxTimestamp) {
                maxTimestamp = lastModified;
                currentKeys.clear();
            }
            if (lastModified == maxTimestamp) {
                currentKeys.add(versionSummary.getKey());
            }
            listCount++;
        }
        bucketLister.setNextMarker();

        commit(context, session, listCount);
        listCount = 0;
    } while (bucketLister.isTruncated());
    currentTimestamp = maxTimestamp;

    final long listMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully listed S3 bucket {} in {} millis", new Object[] { bucket, listMillis });

    if (!commit(context, session, listCount)) {
        if (currentTimestamp > 0) {
            persistState(context);
        }
        getLogger().debug("No new objects in S3 bucket {} to list. Yielding.", new Object[] { bucket });
        context.yield();
    }
}