Example usage for com.amazonaws.services.s3.model S3ObjectSummary getETag

List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getETag

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. S3ObjectSummary
  6. getETag

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary getETag.

Prototype

public String getETag()

Source Link

Document

Gets the hex encoded 128-bit MD5 hash of this object's contents as computed by Amazon S3.

Usage

From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java

License:Open Source License

@Override
public ListBucketResponseType listBucket(ListBucketType request) throws S3Exception {
    ListBucketResponseType reply = request.getReply();
    User requestUser = getRequestUser(request);
    OsgInternalS3Client internalS3Client = null;
    try {//w  w  w  .  j av  a2s . co m
        internalS3Client = getS3Client(requestUser);
        AmazonS3Client s3Client = internalS3Client.getS3Client();
        ListObjectsRequest listRequest = new ListObjectsRequest();
        listRequest.setBucketName(request.getBucket());
        listRequest.setDelimiter(Strings.isNullOrEmpty(request.getDelimiter()) ? null : request.getDelimiter());
        listRequest.setMarker(Strings.isNullOrEmpty(request.getMarker()) ? null : request.getMarker());
        listRequest.setMaxKeys((request.getMaxKeys() == null ? null : Integer.parseInt(request.getMaxKeys())));
        listRequest.setPrefix(Strings.isNullOrEmpty(request.getPrefix()) ? null : request.getPrefix());

        ObjectListing response = s3Client.listObjects(listRequest);

        /* Non-optional, must have non-null values */
        reply.setName(request.getBucket());
        reply.setMaxKeys(response.getMaxKeys());
        reply.setMarker(response.getMarker() == null ? "" : response.getMarker());
        reply.setPrefix(response.getPrefix() == null ? "" : response.getPrefix());
        reply.setIsTruncated(response.isTruncated());

        /* Optional */
        reply.setNextMarker(response.getNextMarker());
        reply.setDelimiter(response.getDelimiter());
        if (reply.getContents() == null) {
            reply.setContents(new ArrayList<ListEntry>());
        }
        if (reply.getCommonPrefixesList() == null) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());
        }

        for (S3ObjectSummary obj : response.getObjectSummaries()) {
            //Add entry, note that the canonical user is set based on requesting user, not returned user
            reply.getContents()
                    .add(new ListEntry(obj.getKey(),
                            DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(),
                            obj.getSize(), getCanonicalUser(requestUser), obj.getStorageClass()));
        }

        if (response.getCommonPrefixes() != null && response.getCommonPrefixes().size() > 0) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());

            for (String s : response.getCommonPrefixes()) {
                reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s));
            }
        }

        return reply;
    } catch (AmazonServiceException e) {
        LOG.debug("Error from backend", e);
        throw S3ExceptionMapper.fromAWSJavaSDK(e);
    }
}

From source file:com.proofpoint.event.collector.combiner.S3StorageSystem.java

License:Apache License

@Override
public List<StoredObject> listObjects(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String s3Path = getS3ObjectKey(storageArea);
    Iterator<S3ObjectSummary> iter = new S3ObjectListing(s3Service,
            new ListObjectsRequest(getS3Bucket(storageArea), s3Path, null, "/", null)).iterator();

    ImmutableList.Builder<StoredObject> builder = ImmutableList.builder();
    while (iter.hasNext()) {
        S3ObjectSummary summary = iter.next();
        builder.add(new StoredObject(buildS3Location(storageArea, summary.getKey().substring(s3Path.length())),
                summary.getETag(), summary.getSize(), summary.getLastModified().getTime()));
    }//w w w  . jav a  2s  . com
    return builder.build();
}

From source file:core.connector.s3.sync.S3Connector.java

License:GNU General Public License

@Override
public List<FileInfo> listDirectory(String path) throws ConnectorException {
    try {/*from w w w.  ja  va  2s . c o m*/
        ObjectListing bucketListing = s3
                .listObjects(new ListObjectsRequest().withBucketName(info.getBucketName()).withPrefix(path));

        List<FileInfo> listing = new ArrayList<FileInfo>(bucketListing.getObjectSummaries().size());

        boolean finished = false;
        while (!finished) {
            for (S3ObjectSummary s3s : bucketListing.getObjectSummaries()) {
                String key = s3s.getKey();
                long size = s3s.getSize();
                Date date = s3s.getLastModified();

                listing.add(new FileInfo(key, key.substring(path.length() + 1), size, date, s3s.getETag()));
            }
            if (bucketListing.isTruncated())
                bucketListing = s3.listNextBatchOfObjects(bucketListing);
            else
                finished = true;
        }

        return listing;
    } catch (Exception e) {
        throw new ConnectorException(e);
    }
}

From source file:internal.diff.aws.service.AmazonS3DirectoryMetadataServiceImpl.java

License:Apache License

public void populateMetadata(DirectoryMetadata directoryMetadata, String bucketName, String prefix) {

    String marker = null;/*w  ww .j a  va 2  s  .  co  m*/

    do {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName, prefix, marker, "/", null);

        ObjectListing objectListing = this.amazonS3Client.listObjects(listObjectsRequest);

        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {

            if (objectSummary.getKey().equals(prefix)) {
                continue;
            }

            FileMetadata file = new FileMetadata();

            file.setName(objectSummary.getKey().substring(prefix.length()));
            file.setSizeInBytes(objectSummary.getSize());

            if (objectSummary.getETag().contains("-")) {
                file.addChecksum(ChecksumType.S3_MULTIPART_ETAG, objectSummary.getETag());
            } else {
                file.addChecksum(ChecksumType.MD5, objectSummary.getETag());
            }

            directoryMetadata.addFile(file);
        }

        for (String subdirectoryPath : objectListing.getCommonPrefixes()) {

            DirectoryMetadata subdirectory = new DirectoryMetadata();
            subdirectory.setName(subdirectoryPath.substring(prefix.length(), subdirectoryPath.length() - 1));

            directoryMetadata.addSubdirectory(subdirectory);

            populateMetadata(subdirectory, bucketName, subdirectoryPath);
        }

        marker = objectListing.getNextMarker();
    } while (marker != null);
}

From source file:io.jeffrey.web.assemble.S3PutObjectTarget.java

/**
 * @param bucket the bucket where we intend to upload the files
 * @param s3     the S3 client/*from  w  ww  .j  a  v a 2 s  .com*/
 */
public S3PutObjectTarget(final String bucket, final AmazonS3 s3) {
    this.bucket = bucket;
    this.s3 = s3;
    this.etags = new HashMap<>();
    String marker = "";
    boolean again = true;
    while (again) {
        again = false;
        ListObjectsRequest request = new ListObjectsRequest(bucket, "", marker, null, 1000);
        for (S3ObjectSummary obj : s3.listObjects(request).getObjectSummaries()) {
            marker = obj.getKey();
            if (marker.startsWith("charts/")) {
                s3.deleteObject(bucket, marker);
            }
            System.out.println(marker + "->" + obj.getETag());
            etags.put(obj.getKey(), obj.getETag());
            again = true;
        }
    }
}

From source file:org.alanwilliamson.amazon.s3.List.java

License:Open Source License

public cfData execute(cfSession _session, cfArgStructData argStruct) throws cfmRunTimeException {

    AmazonKey amazonKey = getAmazonKey(_session, argStruct);
    AmazonS3 s3Client = getAmazonS3(amazonKey);

    String bucket = getNamedStringParam(argStruct, "bucket", null);
    String prefix = getNamedStringParam(argStruct, "prefix", "");

    if (bucket == null)
        throwException(_session, "Please specify a bucket");

    try {/*from ww  w .j av  a2 s .c  o m*/
        // Create the results
        cfQueryResultData qD = new cfQueryResultData(new String[] { "key", "size", "modified", "etag" }, null);
        qD.setQuerySource("AmazonS3." + amazonKey.getDataSource());

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
                .withDelimiter("/").withPrefix(prefix);
        ObjectListing objectListing;

        do {
            objectListing = s3Client.listObjects(listObjectsRequest);

            java.util.List<String> prefixes = objectListing.getCommonPrefixes();

            // first add the prefixes
            for (String nextPrefix : prefixes) {
                qD.addRow(1);
                qD.setCurrentRow(qD.getSize());

                qD.setCell(1, new cfStringData(nextPrefix));
                qD.setCell(2, new cfNumberData(0));
                qD.setCell(3, cfNullData.NULL);
                qD.setCell(4, cfNullData.NULL);

            }

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {

                // don't include the prefix being listed
                if (objectSummary.getKey().equals(prefix)) {
                    continue;
                }
                qD.addRow(1);
                qD.setCurrentRow(qD.getSize());

                qD.setCell(1, new cfStringData(objectSummary.getKey()));
                qD.setCell(2, new cfNumberData(objectSummary.getSize()));
                qD.setCell(3, new cfDateData(objectSummary.getLastModified()));
                qD.setCell(4, new cfStringData(objectSummary.getETag()));
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

        return qD;
    } catch (Exception e) {
        throwException(_session, "AmazonS3: " + e.getMessage());
        return cfBooleanData.FALSE;
    }
}

From source file:org.nuxeo.liveconnect.importer.aws.S3Importer.java

License:Apache License

public void importBucket(DocumentModel rootFolder) {
    LiveconnectS3Blobprovider blobprovider = (LiveconnectS3Blobprovider) Framework.getService(BlobManager.class)
            .getBlobProvider(provider);/*  w  w w.  j  a  v a2s  . c om*/

    AmazonS3 s3 = blobprovider.getClient();
    String bucketName = blobprovider.getBucketName();

    final ListObjectsRequest req = new ListObjectsRequest().withBucketName(bucketName);
    ObjectListing result;
    int docsCount = 0;
    do {
        result = s3.listObjects(req);
        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            String name = S3LiveConnectFile.getFilename(objectSummary.getKey());
            if (name == null)
                continue;

            DocumentModel fileDoc = getOrCreateFileDocument(rootFolder, objectSummary.getKey(), name);

            //import object
            LiveConnectFileInfo info = new LiveConnectFileInfo(
                    rootFolder.getCoreSession().getPrincipal().getName(), objectSummary.getKey(),
                    objectSummary.getETag());

            LiveConnectBlobProvider blobProvider = (LiveConnectBlobProvider) Framework
                    .getService(BlobManager.class).getBlobProvider(provider);
            try {
                Blob blob = blobProvider.toBlob(info);
                fileDoc.setPropertyValue("file:content", (Serializable) blob);
                fileDoc.getCoreSession().saveDocument(fileDoc);
            } catch (IOException e) {
                log.warn("Couldn't get Blob with ID " + info.getFileId(), e);
            }
            docsCount++;
            if (docsCount % 10 == 0) {
                rootFolder.getCoreSession().save();
                if (TransactionHelper.isTransactionActive()) {
                    TransactionHelper.commitOrRollbackTransaction();
                    TransactionHelper.startTransaction();
                }
            }
        }
        req.setMarker(result.getNextMarker());
    } while (result.isTruncated());

    rootFolder.getCoreSession().save();
    if (TransactionHelper.isTransactionActive()) {
        TransactionHelper.commitOrRollbackTransaction();
        TransactionHelper.startTransaction();
    }

}

From source file:org.springframework.integration.aws.s3.core.AmazonS3OperationsImpl.java

License:Apache License

public PaginatedObjectsView listObjects(String bucketName, String folder, String nextMarker, int pageSize) {
    if (logger.isDebugEnabled()) {
        logger.debug("Listing objects from bucket " + bucketName + " and folder " + folder);
        logger.debug("Next marker is " + nextMarker + " and pageSize is " + pageSize);
    }//from w  w w . j ava 2 s . c  o  m

    Assert.notNull(StringUtils.hasText(bucketName), "Bucket name should be non null and non empty");
    String prefix = null;
    if (folder != null && !"/".equals(folder)) {
        prefix = folder;
    }
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix(prefix).withMarker(nextMarker);

    if (pageSize > 0) {
        listObjectsRequest.withMaxKeys(pageSize);
    }

    ObjectListing listing = client.listObjects(listObjectsRequest);
    PaginatedObjectsView view = null;
    List<com.amazonaws.services.s3.model.S3ObjectSummary> summaries = listing.getObjectSummaries();
    if (summaries != null && !summaries.isEmpty()) {
        List<S3ObjectSummary> objectSummaries = new ArrayList<S3ObjectSummary>();
        for (final com.amazonaws.services.s3.model.S3ObjectSummary summary : summaries) {
            S3ObjectSummary summ = new S3ObjectSummary() {

                public long getSize() {
                    return summary.getSize();
                }

                public Date getLastModified() {
                    return summary.getLastModified();
                }

                public String getKey() {
                    return summary.getKey();
                }

                public String getETag() {
                    return summary.getETag();
                }

                public String getBucketName() {
                    return summary.getBucketName();
                }
            };
            objectSummaries.add(summ);
        }
        view = new PagninatedObjectsViewImpl(objectSummaries, listing.getNextMarker());
    }
    return view;
}

From source file:org.springframework.integration.aws.s3.core.DefaultAmazonS3Operations.java

License:Apache License

/**
 * The implementation that uses the AWS SDK to list objects from the given bucket
 *
 * @param bucketName The bucket in which we want to list the objects in
 * @param nextMarker The number of objects can be very large and this serves as the marker
 *                 for remembering the last record fetch in the last retrieve operation.
 * @param pageSize The max number of records to be retrieved in one list object operation.
 * @param prefix The prefix for the list operation, this can serve as the folder whose contents
 *              are to be listed.//from   ww  w  .j a  v a  2  s.c om
 */
@Override
protected PaginatedObjectsView doListObjects(String bucketName, String nextMarker, int pageSize,
        String prefix) {

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix(prefix).withMarker(nextMarker);

    if (pageSize > 0) {
        listObjectsRequest.withMaxKeys(pageSize);
    }

    ObjectListing listing = client.listObjects(listObjectsRequest);
    PaginatedObjectsView view = null;
    List<com.amazonaws.services.s3.model.S3ObjectSummary> summaries = listing.getObjectSummaries();
    if (summaries != null && !summaries.isEmpty()) {
        List<S3ObjectSummary> objectSummaries = new ArrayList<S3ObjectSummary>();
        for (final com.amazonaws.services.s3.model.S3ObjectSummary summary : summaries) {
            S3ObjectSummary summ = new S3ObjectSummary() {

                public long getSize() {
                    return summary.getSize();
                }

                public Date getLastModified() {
                    return summary.getLastModified();
                }

                public String getKey() {
                    return summary.getKey();
                }

                public String getETag() {
                    return summary.getETag();
                }

                public String getBucketName() {
                    return summary.getBucketName();
                }
            };
            objectSummaries.add(summ);
        }
        view = new PagninatedObjectsViewImpl(objectSummaries, listing.getNextMarker());
    }
    return view;
}