Example usage for com.amazonaws.services.s3.iterable S3Objects withPrefix

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.iterable S3Objects withPrefix.

Prototype

public static S3Objects withPrefix(AmazonS3 s3, String bucketName, String prefix) 

Source Link

Document

Constructs an iterable that covers the objects in an Amazon S3 bucket where the key begins with the given prefix.

Usage

From source file:com.streamsets.pipeline.stage.origin.s3.AmazonS3Util.java

License:Apache License

/**
 * Lists objects from AmazonS3 in chronological order [lexicographical order if 2 files have same timestamp] which are
 * later than or equal to the timestamp of the previous offset object
 *
 * @param s3Client//from   ww w.  j a  v  a  2s  . c om
 * @param s3ConfigBean
 * @param pathMatcher glob patterns to match file name against
 * @param s3Offset current offset which provides the timestamp of the previous object
 * @param fetchSize number of objects to fetch in one go
 * @return
 * @throws AmazonClientException
 */
static List<S3ObjectSummary> listObjectsChronologically(AmazonS3Client s3Client, S3ConfigBean s3ConfigBean,
        PathMatcher pathMatcher, AmazonS3Source.S3Offset s3Offset, int fetchSize) throws AmazonClientException {

    //Algorithm:
    // - Full scan all objects that match the file name pattern and which are later than the file in the offset
    // - Select the oldest "fetchSize" number of files and return them.
    TreeSet<S3ObjectSummary> treeSet = new TreeSet<>(new Comparator<S3ObjectSummary>() {
        @Override
        public int compare(S3ObjectSummary o1, S3ObjectSummary o2) {
            int result = o1.getLastModified().compareTo(o2.getLastModified());
            if (result != 0) {
                //same modified time. Use name to sort
                return result;
            }
            return o1.getKey().compareTo(o2.getKey());
        }
    });

    S3Objects s3ObjectSummaries = S3Objects
            .withPrefix(s3Client, s3ConfigBean.s3Config.bucket, s3ConfigBean.s3Config.folder)
            .withBatchSize(BATCH_SIZE);
    for (S3ObjectSummary s : s3ObjectSummaries) {
        String fileName = s.getKey().substring(s3ConfigBean.s3Config.folder.length(), s.getKey().length());
        if (!fileName.isEmpty()) {
            //fileName can be empty.
            //If the user manually creates a folder "myFolder/mySubFolder" in bucket "myBucket" and uploads "myObject",
            // then the first objects returned here are:
            // myFolder/mySubFolder
            // myFolder/mySubFolder/myObject
            //
            // All is good when pipeline is run but preview returns with no data. So we should ignore the empty file as it
            // has no data
            if (pathMatcher.matches(Paths.get(fileName)) && isEligible(s, s3Offset)) {
                treeSet.add(s);
            }
            if (treeSet.size() > fetchSize) {
                treeSet.pollLast();
            }
        }
    }

    return new ArrayList<>(treeSet);
}

---

From source file:com.streamsets.pipeline.stage.origin.s3.AmazonS3Util.java

License:Apache License

static S3ObjectSummary getObjectSummary(AmazonS3Client s3Client, String bucket, String objectKey) {
    S3ObjectSummary s3ObjectSummary = null;
    S3Objects s3ObjectSummaries = S3Objects.withPrefix(s3Client, bucket, objectKey);
    for (S3ObjectSummary s : s3ObjectSummaries) {
        if (s.getKey().equals(objectKey)) {
            s3ObjectSummary = s;//from w  ww . j  av a  2s .  c  o m
            break;
        }
    }
    return s3ObjectSummary;
}

---

From source file:org.apache.nifi.minifi.c2.cache.s3.S3CacheFileInfoImpl.java

License:Apache License

@Override
public Stream<WriteableConfiguration> getCachedConfigurations() throws IOException {

    Iterable<S3ObjectSummary> objectSummaries = S3Objects.withPrefix(s3, bucket, prefix);
    Stream<S3ObjectSummary> objectStream = StreamSupport.stream(objectSummaries.spliterator(), false);

    return objectStream.map(p -> {
        Integer version = getVersionIfMatch(p.getKey());
        if (version == null) {
            return null;
        }//from www  . j  a  va2  s.c o m
        return new Pair<>(version, p);
    }).filter(Objects::nonNull)
            .sorted(Comparator.comparing(pair -> ((Pair<Integer, S3ObjectSummary>) pair).getFirst()).reversed())
            .map(pair -> new S3WritableConfiguration(s3, pair.getSecond(), Integer.toString(pair.getFirst())));

}

---

From source file:org.geowebcache.s3.S3Ops.java

License:Open Source License

/**
 * Simply checks if there are objects starting with {@code prefix}
 *//*from   ww w  .jav  a2 s.c  om*/
public boolean prefixExists(String prefix) {
    boolean hasNext = S3Objects.withPrefix(conn, bucketName, prefix).withBatchSize(1).iterator().hasNext();
    return hasNext;
}

---

From source file:org.geowebcache.s3.S3Ops.java

License:Open Source License

public Stream<S3ObjectSummary> objectStream(String prefix) {
    return StreamSupport.stream(S3Objects.withPrefix(conn, bucketName, prefix).spliterator(), false);
}

---

From source file:org.geowebcache.s3.TemporaryS3Folder.java

License:Open Source License

public void delete() {
    checkState(isConfigured(), "client not configured.");
    if (temporaryPrefix == null) {
        return;/*from w  w  w .  j a va 2s . co m*/
    }

    Iterable<S3ObjectSummary> objects = S3Objects.withPrefix(s3, bucket, temporaryPrefix);
    Iterable<List<S3ObjectSummary>> partition = Iterables.partition(objects, 1000);
    for (List<S3ObjectSummary> os : partition) {
        List<KeyVersion> keys = Lists.transform(os, new Function<S3ObjectSummary, KeyVersion>() {
            @Override
            public KeyVersion apply(S3ObjectSummary input) {
                KeyVersion k = new KeyVersion(input.getKey());
                return k;
            }
        });
        DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket);
        deleteRequest.setKeys(keys);
        s3.deleteObjects(deleteRequest);
    }
}

---