Example usage for com.amazonaws.services.s3.model ListObjectsRequest ListObjectsRequest

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ListObjectsRequest ListObjectsRequest.

Prototype

public ListObjectsRequest(String bucketName, String prefix, String marker, String delimiter, Integer maxKeys) 

Source Link

Document

Constructs a new ListObjectsRequest object and initializes all required and optional object fields.

Usage

From source file:aot.storage.s3.CustomStorage.java

License:Open Source License

@Override
public Iterable<String> find(String prefix, String filter) {
    final String pfx = this.prefix + prefix;
    final Pattern fltr = (filter == null) ? null : Pattern.compile(filter);
    return new Iterable<String>() {
        private final ListObjectsRequest request = new ListObjectsRequest(bucket, pfx, null, null, 65536);

        @Override/*  w w  w .j  a  va 2s. co m*/
        public Iterator<String> iterator() {
            try {
                return new Iterator<String>() {
                    private final Pattern filter = fltr;
                    private ObjectListing listing = s3.listObjects(request);
                    private Iterator<S3ObjectSummary> iterator = listing.getObjectSummaries().iterator();
                    private String key = findNext();

                    private String findNext() {
                        while (iterator.hasNext()) {
                            if (filter == null) {
                                return iterator.next().getKey();
                            } else {
                                String el = iterator.next().getKey();
                                if (filter.matcher(el).matches()) {
                                    return el;
                                }
                            }
                        }
                        while (listing.isTruncated()) {
                            listing = s3.listNextBatchOfObjects(listing);
                            iterator = listing.getObjectSummaries().iterator();
                            while (iterator.hasNext()) {
                                if (filter == null) {
                                    return iterator.next().getKey();
                                } else {
                                    String el = iterator.next().getKey();
                                    if (filter.matcher(el).matches()) {
                                        return el;
                                    }
                                }
                            }
                        }
                        return null;
                    }

                    @Override
                    public boolean hasNext() {
                        return key != null;
                    }

                    @Override
                    public String next() {
                        String v = key;
                        if (v != null) {
                            try {
                                key = findNext();
                                return v;
                            } catch (Exception e) {
                                throw new FindStorageException(CustomStorage.this, e);
                            }
                        } else {
                            throw new NoSuchElementException("Next key is not found");
                        }
                    }

                    @Override
                    public void remove() {
                        throw new UnsupportedOperationException("remove()");
                    }
                };
            } catch (Exception e) {
                throw new FindStorageException(CustomStorage.this, e);
            }
        }
    };
}

---

From source file:aot.storage.s3.CustomStorage.java

License:Open Source License

@Override
public Iterable<String> list(String prefix, String filter) {
    final String pfx = this.prefix + prefix;
    final Pattern fltr = (filter == null) ? null : Pattern.compile(filter);
    return new Iterable<String>() {
        private final ListObjectsRequest request = new ListObjectsRequest(bucket, pfx, null, "/", 65536);

        @Override/* ww  w  .j  a v  a 2s. c  om*/
        public Iterator<String> iterator() {
            try {
                return new Iterator<String>() {
                    private final Pattern filter = fltr;
                    private ObjectListing listing = s3.listObjects(request);
                    private Iterator<String> iterator = listing.getCommonPrefixes().iterator();
                    private String directory = findNext();

                    private String findNext() {
                        while (iterator.hasNext()) {
                            if (filter == null) {
                                return iterator.next();
                            } else {
                                String el = iterator.next();
                                if (filter.matcher(el).matches()) {
                                    return el;
                                }
                            }
                        }
                        while (listing.isTruncated()) {
                            listing = s3.listNextBatchOfObjects(listing);
                            iterator = listing.getCommonPrefixes().iterator();
                            while (iterator.hasNext()) {
                                if (filter == null) {
                                    return iterator.next();
                                } else {
                                    String el = iterator.next();
                                    if (filter.matcher(el).matches()) {
                                        return el;
                                    }
                                }
                            }
                        }
                        return null;
                    }

                    @Override
                    public boolean hasNext() {
                        return directory != null;
                    }

                    @Override
                    public String next() {
                        String v = directory;
                        if (v != null) {
                            try {
                                directory = findNext();
                                return v;
                            } catch (Exception e) {
                                throw new ListStorageException(CustomStorage.this, e);
                            }
                        } else {
                            throw new NoSuchElementException("Next directory is not found");
                        }
                    }

                    @Override
                    public void remove() {
                        throw new UnsupportedOperationException("remove()");
                    }
                };
            } catch (Exception e) {
                throw new ListStorageException(CustomStorage.this, e);
            }
        }
    };
}

---

From source file:apphub.storage.s3.CustomStorage.java

License:Open Source License

@Override
public Iterable<String> find(String prefix, String filter) {
    final String pfx = this.prefix + prefix;
    final Pattern fltr = (filter == null) ? null : Pattern.compile(filter);
    return new Iterable<String>() {
        private final ListObjectsRequest request = new ListObjectsRequest(bucket, pfx, null, null, 65536);

        @Override//from w w  w  . j  a v a  2  s .  c  o m
        public Iterator<String> iterator() {
            try {
                return new Iterator<String>() {
                    private final Pattern filter = fltr;
                    private ObjectListing listing = s3.listObjects(request);
                    private Iterator<S3ObjectSummary> iterator = listing.getObjectSummaries().iterator();
                    private String key = findNext();

                    private String findNext() {
                        while (iterator.hasNext()) {
                            if (filter == null) {
                                return iterator.next().getKey();
                            } else {
                                String el = iterator.next().getKey();
                                if (filter.matcher(el).matches()) {
                                    return el;
                                }
                            }
                        }
                        while (listing.isTruncated()) {
                            listing = s3.listNextBatchOfObjects(listing);
                            iterator = listing.getObjectSummaries().iterator();
                            while (iterator.hasNext()) {
                                if (filter == null) {
                                    return iterator.next().getKey();
                                } else {
                                    String el = iterator.next().getKey();
                                    if (filter.matcher(el).matches()) {
                                        return el;
                                    }
                                }
                            }
                        }
                        return null;
                    }

                    @Override
                    public boolean hasNext() {
                        return key != null;
                    }

                    @Override
                    public String next() {
                        String v = key;
                        if (v != null) {
                            try {
                                key = findNext();
                                return v;
                            } catch (Exception e) {
                                throw new FindStorageException(CustomStorage.this.getUrl(), e);
                            }
                        } else {
                            throw new NoSuchElementException("Next key is not found");
                        }
                    }

                    @Override
                    public void remove() {
                        throw new UnsupportedOperationException("remove()");
                    }
                };
            } catch (Exception e) {
                throw new FindStorageException(CustomStorage.this.getUrl(), e);
            }
        }
    };
}

---

From source file:apphub.storage.s3.CustomStorage.java

License:Open Source License

@Override
public Iterable<String> list(String prefix, String filter) {
    final String pfx = this.prefix + prefix;
    final Pattern fltr = (filter == null) ? null : Pattern.compile(filter);
    return new Iterable<String>() {
        private final ListObjectsRequest request = new ListObjectsRequest(bucket, pfx, null, "/", 65536);

        @Override//from   w ww .  j a  v  a2  s  .c  o m
        public Iterator<String> iterator() {
            try {
                return new Iterator<String>() {
                    private final Pattern filter = fltr;
                    private ObjectListing listing = s3.listObjects(request);
                    private Iterator<String> iterator = listing.getCommonPrefixes().iterator();
                    private String directory = findNext();

                    private String findNext() {
                        while (iterator.hasNext()) {
                            if (filter == null) {
                                return iterator.next();
                            } else {
                                String el = iterator.next();
                                if (filter.matcher(el).matches()) {
                                    return el;
                                }
                            }
                        }
                        while (listing.isTruncated()) {
                            listing = s3.listNextBatchOfObjects(listing);
                            iterator = listing.getCommonPrefixes().iterator();
                            while (iterator.hasNext()) {
                                if (filter == null) {
                                    return iterator.next();
                                } else {
                                    String el = iterator.next();
                                    if (filter.matcher(el).matches()) {
                                        return el;
                                    }
                                }
                            }
                        }
                        return null;
                    }

                    @Override
                    public boolean hasNext() {
                        return directory != null;
                    }

                    @Override
                    public String next() {
                        String v = directory;
                        if (v != null) {
                            try {
                                directory = findNext();
                                return v;
                            } catch (Exception e) {
                                throw new ListStorageException(CustomStorage.this.getUrl(), e);
                            }
                        } else {
                            throw new NoSuchElementException("Next directory is not found");
                        }
                    }

                    @Override
                    public void remove() {
                        throw new UnsupportedOperationException("remove()");
                    }
                };
            } catch (Exception e) {
                throw new ListStorageException(CustomStorage.this.getUrl(), e);
            }
        }
    };
}

---

From source file:com.blacklocus.qs.aws.s3.AmazonS3QueueItemProvider.java

License:Apache License

@Override
public Collection<S3ObjectSummary> next() {
    if (page == null) {
        page = s3.listObjects(new ListObjectsRequest(bucket, prefix, null, delimiter, 1000));
    } else {/*www .j av a 2s  .  c om*/
        assert page.isTruncated();
        page = s3.listNextBatchOfObjects(page);
    }
    return page.getObjectSummaries();
}

---

From source file:com.blacklocus.qs.worker.aws.AmazonS3TaskService.java

License:Apache License

public AmazonS3TaskService(String taskHandlerIdentifier, String bucket, String prefix, String delimiter,
        AmazonS3 s3) {//from  ww w  . ja  v a 2s.c om
    this.taskHandlerIdentifier = taskHandlerIdentifier;
    this.bucket = bucket;
    this.prefix = prefix;
    this.delimiter = delimiter;
    this.s3 = s3;

    this.objectListing = s3.listObjects(new ListObjectsRequest(bucket, prefix, null, delimiter, 1000));
    this.listingBatchId = IdSupplier.newId();
    this.iterator = objectListing.getObjectSummaries().iterator();

}

---

From source file:com.blacklocus.qs.worker.aws.AmazonS3TaskService.java

License:Apache License

private void pageForward() {
    while (!iterator.hasNext()) {
        if (objectListing.isTruncated()) {
            // next page of the current listing
            objectListing = s3.listNextBatchOfObjects(objectListing);

        } else {//from ww  w  .j  a  v a 2  s. c  om
            // reset back to the beginning after quiet period
            try {
                Thread.sleep(RESTART_DELAY_MS);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            objectListing = s3.listObjects(new ListObjectsRequest(bucket, prefix, null, delimiter, 1000));
        }

        iterator = objectListing.getObjectSummaries().iterator();
    }
}

---

From source file:com.netflix.spinnaker.front50.model.S3StorageService.java

License:Apache License

@Override
public <T extends Timestamped> Collection<T> loadObjectsWithPrefix(ObjectType objectType, String prefix,
        int maxResults) {
    ObjectListing bucketListing = amazonS3.listObjects(new ListObjectsRequest(bucket,
            (buildTypedFolder(rootFolder, objectType.group) + "/" + prefix).toLowerCase(), null, null,
            maxResults));/*from w  ww  .  j  a va2s  .  c  om*/
    List<S3ObjectSummary> summaries = bucketListing.getObjectSummaries();

    // TODO-AJ this is naive and inefficient
    return summaries.stream().map(
            s3ObjectSummary -> amazonS3.getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()))
            .map(s3Object -> {
                T item = null;
                try {
                    item = deserialize(s3Object, (Class<T>) objectType.clazz);
                    item.setLastModified(s3Object.getObjectMetadata().getLastModified().getTime());
                } catch (IOException e) {
                    // do nothing
                }
                return item;
            }).collect(Collectors.toSet());
}

---

From source file:com.netflix.spinnaker.front50.model.S3StorageService.java

License:Apache License

@Override
public Map<String, Long> listObjectKeys(ObjectType objectType) {
    ObjectListing bucketListing = amazonS3.listObjects(
            new ListObjectsRequest(bucket, buildTypedFolder(rootFolder, objectType.group), null, null, 10000));
    List<S3ObjectSummary> summaries = bucketListing.getObjectSummaries();

    while (bucketListing.isTruncated()) {
        bucketListing = amazonS3.listNextBatchOfObjects(bucketListing);
        summaries.addAll(bucketListing.getObjectSummaries());
    }/* www. j a  va2 s .  c  o m*/

    return summaries.stream().filter(s -> filterS3ObjectSummary(s, objectType.defaultMetadataFilename))
            .collect(Collectors.toMap((s -> buildObjectKey(objectType, s.getKey())),
                    (s -> s.getLastModified().getTime())));
}

---

From source file:com.netflix.spinnaker.front50.model.S3Support.java

License:Apache License

/**
 * Fetch any previously cached applications that have been updated since last retrieved.
 *
 * @param existingItems Previously cached applications
 * @return Refreshed applications//ww  w  .j  a v  a  2 s .  c o  m
 */
protected Set<T> fetchAllItems(Set<T> existingItems) {
    if (existingItems == null) {
        existingItems = new HashSet<>();
    }

    Long refreshTime = System.currentTimeMillis();

    ObjectListing bucketListing = amazonS3
            .listObjects(new ListObjectsRequest(bucket, rootFolder, null, null, 10000));
    List<S3ObjectSummary> summaries = bucketListing.getObjectSummaries();

    while (bucketListing.isTruncated()) {
        bucketListing = amazonS3.listNextBatchOfObjects(bucketListing);
        summaries.addAll(bucketListing.getObjectSummaries());
    }

    Map<String, S3ObjectSummary> summariesByName = summaries.stream().filter(this::filterS3ObjectSummary)
            .collect(Collectors.toMap(S3ObjectSummary::getKey, Function.identity()));

    Map<String, T> existingItemsByName = existingItems.stream()
            .filter(a -> summariesByName.containsKey(buildS3Key(a)))
            .collect(Collectors.toMap(Timestamped::getId, Function.identity()));

    summaries = summariesByName.values().stream().filter(s3ObjectSummary -> {
        String itemName = extractItemName(s3ObjectSummary);
        T existingItem = existingItemsByName.get(itemName);

        return existingItem == null || existingItem.getLastModified() == null
                || s3ObjectSummary.getLastModified().after(new Date(existingItem.getLastModified()));
    }).collect(Collectors.toList());

    Observable.from(summaries).buffer(10).flatMap(ids -> Observable.from(ids).flatMap(s3ObjectSummary -> {
        try {
            return Observable
                    .just(amazonS3.getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()));
        } catch (AmazonS3Exception e) {
            if (e.getStatusCode() == 404) {
                // an item has been removed between the time that object summaries were fetched and now
                existingItemsByName.remove(extractItemName(s3ObjectSummary));
                return Observable.empty();
            }

            throw e;
        }
    }).subscribeOn(scheduler)).map(s3Object -> {
        try {
            T item = deserialize(s3Object);
            item.setLastModified(s3Object.getObjectMetadata().getLastModified().getTime());
            return item;
        } catch (IOException e) {
            throw new IllegalStateException(e);
        }
    }).subscribeOn(scheduler).toList().toBlocking().single().forEach(item -> {
        existingItemsByName.put(item.getId().toLowerCase(), item);
    });

    existingItems = existingItemsByName.values().stream().collect(Collectors.toSet());
    this.lastRefreshedTime = refreshTime;
    return existingItems;
}

---

• «
• 1
• 2
• 3
• 4
• »