Example usage for com.amazonaws.services.s3.model ListObjectsV2Request ListObjectsV2Request

List of usage examples for com.amazonaws.services.s3.model ListObjectsV2Request ListObjectsV2Request

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. ListObjectsV2Request
  6. ListObjectsV2Request

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ListObjectsV2Request ListObjectsV2Request.

Prototype

ListObjectsV2Request

Source Link

Usage

From source file:mx.iteso.desi.cloud.hw3.AWSFaceCompare.java

License:Apache License

public Face compare(ByteBuffer imageBuffer) {
    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(srcBucket).withPrefix("Faces/");
    ListObjectsV2Result result;//from  www .jav  a 2s .  com
    ArrayList<String> s3Files = new ArrayList<>();
    Face empty = new Face("", 0.0f);

    try {
        String sourceFile = "source.jpg";
        Path file = Paths.get(sourceFile);
        Files.write(file, imageBuffer.array(), StandardOpenOption.CREATE);
        upload(sourceFile);
    } catch (IOException ex) {
        Logger.getLogger(FaceAddFrame.class.getName()).log(Level.SEVERE, null, ex);
    }

    do {
        result = s3.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            s3Files.add(objectSummary.getKey());
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated() == true);

    for (String s3File : s3Files) {
        if (s3File.endsWith(".jpg")) {
            System.out.println("Checking " + s3File + "...");
            Face face = compare("Compare/source.jpg", s3File);
            if (face.getCofidence() > SIMILARITY_THRESHOLD) {
                return face;
            }
        }
    }
    return empty;
}

From source file:net.solarnetwork.node.backup.s3.SdkS3Client.java

License:Open Source License

@Override
public Set<S3ObjectReference> listObjects(String prefix) throws IOException {
    AmazonS3 client = getClient();/*w w w. ja va 2 s. c  o  m*/
    Set<S3ObjectReference> result = new LinkedHashSet<>(100);
    try {
        final ListObjectsV2Request req = new ListObjectsV2Request();
        req.setBucketName(bucketName);
        req.setMaxKeys(maximumKeysPerRequest);
        req.setPrefix(prefix);
        ListObjectsV2Result listResult;
        do {
            listResult = client.listObjectsV2(req);

            for (S3ObjectSummary objectSummary : listResult.getObjectSummaries()) {
                result.add(new S3ObjectReference(objectSummary.getKey(), objectSummary.getSize(),
                        objectSummary.getLastModified()));
            }
            req.setContinuationToken(listResult.getNextContinuationToken());
        } while (listResult.isTruncated() == true);

    } catch (AmazonServiceException e) {
        log.warn("AWS error: {}; HTTP code {}; AWS code {}; type {}; request ID {}", e.getMessage(),
                e.getStatusCode(), e.getErrorCode(), e.getErrorType(), e.getRequestId());
        throw new RemoteServiceException("Error listing S3 objects at " + prefix, e);
    } catch (AmazonClientException e) {
        log.debug("Error communicating with AWS: {}", e.getMessage());
        throw new IOException("Error communicating with AWS", e);
    }
    return result;
}

From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java

License:Apache License

private ExpandedGlob expandGlob(S3ResourceId glob) {
    // The S3 API can list objects, filtered by prefix, but not by wildcard.
    // Here, we find the longest prefix without wildcard "*",
    // then filter the results with a regex.
    checkArgument(glob.isWildcard(), "isWildcard");
    String keyPrefix = glob.getKeyNonWildcardPrefix();
    Pattern wildcardRegexp = Pattern.compile(wildcardToRegexp(glob.getKey()));

    LOG.debug("expanding bucket {}, prefix {}, against pattern {}", glob.getBucket(), keyPrefix,
            wildcardRegexp.toString());//from  ww  w .  j a  va 2s  .c  o m

    ImmutableList.Builder<S3ResourceId> expandedPaths = ImmutableList.builder();
    String continuationToken = null;

    do {
        ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(glob.getBucket())
                .withPrefix(keyPrefix).withContinuationToken(continuationToken);
        ListObjectsV2Result result;
        try {
            result = amazonS3.get().listObjectsV2(request);
        } catch (AmazonClientException e) {
            return ExpandedGlob.create(glob, new IOException(e));
        }
        continuationToken = result.getNextContinuationToken();

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            // Filter against regex.
            if (wildcardRegexp.matcher(objectSummary.getKey()).matches()) {
                S3ResourceId expandedPath = S3ResourceId
                        .fromComponents(objectSummary.getBucketName(), objectSummary.getKey())
                        .withSize(objectSummary.getSize()).withLastModified(objectSummary.getLastModified());
                LOG.debug("Expanded S3 object path {}", expandedPath);
                expandedPaths.add(expandedPath);
            }
        }
    } while (continuationToken != null);

    return ExpandedGlob.create(glob, expandedPaths.build());
}

From source file:org.apache.druid.storage.s3.S3DataSegmentMover.java

License:Apache License

/**
 * Copies an object and after that checks that the object is present at the target location, via a separate API call.
 * If it is not, an exception is thrown, and the object is not deleted at the old location. This "paranoic" check
 * is added after it was observed that S3 may report a successful move, and the object is not found at the target
 * location.//from www.  ja  va  2 s . c  o m
 */
private void selfCheckingMove(String s3Bucket, String targetS3Bucket, String s3Path, String targetS3Path,
        String copyMsg) throws IOException, SegmentLoadingException {
    if (s3Bucket.equals(targetS3Bucket) && s3Path.equals(targetS3Path)) {
        log.info("No need to move file[s3://%s/%s] onto itself", s3Bucket, s3Path);
        return;
    }
    if (s3Client.doesObjectExist(s3Bucket, s3Path)) {
        final ListObjectsV2Result listResult = s3Client.listObjectsV2(
                new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3Path).withMaxKeys(1));
        // Using getObjectSummaries().size() instead of getKeyCount as, in some cases
        // it is observed that even though the getObjectSummaries returns some data
        // keyCount is still zero.
        if (listResult.getObjectSummaries().size() == 0) {
            // should never happen
            throw new ISE("Unable to list object [s3://%s/%s]", s3Bucket, s3Path);
        }
        final S3ObjectSummary objectSummary = listResult.getObjectSummaries().get(0);
        if (objectSummary.getStorageClass() != null
                && StorageClass.fromValue(StringUtils.toUpperCase(objectSummary.getStorageClass()))
                        .equals(StorageClass.Glacier)) {
            throw new AmazonServiceException(StringUtils.format(
                    "Cannot move file[s3://%s/%s] of storage class glacier, skipping.", s3Bucket, s3Path));
        } else {
            log.info("Moving file %s", copyMsg);
            final CopyObjectRequest copyRequest = new CopyObjectRequest(s3Bucket, s3Path, targetS3Bucket,
                    targetS3Path);
            if (!config.getDisableAcl()) {
                copyRequest
                        .setAccessControlList(S3Utils.grantFullControlToBucketOwner(s3Client, targetS3Bucket));
            }
            s3Client.copyObject(copyRequest);
            if (!s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) {
                throw new IOE(
                        "After copy was reported as successful the file doesn't exist in the target location [%s]",
                        copyMsg);
            }
            deleteWithRetriesSilent(s3Bucket, s3Path);
            log.debug("Finished moving file %s", copyMsg);
        }
    } else {
        // ensure object exists in target location
        if (s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) {
            log.info("Not moving file [s3://%s/%s], already present in target location [s3://%s/%s]", s3Bucket,
                    s3Path, targetS3Bucket, targetS3Path);
        } else {
            throw new SegmentLoadingException(
                    "Unable to move file %s, not present in either source or target location", copyMsg);
        }
    }
}

From source file:org.apache.druid.storage.s3.S3Utils.java

License:Apache License

public static Iterator<S3ObjectSummary> objectSummaryIterator(final ServerSideEncryptingAmazonS3 s3Client,
        final String bucket, final String prefix, final int numMaxKeys) {
    final ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket).withPrefix(prefix)
            .withMaxKeys(numMaxKeys);//  w  w w  . j a va  2 s  .co  m

    return new Iterator<S3ObjectSummary>() {
        private ListObjectsV2Result result;
        private Iterator<S3ObjectSummary> objectSummaryIterator;

        {
            fetchNextBatch();
        }

        private void fetchNextBatch() {
            result = s3Client.listObjectsV2(request);
            objectSummaryIterator = result.getObjectSummaries().iterator();
            request.setContinuationToken(result.getContinuationToken());
        }

        @Override
        public boolean hasNext() {
            return objectSummaryIterator.hasNext() || result.isTruncated();
        }

        @Override
        public S3ObjectSummary next() {
            if (!hasNext()) {
                throw new NoSuchElementException();
            }

            if (objectSummaryIterator.hasNext()) {
                return objectSummaryIterator.next();
            }

            if (result.isTruncated()) {
                fetchNextBatch();
            }

            if (!objectSummaryIterator.hasNext()) {
                throw new ISE(
                        "Failed to further iterate on bucket[%s] and prefix[%s]. The last continuationToken was [%s]",
                        bucket, prefix, result.getContinuationToken());
            }

            return objectSummaryIterator.next();
        }
    };
}

From source file:org.apache.druid.storage.s3.S3Utils.java

License:Apache License

/**
 * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple
 * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique
 * in the given bucket.//from   w  w w  .j  a  v  a 2 s  .  c om
 *
 * @param s3Client s3 client
 * @param bucket   s3 bucket
 * @param key      unique key for the object to be retrieved
 */
public static S3ObjectSummary getSingleObjectSummary(ServerSideEncryptingAmazonS3 s3Client, String bucket,
        String key) {
    final ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket).withPrefix(key)
            .withMaxKeys(1);
    final ListObjectsV2Result result = s3Client.listObjectsV2(request);

    // Using getObjectSummaries().size() instead of getKeyCount as, in some cases
    // it is observed that even though the getObjectSummaries returns some data
    // keyCount is still zero.
    if (result.getObjectSummaries().size() == 0) {
        throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key);
    }
    final S3ObjectSummary objectSummary = result.getObjectSummaries().get(0);
    if (!objectSummary.getBucketName().equals(bucket) || !objectSummary.getKey().equals(key)) {
        throw new ISE("Wrong object[%s] for bucket[%s] and key[%s]", objectSummary, bucket, key);
    }

    return objectSummary;
}

From source file:org.apache.flink.cloudsort.io.aws.AwsInput.java

License:Apache License

@Override
public List<InputSplit> list() {
    Preconditions.checkNotNull(bucket);//from  w  w  w. ja v a  2s  .co  m
    Preconditions.checkNotNull(prefix);

    List<InputSplit> objectNames = new ArrayList<>();

    // this will read credentials from user's home directory
    AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());

    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket).withPrefix(prefix);

    ListObjectsV2Result result;
    int index = 0;
    do {
        result = s3client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            String objectName = objectSummary.getKey();
            long objectSize = objectSummary.getSize();
            objectNames.add(new InputSplit(index++, objectName, objectSize));
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated());

    return objectNames;
}

From source file:org.exem.flamingo.web.filesystem.s3.S3BrowserServiceImpl.java

License:Apache License

@Override
public void deleteBucket(String bucketName) {
    ListObjectsV2Request request = new ListObjectsV2Request();
    request.withBucketName(bucketName);//from w  w w .j av  a  2s.c o  m
    ListObjectsV2Result result;

    while (true) {
        result = this.s3.listObjectsV2(request);
        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            this.s3.deleteObject(bucketName, objectSummary.getKey());
        }
        if (result.isTruncated()) {
            request.setContinuationToken(result.getNextContinuationToken());
        } else {
            break;
        }
    }

    this.s3.deleteBucket(bucketName);
}

From source file:org.exem.flamingo.web.filesystem.s3.S3BrowserServiceImpl.java

License:Apache License

@Override
public void deleteFolder(String bucketName, String key) {
    ListObjectsV2Request request = new ListObjectsV2Request();
    request.withBucketName(bucketName).withPrefix(key).withDelimiter(S3Constansts.DELIMITER);

    ListObjectsV2Result result;/*w w w. j  a va  2  s. c om*/

    while (true) {
        result = this.s3.listObjectsV2(request);
        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            this.s3.deleteObject(bucketName, objectSummary.getKey());
        }
        if (result.isTruncated()) {
            request.setContinuationToken(result.getNextContinuationToken());
        } else {
            break;
        }
    }
}

From source file:org.exem.flamingo.web.filesystem.s3.S3BrowserServiceImpl.java

License:Apache License

@Override
public ListObjectsV2Result listObjects(String bucketName, String prefix, String continuationToken) {
    ListObjectsV2Request request = new ListObjectsV2Request();
    request.withBucketName(bucketName).withPrefix(prefix).withDelimiter(S3Constansts.DELIMITER);

    if (StringUtils.isNotEmpty(continuationToken)) {
        request.withContinuationToken(continuationToken);
    }//from ww  w. j a v a 2 s.  co  m

    return this.s3.listObjectsV2(request);
}