Example usage for com.amazonaws.services.s3.model ListObjectsV2Result getNextContinuationToken

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ListObjectsV2Result getNextContinuationToken.

Prototype

public String getNextContinuationToken() 

Source Link

Document

Gets the optional NextContinuationToken.

Usage

From source file:alluxio.underfs.s3a.S3AUnderFileSystem.java

License:Apache License

/**
 * Lists the files in the given path, the paths will be their logical names and not contain the
 * folder suffix. Note that, the list results are unsorted.
 *
 * @param path the key to list/* w  ww  . j  a  va2s.co  m*/
 * @param recursive if true will list children directories as well
 * @return an array of the file and folder names in this directory
 * @throws IOException if an I/O error occurs
 */
private String[] listInternal(String path, boolean recursive) throws IOException {
    path = stripPrefixIfPresent(path);
    path = PathUtils.normalizePath(path, PATH_SEPARATOR);
    path = path.equals(PATH_SEPARATOR) ? "" : path;
    String delimiter = recursive ? "" : PATH_SEPARATOR;
    Set<String> children = new HashSet<>();
    try {
        ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(mBucketName).withPrefix(path)
                .withDelimiter(delimiter).withMaxKeys(LISTING_LENGTH);
        ListObjectsV2Result result = null;
        while (result == null || result.isTruncated()) {
            // Query S3 for the next batch of objects
            result = mClient.listObjectsV2(request);
            // Advance the request continuation token to the next set of objects
            request.setContinuationToken(result.getNextContinuationToken());

            // Directories in S3 UFS can be possibly encoded in two different ways:
            // (1) as file objects with FOLDER_SUFFIX for directories created through Alluxio or
            // (2) as "common prefixes" of other files objects for directories not created through
            // Alluxio
            //
            // Case (1) (and file objects) is accounted for by iterating over chunk.getObjects() while
            // case (2) is accounted for by iterating over chunk.getCommonPrefixes().
            //
            // An example, with prefix="ufs" and delimiter="/" and LISTING_LENGTH=5
            // - objects.key = ufs/, child =
            // - objects.key = ufs/dir1_$folder$, child = dir1
            // - objects.key = ufs/file, child = file
            // - commonPrefix = ufs/dir1/, child = dir1
            // - commonPrefix = ufs/dir2/, child = dir2

            // Handle case (1)
            for (S3ObjectSummary obj : result.getObjectSummaries()) {
                // Remove parent portion of the key
                String child = getChildName(obj.getKey(), path);
                // Prune the special folder suffix
                child = CommonUtils.stripSuffixIfPresent(child, FOLDER_SUFFIX);
                // Only add if the path is not empty (removes results equal to the path)
                if (!child.isEmpty()) {
                    children.add(child);
                }
            }
            // Handle case (2)
            for (String commonPrefix : result.getCommonPrefixes()) {
                // Remove parent portion of the key
                String child = getChildName(commonPrefix, path);
                // Remove any portion after the last path delimiter
                int childNameIndex = child.lastIndexOf(PATH_SEPARATOR);
                child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
                if (!child.isEmpty() && !children.contains(child)) {
                    // This directory has not been created through Alluxio.
                    mkdirsInternal(commonPrefix);
                    children.add(child);
                }
            }
        }
        return children.toArray(new String[children.size()]);
    } catch (AmazonClientException e) {
        LOG.error("Failed to list path {}", path, e);
        return null;
    }
}

---

From source file:com.dustindoloff.s3websitedeploy.Main.java

License:Apache License

private static boolean emptyBucket(final AmazonS3 s3Client, final String bucket) {
    final ListObjectsV2Request request = new ListObjectsV2Request();
    request.setBucketName(bucket);/* w ww  .  ja v a2 s. com*/

    String continuationToken = null;
    ListObjectsV2Result result;
    do {
        request.setContinuationToken(continuationToken);
        result = s3Client.listObjectsV2(bucket);
        for (final S3ObjectSummary summary : result.getObjectSummaries()) {
            s3Client.deleteObject(bucket, summary.getKey());
        }

        continuationToken = result.getNextContinuationToken();
    } while (result.isTruncated());

    return true;
}

---

From source file:com.handywedge.binarystore.store.aws.BinaryStoreManagerImpl.java

License:MIT License

@Override
public List<BinaryInfo> list(StorageInfo storage, BinaryInfo binary) throws StoreException {
    logger.debug("={}", storage);
    logger.debug("?={}", binary);

    List<BinaryInfo> objInfoList = new ArrayList<BinaryInfo>();

    AmazonS3 s3client = getS3Client(binary.getBucketName());

    try {//from  w ww . j a v a 2  s . com
        logger.debug("Listing binaries");
        final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(binary.getBucketName())
                .withMaxKeys(2);
        ListObjectsV2Result result;
        do {
            result = s3client.listObjectsV2(req);
            for (S3ObjectSummary binarySummary : result.getObjectSummaries()) {
                logger.debug(" - {}(size={})", binarySummary.getKey(), binarySummary.getSize());
                if (binarySummary.getSize() != 0) {
                    BinaryInfo objInfo = new BinaryInfo(binary.getBucketName());
                    objInfo.setFileName(binarySummary.getKey());
                    objInfo.setSize(binarySummary.getSize());
                    S3Object s3Object = s3client
                            .getObject(new GetObjectRequest(binary.getBucketName(), binarySummary.getKey()));
                    objInfo.setContentType(s3Object.getObjectMetadata().getContentType());
                    objInfo.setUrl(s3client.getUrl(binary.getBucketName(), binarySummary.getKey()).toString());

                    logger.debug("Generating pre-signed URL.");
                    URL PresignedUrl = getPresignedUrl(s3client, binary.getBucketName(),
                            binarySummary.getKey());
                    objInfo.setPresignedUrl(PresignedUrl.toString());
                    logger.debug("Pre-Signed URL = " + PresignedUrl.toString());

                    objInfoList.add(objInfo);
                }
            }
            logger.debug("Next Continuation Token : " + result.getNextContinuationToken());
            req.setContinuationToken(result.getNextContinuationToken());
        } while (result.isTruncated() == true);

    } catch (AmazonServiceException ase) {
        throw new StoreException(HttpStatus.SC_BAD_REQUEST, ErrorClassification.LIST_FAIL, ase,
                binary.getFileName());
    } catch (AmazonClientException ace) {
        throw new StoreException(HttpStatus.SC_BAD_REQUEST, ErrorClassification.LIST_FAIL, ace,
                binary.getFileName());
    }

    logger.info(" ={}", objInfoList.size());
    return objInfoList;
}

---

From source file:com.mesosphere.dcos.cassandra.executor.backup.S3StorageDriver.java

License:Apache License

private static Map<String, Long> listSnapshotFiles(AmazonS3Client amazonS3Client, String bucketName,
        String backupName) {/* w  w  w  .j  av a  2s .c om*/
    Map<String, Long> snapshotFiles = new HashMap<>();
    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName)
            .withPrefix(backupName);
    ListObjectsV2Result result;
    do {
        result = amazonS3Client.listObjectsV2(req);
        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            snapshotFiles.put(objectSummary.getKey(), objectSummary.getSize());
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated());

    return snapshotFiles;
}

---

From source file:edu.harvard.hms.dbmi.bd2k.irct.aws.event.result.S3AfterGetResult.java

License:Mozilla Public License

@Override
public void fire(Result result) {
    if (result.getResultStatus() != ResultStatus.AVAILABLE) {
        return;/*from www  . java  2s  .c o  m*/
    }
    if (!result.getResultSetLocation().startsWith("S3://")) {
        File temp = new File(result.getResultSetLocation());
        if (temp.exists()) {
            return;
        } else {
            result.setResultSetLocation(
                    "S3://" + s3Folder + result.getResultSetLocation().replaceAll(irctSaveLocation + "/", ""));
        }
    }
    String location = result.getResultSetLocation().substring(5);
    // List the files in that bucket path
    try {

        final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName)
                .withPrefix(location);

        // Loop Through all the files
        ListObjectsV2Result s3Files;
        do {
            s3Files = s3client.listObjectsV2(req);
            for (S3ObjectSummary objectSummary : s3Files.getObjectSummaries()) {
                // Download the files to the directory specified
                String keyName = objectSummary.getKey();
                String fileName = irctSaveLocation + keyName.replace(location, "");
                log.info("Downloading: " + keyName + " --> " + fileName);
                s3client.getObject(new GetObjectRequest(bucketName, keyName), new File(fileName));
            }
            req.setContinuationToken(s3Files.getNextContinuationToken());
        } while (s3Files.isTruncated() == true);

        // Update the result set id
        result.setResultSetLocation(irctSaveLocation + "/" + location.replace(s3Folder, ""));

    } catch (AmazonServiceException ase) {
        log.warn("Caught an AmazonServiceException, which " + "means your request made it "
                + "to Amazon S3, but was rejected with an error response" + " for some reason.");
        log.warn("Error Message:    " + ase.getMessage());
        log.warn("HTTP Status Code: " + ase.getStatusCode());
        log.warn("AWS Error Code:   " + ase.getErrorCode());
        log.warn("Error Type:       " + ase.getErrorType());
        log.warn("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        log.warn("Caught an AmazonClientException, which " + "means the client encountered "
                + "an internal error while trying to " + "communicate with S3, "
                + "such as not being able to access the network.");
        log.warn("Error Message: " + ace.getMessage());
    }
}

---

From source file:mx.iteso.desi.cloud.hw3.AWSFaceCompare.java

License:Apache License

public Face compare(ByteBuffer imageBuffer) {
    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(srcBucket).withPrefix("Faces/");
    ListObjectsV2Result result;
    ArrayList<String> s3Files = new ArrayList<>();
    Face empty = new Face("", 0.0f);

    try {/*from  w  ww.j a v  a2s  .c  o m*/
        String sourceFile = "source.jpg";
        Path file = Paths.get(sourceFile);
        Files.write(file, imageBuffer.array(), StandardOpenOption.CREATE);
        upload(sourceFile);
    } catch (IOException ex) {
        Logger.getLogger(FaceAddFrame.class.getName()).log(Level.SEVERE, null, ex);
    }

    do {
        result = s3.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            s3Files.add(objectSummary.getKey());
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated() == true);

    for (String s3File : s3Files) {
        if (s3File.endsWith(".jpg")) {
            System.out.println("Checking " + s3File + "...");
            Face face = compare("Compare/source.jpg", s3File);
            if (face.getCofidence() > SIMILARITY_THRESHOLD) {
                return face;
            }
        }
    }
    return empty;
}

---

From source file:net.solarnetwork.node.backup.s3.SdkS3Client.java

License:Open Source License

@Override
public Set<S3ObjectReference> listObjects(String prefix) throws IOException {
    AmazonS3 client = getClient();/*from w w  w .ja v a  2s .c  o m*/
    Set<S3ObjectReference> result = new LinkedHashSet<>(100);
    try {
        final ListObjectsV2Request req = new ListObjectsV2Request();
        req.setBucketName(bucketName);
        req.setMaxKeys(maximumKeysPerRequest);
        req.setPrefix(prefix);
        ListObjectsV2Result listResult;
        do {
            listResult = client.listObjectsV2(req);

            for (S3ObjectSummary objectSummary : listResult.getObjectSummaries()) {
                result.add(new S3ObjectReference(objectSummary.getKey(), objectSummary.getSize(),
                        objectSummary.getLastModified()));
            }
            req.setContinuationToken(listResult.getNextContinuationToken());
        } while (listResult.isTruncated() == true);

    } catch (AmazonServiceException e) {
        log.warn("AWS error: {}; HTTP code {}; AWS code {}; type {}; request ID {}", e.getMessage(),
                e.getStatusCode(), e.getErrorCode(), e.getErrorType(), e.getRequestId());
        throw new RemoteServiceException("Error listing S3 objects at " + prefix, e);
    } catch (AmazonClientException e) {
        log.debug("Error communicating with AWS: {}", e.getMessage());
        throw new IOException("Error communicating with AWS", e);
    }
    return result;
}

---

From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java

License:Apache License

private ExpandedGlob expandGlob(S3ResourceId glob) {
    // The S3 API can list objects, filtered by prefix, but not by wildcard.
    // Here, we find the longest prefix without wildcard "*",
    // then filter the results with a regex.
    checkArgument(glob.isWildcard(), "isWildcard");
    String keyPrefix = glob.getKeyNonWildcardPrefix();
    Pattern wildcardRegexp = Pattern.compile(wildcardToRegexp(glob.getKey()));

    LOG.debug("expanding bucket {}, prefix {}, against pattern {}", glob.getBucket(), keyPrefix,
            wildcardRegexp.toString());/*from  ww  w.  j a  va 2  s  . co  m*/

    ImmutableList.Builder<S3ResourceId> expandedPaths = ImmutableList.builder();
    String continuationToken = null;

    do {
        ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(glob.getBucket())
                .withPrefix(keyPrefix).withContinuationToken(continuationToken);
        ListObjectsV2Result result;
        try {
            result = amazonS3.get().listObjectsV2(request);
        } catch (AmazonClientException e) {
            return ExpandedGlob.create(glob, new IOException(e));
        }
        continuationToken = result.getNextContinuationToken();

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            // Filter against regex.
            if (wildcardRegexp.matcher(objectSummary.getKey()).matches()) {
                S3ResourceId expandedPath = S3ResourceId
                        .fromComponents(objectSummary.getBucketName(), objectSummary.getKey())
                        .withSize(objectSummary.getSize()).withLastModified(objectSummary.getLastModified());
                LOG.debug("Expanded S3 object path {}", expandedPath);
                expandedPaths.add(expandedPath);
            }
        }
    } while (continuationToken != null);

    return ExpandedGlob.create(glob, expandedPaths.build());
}

---

From source file:org.apache.flink.cloudsort.io.aws.AwsInput.java

License:Apache License

@Override
public List<InputSplit> list() {
    Preconditions.checkNotNull(bucket);//from w  w  w  .ja va2  s  . c om
    Preconditions.checkNotNull(prefix);

    List<InputSplit> objectNames = new ArrayList<>();

    // this will read credentials from user's home directory
    AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());

    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket).withPrefix(prefix);

    ListObjectsV2Result result;
    int index = 0;
    do {
        result = s3client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            String objectName = objectSummary.getKey();
            long objectSize = objectSummary.getSize();
            objectNames.add(new InputSplit(index++, objectName, objectSize));
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated());

    return objectNames;
}

---

From source file:org.exem.flamingo.web.filesystem.s3.S3BrowserController.java

License:Apache License

@RequestMapping(value = "listFolders", method = RequestMethod.GET)
@ResponseStatus(HttpStatus.OK)/*  w  ww .  j  a  va2s . co m*/
public Response listFolders(@RequestParam(required = false) String bucketName,
        @RequestParam(required = false) String prefix,
        @RequestParam(required = false) String continuationToken) {

    // Get bucket list
    if (StringUtils.isEmpty(bucketName)) {
        Response response = new Response();
        response.getList().addAll(getBucketList());
        response.setSuccess(true);
        return response;
    }

    // Get folder list
    ListObjectsV2Result result = s3BrowserService.listObjects(bucketName, prefix, continuationToken);

    List<S3ObjectInfo> list = new ArrayList<>();
    List<String> commonPrefixes = result.getCommonPrefixes();
    for (String key : commonPrefixes) {
        S3ObjectInfo object = new S3ObjectInfo();
        object.setKey(key);
        object.setName(getName(key));
        object.setBucketName(bucketName);
        object.setFolder(true);
        list.add(object);
    }

    Map<String, String> map = new HashMap<>();
    map.put(S3Constansts.CONTINUATIONTOKEN, result.getNextContinuationToken());
    map.put(S3Constansts.ISTRUNCATED, BooleanUtils.toStringTrueFalse(result.isTruncated()));

    Response response = new Response();
    response.getList().addAll(list);
    response.getMap().putAll(map);
    response.setSuccess(true);
    return response;
}

---

• «
• 1
• 2
• »