List of usage examples for com.amazonaws.services.s3.model ListObjectsV2Request setContinuationToken
public void setContinuationToken(String continuationToken)
From source file:alluxio.underfs.s3a.S3AUnderFileSystem.java
License:Apache License
/** * Lists the files in the given path, the paths will be their logical names and not contain the * folder suffix. Note that, the list results are unsorted. * * @param path the key to list/*from www . j ava2 s. co m*/ * @param recursive if true will list children directories as well * @return an array of the file and folder names in this directory * @throws IOException if an I/O error occurs */ private String[] listInternal(String path, boolean recursive) throws IOException { path = stripPrefixIfPresent(path); path = PathUtils.normalizePath(path, PATH_SEPARATOR); path = path.equals(PATH_SEPARATOR) ? "" : path; String delimiter = recursive ? "" : PATH_SEPARATOR; Set<String> children = new HashSet<>(); try { ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(mBucketName).withPrefix(path) .withDelimiter(delimiter).withMaxKeys(LISTING_LENGTH); ListObjectsV2Result result = null; while (result == null || result.isTruncated()) { // Query S3 for the next batch of objects result = mClient.listObjectsV2(request); // Advance the request continuation token to the next set of objects request.setContinuationToken(result.getNextContinuationToken()); // Directories in S3 UFS can be possibly encoded in two different ways: // (1) as file objects with FOLDER_SUFFIX for directories created through Alluxio or // (2) as "common prefixes" of other files objects for directories not created through // Alluxio // // Case (1) (and file objects) is accounted for by iterating over chunk.getObjects() while // case (2) is accounted for by iterating over chunk.getCommonPrefixes(). // // An example, with prefix="ufs" and delimiter="/" and LISTING_LENGTH=5 // - objects.key = ufs/, child = // - objects.key = ufs/dir1_$folder$, child = dir1 // - objects.key = ufs/file, child = file // - commonPrefix = ufs/dir1/, child = dir1 // - commonPrefix = ufs/dir2/, child = dir2 // Handle case (1) for (S3ObjectSummary obj : result.getObjectSummaries()) { // Remove parent portion of the key String child = getChildName(obj.getKey(), path); // Prune the special folder suffix child = CommonUtils.stripSuffixIfPresent(child, FOLDER_SUFFIX); // Only add if the path is not empty (removes results equal to the path) if (!child.isEmpty()) { children.add(child); } } // Handle case (2) for (String commonPrefix : result.getCommonPrefixes()) { // Remove parent portion of the key String child = getChildName(commonPrefix, path); // Remove any portion after the last path delimiter int childNameIndex = child.lastIndexOf(PATH_SEPARATOR); child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child; if (!child.isEmpty() && !children.contains(child)) { // This directory has not been created through Alluxio. mkdirsInternal(commonPrefix); children.add(child); } } } return children.toArray(new String[children.size()]); } catch (AmazonClientException e) { LOG.error("Failed to list path {}", path, e); return null; } }
From source file:com.dustindoloff.s3websitedeploy.Main.java
License:Apache License
private static boolean emptyBucket(final AmazonS3 s3Client, final String bucket) { final ListObjectsV2Request request = new ListObjectsV2Request(); request.setBucketName(bucket);//from w w w . j a va 2 s. c o m String continuationToken = null; ListObjectsV2Result result; do { request.setContinuationToken(continuationToken); result = s3Client.listObjectsV2(bucket); for (final S3ObjectSummary summary : result.getObjectSummaries()) { s3Client.deleteObject(bucket, summary.getKey()); } continuationToken = result.getNextContinuationToken(); } while (result.isTruncated()); return true; }
From source file:com.handywedge.binarystore.store.aws.BinaryStoreManagerImpl.java
License:MIT License
@Override public List<BinaryInfo> list(StorageInfo storage, BinaryInfo binary) throws StoreException { logger.debug("={}", storage); logger.debug("?={}", binary); List<BinaryInfo> objInfoList = new ArrayList<BinaryInfo>(); AmazonS3 s3client = getS3Client(binary.getBucketName()); try {/*w w w. j av a2s. co m*/ logger.debug("Listing binaries"); final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(binary.getBucketName()) .withMaxKeys(2); ListObjectsV2Result result; do { result = s3client.listObjectsV2(req); for (S3ObjectSummary binarySummary : result.getObjectSummaries()) { logger.debug(" - {}(size={})", binarySummary.getKey(), binarySummary.getSize()); if (binarySummary.getSize() != 0) { BinaryInfo objInfo = new BinaryInfo(binary.getBucketName()); objInfo.setFileName(binarySummary.getKey()); objInfo.setSize(binarySummary.getSize()); S3Object s3Object = s3client .getObject(new GetObjectRequest(binary.getBucketName(), binarySummary.getKey())); objInfo.setContentType(s3Object.getObjectMetadata().getContentType()); objInfo.setUrl(s3client.getUrl(binary.getBucketName(), binarySummary.getKey()).toString()); logger.debug("Generating pre-signed URL."); URL PresignedUrl = getPresignedUrl(s3client, binary.getBucketName(), binarySummary.getKey()); objInfo.setPresignedUrl(PresignedUrl.toString()); logger.debug("Pre-Signed URL = " + PresignedUrl.toString()); objInfoList.add(objInfo); } } logger.debug("Next Continuation Token : " + result.getNextContinuationToken()); req.setContinuationToken(result.getNextContinuationToken()); } while (result.isTruncated() == true); } catch (AmazonServiceException ase) { throw new StoreException(HttpStatus.SC_BAD_REQUEST, ErrorClassification.LIST_FAIL, ase, binary.getFileName()); } catch (AmazonClientException ace) { throw new StoreException(HttpStatus.SC_BAD_REQUEST, ErrorClassification.LIST_FAIL, ace, binary.getFileName()); } logger.info(" ={}", objInfoList.size()); return objInfoList; }
From source file:com.mesosphere.dcos.cassandra.executor.backup.S3StorageDriver.java
License:Apache License
private static Map<String, Long> listSnapshotFiles(AmazonS3Client amazonS3Client, String bucketName, String backupName) {//from w w w .ja va2 s . c om Map<String, Long> snapshotFiles = new HashMap<>(); final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName) .withPrefix(backupName); ListObjectsV2Result result; do { result = amazonS3Client.listObjectsV2(req); for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { snapshotFiles.put(objectSummary.getKey(), objectSummary.getSize()); } req.setContinuationToken(result.getNextContinuationToken()); } while (result.isTruncated()); return snapshotFiles; }
From source file:edu.harvard.hms.dbmi.bd2k.irct.aws.event.result.S3AfterGetResult.java
License:Mozilla Public License
@Override public void fire(Result result) { if (result.getResultStatus() != ResultStatus.AVAILABLE) { return;/*from w w w . j a v a 2 s.c om*/ } if (!result.getResultSetLocation().startsWith("S3://")) { File temp = new File(result.getResultSetLocation()); if (temp.exists()) { return; } else { result.setResultSetLocation( "S3://" + s3Folder + result.getResultSetLocation().replaceAll(irctSaveLocation + "/", "")); } } String location = result.getResultSetLocation().substring(5); // List the files in that bucket path try { final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName) .withPrefix(location); // Loop Through all the files ListObjectsV2Result s3Files; do { s3Files = s3client.listObjectsV2(req); for (S3ObjectSummary objectSummary : s3Files.getObjectSummaries()) { // Download the files to the directory specified String keyName = objectSummary.getKey(); String fileName = irctSaveLocation + keyName.replace(location, ""); log.info("Downloading: " + keyName + " --> " + fileName); s3client.getObject(new GetObjectRequest(bucketName, keyName), new File(fileName)); } req.setContinuationToken(s3Files.getNextContinuationToken()); } while (s3Files.isTruncated() == true); // Update the result set id result.setResultSetLocation(irctSaveLocation + "/" + location.replace(s3Folder, "")); } catch (AmazonServiceException ase) { log.warn("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); log.warn("Error Message: " + ase.getMessage()); log.warn("HTTP Status Code: " + ase.getStatusCode()); log.warn("AWS Error Code: " + ase.getErrorCode()); log.warn("Error Type: " + ase.getErrorType()); log.warn("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { log.warn("Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); log.warn("Error Message: " + ace.getMessage()); } }
From source file:io.druid.storage.s3.S3Utils.java
License:Apache License
public static Iterator<S3ObjectSummary> objectSummaryIterator(final AmazonS3 s3Client, final String bucket, final String prefix, final int numMaxKeys) { final ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket).withPrefix(prefix) .withMaxKeys(numMaxKeys);//w ww .ja va 2s . co m return new Iterator<S3ObjectSummary>() { private ListObjectsV2Result result; private Iterator<S3ObjectSummary> objectSummaryIterator; { fetchNextBatch(); } private void fetchNextBatch() { result = s3Client.listObjectsV2(request); objectSummaryIterator = result.getObjectSummaries().iterator(); request.setContinuationToken(result.getContinuationToken()); } @Override public boolean hasNext() { return objectSummaryIterator.hasNext() || result.isTruncated(); } @Override public S3ObjectSummary next() { if (!hasNext()) { throw new NoSuchElementException(); } if (objectSummaryIterator.hasNext()) { return objectSummaryIterator.next(); } if (result.isTruncated()) { fetchNextBatch(); } if (!objectSummaryIterator.hasNext()) { throw new ISE( "Failed to further iterate on bucket[%s] and prefix[%s]. The last continuationToken was [%s]", bucket, prefix, result.getContinuationToken()); } return objectSummaryIterator.next(); } }; }
From source file:mx.iteso.desi.cloud.hw3.AWSFaceCompare.java
License:Apache License
public Face compare(ByteBuffer imageBuffer) { final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(srcBucket).withPrefix("Faces/"); ListObjectsV2Result result;/*from ww w. j a v a 2 s .co m*/ ArrayList<String> s3Files = new ArrayList<>(); Face empty = new Face("", 0.0f); try { String sourceFile = "source.jpg"; Path file = Paths.get(sourceFile); Files.write(file, imageBuffer.array(), StandardOpenOption.CREATE); upload(sourceFile); } catch (IOException ex) { Logger.getLogger(FaceAddFrame.class.getName()).log(Level.SEVERE, null, ex); } do { result = s3.listObjectsV2(req); for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { s3Files.add(objectSummary.getKey()); } req.setContinuationToken(result.getNextContinuationToken()); } while (result.isTruncated() == true); for (String s3File : s3Files) { if (s3File.endsWith(".jpg")) { System.out.println("Checking " + s3File + "..."); Face face = compare("Compare/source.jpg", s3File); if (face.getCofidence() > SIMILARITY_THRESHOLD) { return face; } } } return empty; }
From source file:net.solarnetwork.node.backup.s3.SdkS3Client.java
License:Open Source License
@Override public Set<S3ObjectReference> listObjects(String prefix) throws IOException { AmazonS3 client = getClient();//from w w w . j av a 2 s .c o m Set<S3ObjectReference> result = new LinkedHashSet<>(100); try { final ListObjectsV2Request req = new ListObjectsV2Request(); req.setBucketName(bucketName); req.setMaxKeys(maximumKeysPerRequest); req.setPrefix(prefix); ListObjectsV2Result listResult; do { listResult = client.listObjectsV2(req); for (S3ObjectSummary objectSummary : listResult.getObjectSummaries()) { result.add(new S3ObjectReference(objectSummary.getKey(), objectSummary.getSize(), objectSummary.getLastModified())); } req.setContinuationToken(listResult.getNextContinuationToken()); } while (listResult.isTruncated() == true); } catch (AmazonServiceException e) { log.warn("AWS error: {}; HTTP code {}; AWS code {}; type {}; request ID {}", e.getMessage(), e.getStatusCode(), e.getErrorCode(), e.getErrorType(), e.getRequestId()); throw new RemoteServiceException("Error listing S3 objects at " + prefix, e); } catch (AmazonClientException e) { log.debug("Error communicating with AWS: {}", e.getMessage()); throw new IOException("Error communicating with AWS", e); } return result; }
From source file:org.apache.druid.storage.s3.S3Utils.java
License:Apache License
public static Iterator<S3ObjectSummary> objectSummaryIterator(final ServerSideEncryptingAmazonS3 s3Client, final String bucket, final String prefix, final int numMaxKeys) { final ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket).withPrefix(prefix) .withMaxKeys(numMaxKeys);/* w w w .j a v a 2s .c o m*/ return new Iterator<S3ObjectSummary>() { private ListObjectsV2Result result; private Iterator<S3ObjectSummary> objectSummaryIterator; { fetchNextBatch(); } private void fetchNextBatch() { result = s3Client.listObjectsV2(request); objectSummaryIterator = result.getObjectSummaries().iterator(); request.setContinuationToken(result.getContinuationToken()); } @Override public boolean hasNext() { return objectSummaryIterator.hasNext() || result.isTruncated(); } @Override public S3ObjectSummary next() { if (!hasNext()) { throw new NoSuchElementException(); } if (objectSummaryIterator.hasNext()) { return objectSummaryIterator.next(); } if (result.isTruncated()) { fetchNextBatch(); } if (!objectSummaryIterator.hasNext()) { throw new ISE( "Failed to further iterate on bucket[%s] and prefix[%s]. The last continuationToken was [%s]", bucket, prefix, result.getContinuationToken()); } return objectSummaryIterator.next(); } }; }
From source file:org.apache.flink.cloudsort.io.aws.AwsInput.java
License:Apache License
@Override public List<InputSplit> list() { Preconditions.checkNotNull(bucket);/*from ww w .j a v a 2 s . c om*/ Preconditions.checkNotNull(prefix); List<InputSplit> objectNames = new ArrayList<>(); // this will read credentials from user's home directory AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket).withPrefix(prefix); ListObjectsV2Result result; int index = 0; do { result = s3client.listObjectsV2(req); for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { String objectName = objectSummary.getKey(); long objectSize = objectSummary.getSize(); objectNames.add(new InputSplit(index++, objectName, objectSize)); } req.setContinuationToken(result.getNextContinuationToken()); } while (result.isTruncated()); return objectNames; }