List of usage examples for com.amazonaws.services.s3.model ObjectListing getNextMarker
public String getNextMarker()
listObjects request in order to see the next page of results. From source file:cloudExplorer.BucketClass.java
License:Open Source License
String listBucketContents(String access_key, String secret_key, String bucket, String endpoint) {
objectlist = null;/* www. j ava 2s .co m*/
AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
AmazonS3 s3Client = new AmazonS3Client(credentials,
new ClientConfiguration().withSignerOverride("S3SignerType"));
s3Client.setEndpoint(endpoint);
try {
ObjectListing current = s3Client.listObjects((bucket));
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
ObjectListing objectListing;
do {
objectListing = s3Client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
objectlist = objectlist + "@@" + objectSummary.getKey();
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (Exception listBucket) {
mainFrame.jTextArea1.append("\n" + listBucket.getMessage());
}
String parse = null;
if (objectlist != null) {
parse = objectlist;
} else {
parse = "No objects_found.";
}
return parse;
}
From source file:cloudExplorer.BucketClass.java
License:Open Source License
String getObjectInfo(String key, String access_key, String secret_key, String bucket, String endpoint,
String process) {/*from w ww . j a v a 2s . c o m*/
AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
AmazonS3 s3Client = new AmazonS3Client(credentials,
new ClientConfiguration().withSignerOverride("S3SignerType"));
s3Client.setEndpoint(endpoint);
objectlist = null;
try {
ObjectListing current = s3Client.listObjects((bucket));
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
ObjectListing objectListing;
do {
objectListing = s3Client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (process.contains("objectsize")) {
if (objectSummary.getKey().contains(key)) {
objectlist = String.valueOf(objectSummary.getSize());
break;
}
}
if (process.contains("objectdate")) {
if (objectSummary.getKey().contains(key)) {
objectlist = String.valueOf(objectSummary.getLastModified());
break;
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (Exception listBucket) {
mainFrame.jTextArea1.append("\n" + listBucket.getMessage());
}
return objectlist;
}
From source file:com.aegeus.aws.SimpleStorageService.java
License:Apache License
/** * List all files under the current directory based on bucket and prefix parameters * * @param bucket Bucket name/*from w w w .ja v a2 s . co m*/ * @param prefix Prefix name * @return File list */ public List<S3ObjectSummary> listObjects(String bucket, String prefix) { ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket); if (prefix != null) { request.setPrefix(prefix); } List<S3ObjectSummary> summaries = new ArrayList<>(); ObjectListing listing; do { listing = s3.listObjects(request); summaries.addAll(listing.getObjectSummaries()); request.setMarker(listing.getNextMarker()); } while (listing.isTruncated()); return summaries; }
From source file:com.altoukhov.svsync.fileviews.S3FileSpace.java
License:Apache License
@Override protected Snapshot scan(List<Pattern> filters) { try {//from www. j ava 2 s . c o m Map<String, FileSnapshot> files = new LinkedHashMap<>(); Set<String> dirs = new HashSet<>(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withPrefix(rootPath.isEmpty() ? "" : rootPath + "/"); ObjectListing objectListing; do { objectListing = listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { if (isExcluded(objectSummary.getKey()) || isFiltered(objectSummary.getKey(), filters)) continue; if (objectSummary.getKey().endsWith("/")) { String filePath = trimPath(objectSummary.getKey()); filePath = filePath.equals(rootPath) ? "" : filePath.substring(rootPath.length() + (rootPath.isEmpty() ? 0 : 1)); dirs.add(filePath); System.out .println(String.format("Scanning s3://%s/%s", bucketName, objectSummary.getKey())); } else { String fileName = objectSummary.getKey(); String filePath = ""; if (fileName.contains("/")) { int fileNameSplitIndex = fileName.lastIndexOf("/"); filePath = fileName.substring(0, fileNameSplitIndex); fileName = fileName.substring(fileNameSplitIndex + 1); filePath = filePath.equals(rootPath) ? "" : filePath.substring(rootPath.length() + (rootPath.isEmpty() ? 0 : 1)); } if (filePath.equals("")) { filePath = fileName; } else { filePath = filePath + "/" + fileName; } ObjectMetadata meta = getObjectInfo(objectSummary); String lmd = meta.getUserMetaDataOf("lmd"); Date lastModified = (lmd == null) ? objectSummary.getLastModified() : new Date(Long.parseLong(lmd)); FileSnapshot file = new FileSnapshot(fileName, objectSummary.getSize(), new DateTime(lastModified), filePath); files.put(filePath, file); } } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); Snapshot snapshot = new Snapshot(files, dirs); return snapshot; } catch (AmazonClientException ex) { System.out.println("Failed to scan file space"); System.out.println(ex.getMessage()); } return null; }
From source file:com.climate.oada.dao.impl.S3ResourceDAO.java
License:Open Source License
@Override public List<FileResource> getFileUrls(Long userId, String type) { List<FileResource> retval = new ArrayList<FileResource>(); long validfor = new Long(validHours).longValue() * HOURS_TO_MILLISECONDS; try {/*from w w w.j ava 2 s . c om*/ AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); String prefix = userId.toString() + S3_SEPARATOR + type; LOG.debug("Listing objects from bucket " + bucketName + " with prefix " + prefix); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { LOG.debug(" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); Date expiration = new Date(); long milliSeconds = expiration.getTime(); milliSeconds += validfor; expiration.setTime(milliSeconds); GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest( bucketName, objectSummary.getKey()); generatePresignedUrlRequest.setMethod(HttpMethod.GET); generatePresignedUrlRequest.setExpiration(expiration); FileResource res = new FileResource(); res.setFileURL(s3client.generatePresignedUrl(generatePresignedUrlRequest)); retval.add(res); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (AmazonServiceException ase) { logAWSServiceException(ase); } catch (AmazonClientException ace) { logAWSClientException(ace); } catch (Exception e) { LOG.error("Unable to retrieve S3 file URLs " + e.getMessage()); } return retval; }
From source file:com.cloudhub.aws.extractor.AWSCSVExtractor.java
License:Apache License
/** * Requests billing information from Amazon S3. * This method may spawn multiple threads as needed to complete the task. * *///from w ww . j a va2s.c o m @Override public String getTotalCost() { String totalCost = null; try { log.debug("Listing objects ..."); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (final S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { log.debug(" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); if (objectSummary.getKey().contains(Constants.MATCHER_BILLING_CSV.getKeyPattern())) { totalCost = persist(Constants.MATCHER_BILLING_CSV, objectSummary); } else if (objectSummary.getKey().contains(Constants.MATCHER_COST_ALLOCATION.getKeyPattern())) { totalCost = persist(Constants.MATCHER_COST_ALLOCATION, objectSummary); } } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (AmazonServiceException ase) { log.error("Caught an AmazonServiceException, " + "which means your request made it " + "to Amazon S3, but was rejected with an error response " + "for some reason."); log.error("Error Message: " + ase.getMessage()); log.error("HTTP Status Code: " + ase.getStatusCode()); log.error("AWS Error Code: " + ase.getErrorCode()); log.error("Error Type: " + ase.getErrorType()); log.error("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { log.error("Caught an AmazonClientException, " + "which means the client encountered " + "an internal error while trying to communicate" + " with S3, " + "such as not being able to access the network."); log.error("Error Message: " + ace.getMessage()); } catch (IOException ioe) { log.error("Caught an IOException while writing to disk."); log.error("Error Message: " + ioe.getMessage()); } return totalCost; }
From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java
License:Open Source License
@Override public ListBucketResponseType listBucket(ListBucketType request) throws S3Exception { ListBucketResponseType reply = request.getReply(); User requestUser = getRequestUser(request); OsgInternalS3Client internalS3Client = null; try {//from ww w.j a va 2 s . c om internalS3Client = getS3Client(requestUser); AmazonS3Client s3Client = internalS3Client.getS3Client(); ListObjectsRequest listRequest = new ListObjectsRequest(); listRequest.setBucketName(request.getBucket()); listRequest.setDelimiter(Strings.isNullOrEmpty(request.getDelimiter()) ? null : request.getDelimiter()); listRequest.setMarker(Strings.isNullOrEmpty(request.getMarker()) ? null : request.getMarker()); listRequest.setMaxKeys((request.getMaxKeys() == null ? null : Integer.parseInt(request.getMaxKeys()))); listRequest.setPrefix(Strings.isNullOrEmpty(request.getPrefix()) ? null : request.getPrefix()); ObjectListing response = s3Client.listObjects(listRequest); /* Non-optional, must have non-null values */ reply.setName(request.getBucket()); reply.setMaxKeys(response.getMaxKeys()); reply.setMarker(response.getMarker() == null ? "" : response.getMarker()); reply.setPrefix(response.getPrefix() == null ? "" : response.getPrefix()); reply.setIsTruncated(response.isTruncated()); /* Optional */ reply.setNextMarker(response.getNextMarker()); reply.setDelimiter(response.getDelimiter()); if (reply.getContents() == null) { reply.setContents(new ArrayList<ListEntry>()); } if (reply.getCommonPrefixesList() == null) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); } for (S3ObjectSummary obj : response.getObjectSummaries()) { //Add entry, note that the canonical user is set based on requesting user, not returned user reply.getContents() .add(new ListEntry(obj.getKey(), DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(), obj.getSize(), getCanonicalUser(requestUser), obj.getStorageClass())); } if (response.getCommonPrefixes() != null && response.getCommonPrefixes().size() > 0) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); for (String s : response.getCommonPrefixes()) { reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s)); } } return reply; } catch (AmazonServiceException e) { LOG.debug("Error from backend", e); throw S3ExceptionMapper.fromAWSJavaSDK(e); } }
From source file:com.facebook.presto.kinesis.s3config.S3TableConfigClient.java
License:Apache License
/** * Call S3 to get the most recent object list. * * This is an object list request to AWS in the given "directory". * * @return/*from w ww . j a v a2 s .c o m*/ */ protected List<S3ObjectSummary> getObjectSummaries() { AmazonS3Client s3client = this.clientManager.getS3Client(); AmazonS3URI directoryURI = new AmazonS3URI(this.bucketUrl); ArrayList<S3ObjectSummary> returnList = new ArrayList<S3ObjectSummary>(); try { log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey()); ListObjectsRequest req = new ListObjectsRequest().withBucketName(directoryURI.getBucket()) .withPrefix(directoryURI.getKey() + "/").withDelimiter("/").withMaxKeys(25); ObjectListing result; do { result = s3client.listObjects(req); returnList.addAll(result.getObjectSummaries()); req.setMarker(result.getNextMarker()); } while (result.isTruncated()); log.info("Completed getting S3 object listing."); } catch (AmazonServiceException ase) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonServiceException, which means your request made it "); sb.append("to Amazon S3, but was rejected with an error response for some reason.\n"); sb.append("Error Message: " + ase.getMessage()); sb.append("HTTP Status Code: " + ase.getStatusCode()); sb.append("AWS Error Code: " + ase.getErrorCode()); sb.append("Error Type: " + ase.getErrorType()); sb.append("Request ID: " + ase.getRequestId()); log.error(sb.toString(), ase); } catch (AmazonClientException ace) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonClientException, " + "which means the client encountered " + "an internal error while trying to communicate" + " with S3, " + "such as not being able to access the network."); sb.append("Error Message: " + ace.getMessage()); log.error(sb.toString(), ace); } return returnList; }
From source file:com.jeet.s3.AmazonS3ClientWrapper.java
License:Open Source License
public List<String> getListOfFiles(String basepath) { try {//from ww w.j a v a 2s. c om List<String> keys = new ArrayList(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(Constants.BUCKET_NAME); // listObjectsRequest.setDelimiter(basepath); listObjectsRequest.setPrefix(basepath); ObjectListing objectListing; do { objectListing = s3Client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // System.out.println(" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); keys.add(objectSummary.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); return keys; } catch (Exception e) { e.printStackTrace(); } return new ArrayList<>(); }
From source file:com.netflix.ice.common.AwsUtils.java
License:Apache License
/** * List all object summary with given prefix in the s3 bucket. * @param bucket/*from ww w . j a va2s . c om*/ * @param prefix * @return */ public static List<S3ObjectSummary> listAllObjects(String bucket, String prefix) { ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix); List<S3ObjectSummary> result = Lists.newLinkedList(); ObjectListing page = null; do { if (page != null) request.setMarker(page.getNextMarker()); page = s3Client.listObjects(request); result.addAll(page.getObjectSummaries()); } while (page.isTruncated()); return result; }