List of usage examples for com.amazonaws.services.s3.model ListObjectsRequest setBucketName
public void setBucketName(String bucketName)
From source file:c3.ops.priam.aws.S3FileIterator.java
License:Apache License
public S3FileIterator(Provider<AbstractBackupPath> pathProvider, AmazonS3 s3Client, String path, Date start, Date till) {// w w w . j av a 2 s . co m this.start = start; this.till = till; this.pathProvider = pathProvider; ListObjectsRequest listReq = new ListObjectsRequest(); String[] paths = path.split(String.valueOf(S3BackupPath.PATH_SEP)); listReq.setBucketName(paths[0]); listReq.setPrefix(pathProvider.get().remotePrefix(start, till, path)); this.s3Client = s3Client; objectListing = s3Client.listObjects(listReq); iterator = createIterator(); }
From source file:c3.ops.priam.aws.S3PrefixIterator.java
License:Apache License
private void initListing() { ListObjectsRequest listReq = new ListObjectsRequest(); // Get list of tokens listReq.setBucketName(bucket); listReq.setPrefix(clusterPath);/*w w w . j av a 2 s . c om*/ listReq.setDelimiter(String.valueOf(AbstractBackupPath.PATH_SEP)); logger.info("Using cluster prefix for searching tokens: " + clusterPath); objectListing = s3Client.listObjects(listReq); }
From source file:c3.ops.priam.aws.S3PrefixIterator.java
License:Apache License
/** * Check to see if the path exists for the date *///from ww w . j a v a 2 s. c o m private boolean pathExistsForDate(String tprefix, String datestr) { ListObjectsRequest listReq = new ListObjectsRequest(); // Get list of tokens listReq.setBucketName(bucket); listReq.setPrefix(tprefix + datestr); ObjectListing listing; listing = s3Client.listObjects(listReq); if (listing.getObjectSummaries().size() > 0) return true; return false; }
From source file:cloudtrailviewer.components.S3FileChooserDialog.java
License:Open Source License
private void reloadContents() { List<String> tmp = new ArrayList<String>(); this.files.setAll(tmp); String bucketName = PropertiesSingleton.getInstance().getProperty("Bucket"); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(bucketName); listObjectsRequest.setPrefix(prefix); listObjectsRequest.setDelimiter("/"); AWSCredentials credentials = new BasicAWSCredentials(PropertiesSingleton.getInstance().getProperty("Key"), PropertiesSingleton.getInstance().getProperty("Secret")); AmazonS3 s3Client = new AmazonS3Client(credentials); ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); // these are directories List<String> directories = objectListing.getCommonPrefixes(); for (String directory : directories) { tmp.add(stripPrefix(directory)); }// www . j a v a 2 s .c o m // these are files List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries(); for (final S3ObjectSummary objectSummary : objectSummaries) { tmp.add(stripPrefix(objectSummary.getKey())); } this.files.setAll(tmp); }
From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java
License:Open Source License
@Override public ListBucketResponseType listBucket(ListBucketType request) throws S3Exception { ListBucketResponseType reply = request.getReply(); User requestUser = getRequestUser(request); OsgInternalS3Client internalS3Client = null; try {/* ww w . j av a 2s . c om*/ internalS3Client = getS3Client(requestUser); AmazonS3Client s3Client = internalS3Client.getS3Client(); ListObjectsRequest listRequest = new ListObjectsRequest(); listRequest.setBucketName(request.getBucket()); listRequest.setDelimiter(Strings.isNullOrEmpty(request.getDelimiter()) ? null : request.getDelimiter()); listRequest.setMarker(Strings.isNullOrEmpty(request.getMarker()) ? null : request.getMarker()); listRequest.setMaxKeys((request.getMaxKeys() == null ? null : Integer.parseInt(request.getMaxKeys()))); listRequest.setPrefix(Strings.isNullOrEmpty(request.getPrefix()) ? null : request.getPrefix()); ObjectListing response = s3Client.listObjects(listRequest); /* Non-optional, must have non-null values */ reply.setName(request.getBucket()); reply.setMaxKeys(response.getMaxKeys()); reply.setMarker(response.getMarker() == null ? "" : response.getMarker()); reply.setPrefix(response.getPrefix() == null ? "" : response.getPrefix()); reply.setIsTruncated(response.isTruncated()); /* Optional */ reply.setNextMarker(response.getNextMarker()); reply.setDelimiter(response.getDelimiter()); if (reply.getContents() == null) { reply.setContents(new ArrayList<ListEntry>()); } if (reply.getCommonPrefixesList() == null) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); } for (S3ObjectSummary obj : response.getObjectSummaries()) { //Add entry, note that the canonical user is set based on requesting user, not returned user reply.getContents() .add(new ListEntry(obj.getKey(), DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(), obj.getSize(), getCanonicalUser(requestUser), obj.getStorageClass())); } if (response.getCommonPrefixes() != null && response.getCommonPrefixes().size() > 0) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); for (String s : response.getCommonPrefixes()) { reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s)); } } return reply; } catch (AmazonServiceException e) { LOG.debug("Error from backend", e); throw S3ExceptionMapper.fromAWSJavaSDK(e); } }
From source file:com.haskins.cloudtrailviewer.dialog.s3filechooser.S3FileList.java
License:Open Source License
private ObjectListing s3ListObjects(String pathPrefix, String delimiter) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(currentAccount.getBucket()); listObjectsRequest.setPrefix(pathPrefix); if (delimiter != null) { listObjectsRequest.setDelimiter(delimiter); }/*from w w w. j a va 2 s. c o m*/ AmazonS3 s3Client = getS3Client(); return s3Client.listObjects(listObjectsRequest); }
From source file:com.haskins.cloudtrailviewer.dialog.S3FileChooser.java
License:Open Source License
private void reloadContents() { loadingLabel.setVisible(true);// w w w. jav a 2 s .c o m this.s3ListModel.clear(); String bucketName = currentAccount.getBucket(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(bucketName); listObjectsRequest.setPrefix(prefix); listObjectsRequest.setDelimiter("/"); AWSCredentials credentials = new BasicAWSCredentials(currentAccount.getKey(), currentAccount.getSecret()); AmazonS3 s3Client = new AmazonS3Client(credentials); try { ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); // Add .. if not at root if (prefix.trim().length() != 0) { S3ListModel model = new S3ListModel(MOVE_BACK, MOVE_BACK, S3ListModel.FILE_BACK); this.s3ListModel.addElement(model); } // these are directories List<String> directories = objectListing.getCommonPrefixes(); for (String directory : directories) { String dir = stripPrefix(directory); int lastSlash = dir.lastIndexOf("/"); String strippeDir = dir.substring(0, lastSlash); String alias = dir; if (isAccountNumber(strippeDir)) { if (aliasMap.containsKey(strippeDir)) { alias = aliasMap.get(strippeDir); } } S3ListModel model = new S3ListModel(dir, alias, S3ListModel.FILE_DIR); this.s3ListModel.addElement(model); } // these are files List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries(); for (final S3ObjectSummary objectSummary : objectSummaries) { String file = stripPrefix(objectSummary.getKey()); S3ListModel model = new S3ListModel(file, file, S3ListModel.FILE_DOC); this.s3ListModel.addElement(model); } loadingLabel.setVisible(false); } catch (Exception e) { e.printStackTrace(); } }
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
@Override public FileStatus getFileStatus(String hostName, Path path, String msg) throws IOException, FileNotFoundException { FileStatus res = null;//from w w w. j av a 2s . c om FileStatus cached = memoryCache.getFileStatus(path.toString()); if (cached != null) { return cached; } LOG.trace("getFileStatus(start) for {}, hostname: {}", path, hostName); /* * The requested path is equal to hostName. HostName is equal to * hostNameScheme, thus the container. Therefore we have no object to look * for and we return the FileStatus as a directory. Containers have to * lastModified. */ if (path.toString().equals(hostName) || (path.toString().length() + 1 == hostName.length())) { LOG.trace("getFileStatus(completed) {}", path); res = new FileStatus(0L, true, 1, mBlockSize, 0L, path); memoryCache.putFileStatus(path.toString(), res); return res; } if (path.toString().contains(HADOOP_TEMPORARY)) { LOG.debug("getFileStatus on temp object {}. Return not found", path.toString()); throw new FileNotFoundException("Not found " + path.toString()); } String key = pathToKey(hostName, path); LOG.debug("getFileStatus: on original key {}", key); try { FileStatus fileStatus = null; try { fileStatus = getFileStatusKeyBased(key, path); } catch (AmazonS3Exception e) { if (e.getStatusCode() != 404) { throw new IOException(e); } } if (fileStatus != null) { LOG.trace("getFileStatus(completed) {}", path); memoryCache.putFileStatus(path.toString(), fileStatus); return fileStatus; } // means key returned not found. Trying to call get file status on key/ // probably not needed this call if (!key.endsWith("/")) { String newKey = key + "/"; try { LOG.debug("getFileStatus: original key not found. Alternative key {}", key); fileStatus = getFileStatusKeyBased(newKey, path); } catch (AmazonS3Exception e) { if (e.getStatusCode() != 404) { throw new IOException(e); } } if (fileStatus != null) { LOG.trace("getFileStatus(completed) {}", path); memoryCache.putFileStatus(path.toString(), fileStatus); return fileStatus; } else { // if here: both key and key/ returned not found. // trying to see if pseudo directory of the form // a/b/key/d/e (a/b/key/ doesn't exists by itself) // perform listing on the key LOG.debug("getFileStatus: Modifined key {} not found. Trying to lisr", key); key = maybeAddTrailingSlash(key); ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(mBucket); request.setPrefix(key); request.setDelimiter("/"); request.setMaxKeys(1); ObjectListing objects = mClient.listObjects(request); if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) { LOG.trace("getFileStatus(completed) {}", path); res = new FileStatus(0, true, 1, 0, 0, path); memoryCache.putFileStatus(path.toString(), res); return res; } else if (key.isEmpty()) { LOG.debug("Found root directory"); LOG.trace("getFileStatus(completed) {}", path); res = new FileStatus(0, true, 1, 0, 0, path); memoryCache.putFileStatus(path.toString(), res); return res; } } } } catch (AmazonS3Exception e) { if (e.getStatusCode() == 403) { throw new IOException(e); } } catch (Exception e) { LOG.debug("Not found {}", path.toString()); LOG.warn(e.getMessage()); throw new FileNotFoundException("Not found " + path.toString()); } throw new FileNotFoundException("Not found " + path.toString()); }
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
/** * {@inheritDoc}//from w w w . java 2s. c o m * * Prefix based * Return everything that starts with the prefix * Fill listing * Return all objects, even zero size * If fileStatus is null means the path is part of some name, neither object * or pseudo directory. Was called by Globber * * @param hostName hostName * @param path path * @param fullListing Return all objects, even zero size * @param prefixBased Return everything that starts with the prefix * @return list * @throws IOException if error */ /* public FileStatus[] list(String hostName, Path path, boolean fullListing, boolean prefixBased) throws IOException { String key = pathToKey(hostName, path); ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>(); ListObjectsRequest request = new ListObjectsRequest().withBucketName(mBucket).withPrefix(key); String curObj; if (path.toString().equals(mBucket)) { curObj = ""; } else if (path.toString().startsWith(mBucket + "/")) { curObj = path.toString().substring(mBucket.length() + 1); } else if (path.toString().startsWith(hostName)) { curObj = path.toString().substring(hostName.length()); } else { curObj = path.toString(); } ObjectListing objectList = mClient.listObjects(request); List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries(); if (objectSummaries.size() == 0) { FileStatus[] emptyRes = {}; LOG.debug("List for bucket {} is empty", mBucket); return emptyRes; } boolean objectScanContinue = true; S3ObjectSummary prevObj = null; while (objectScanContinue) { for (S3ObjectSummary obj : objectSummaries) { if (prevObj == null) { prevObj = obj; continue; } String objKey = obj.getKey(); String unifiedObjectName = extractUnifiedObjectName(objKey); if (!prefixBased && !curObj.equals("") && !path.toString().endsWith("/") && !unifiedObjectName.equals(curObj) && !unifiedObjectName.startsWith(curObj + "/")) { LOG.trace("{} does not match {}. Skipped", unifiedObjectName, curObj); continue; } if (isSparkOrigin(unifiedObjectName) && !fullListing) { LOG.trace("{} created by Spark", unifiedObjectName); if (!isJobSuccessful(unifiedObjectName)) { LOG.trace("{} created by failed Spark job. Skipped", unifiedObjectName); if (fModeAutomaticDelete) { delete(hostName, new Path(objKey), true); } continue; } else { // if we here - data created by spark and job completed // successfully // however there be might parts of failed tasks that // were not aborted // we need to make sure there are no failed attempts if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) { // found failed that was not aborted. LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey); if (prevObj.getSize() < obj.getSize()) { LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey()); prevObj = obj; } continue; } } } if (prevObj.getSize() > 0 || fullListing) { FileStatus fs = getFileStatusObjSummaryBased(prevObj, hostName, path); tmpResult.add(fs); } prevObj = obj; } boolean isTruncated = objectList.isTruncated(); if (isTruncated) { objectList = mClient.listNextBatchOfObjects(objectList); objectSummaries = objectList.getObjectSummaries(); } else { objectScanContinue = false; } } if (prevObj != null && (prevObj.getSize() > 0 || fullListing)) { FileStatus fs = getFileStatusObjSummaryBased(prevObj, hostName, path); tmpResult.add(fs); } if (LOG.isTraceEnabled()) { LOG.trace("COS List to return length {}", tmpResult.size()); for (FileStatus fs: tmpResult) { LOG.trace("{}", fs.getPath()); } } return tmpResult.toArray(new FileStatus[tmpResult.size()]); } */ @Override public FileStatus[] list(String hostName, Path path, boolean fullListing, boolean prefixBased, Boolean isDirectory, boolean flatListing, PathFilter filter) throws FileNotFoundException, IOException { LOG.debug("Native direct list status for {}", path); ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>(); String key = pathToKey(hostName, path); if (isDirectory != null && isDirectory.booleanValue() && !key.endsWith("/")) { key = key + "/"; LOG.debug("listNativeDirect modify key to {}", key); } Map<String, FileStatus> emptyObjects = new HashMap<String, FileStatus>(); ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(mBucket); request.setMaxKeys(5000); request.setPrefix(key); if (!flatListing) { request.setDelimiter("/"); } ObjectListing objectList = mClient.listObjects(request); List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries(); List<String> commonPrefixes = objectList.getCommonPrefixes(); boolean objectScanContinue = true; S3ObjectSummary prevObj = null; // start FTA logic boolean stocatorOrigin = isSparkOrigin(key, path.toString()); if (stocatorOrigin) { LOG.debug("Stocator origin is true for {}", key); if (!isJobSuccessful(key)) { LOG.debug("{} created by failed Spark job. Skipped", key); if (fModeAutomaticDelete) { delete(hostName, new Path(key), true); } return new FileStatus[0]; } } while (objectScanContinue) { for (S3ObjectSummary obj : objectSummaries) { if (prevObj == null) { prevObj = obj; continue; } String objKey = obj.getKey(); String unifiedObjectName = extractUnifiedObjectName(objKey); LOG.debug("list candidate {}, unified name {}", objKey, unifiedObjectName); if (stocatorOrigin && !fullListing) { LOG.trace("{} created by Spark", unifiedObjectName); // if we here - data created by spark and job completed // successfully // however there be might parts of failed tasks that // were not aborted // we need to make sure there are no failed attempts if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) { // found failed that was not aborted. LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey); if (prevObj.getSize() < obj.getSize()) { LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey()); prevObj = obj; } continue; } } FileStatus fs = createFileStatus(prevObj, hostName, path); if (fs.getLen() > 0 || fullListing) { LOG.debug("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen()); if (filter == null) { tmpResult.add(fs); } else if (filter != null && filter.accept(fs.getPath())) { tmpResult.add(fs); } else { LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter); } } else { emptyObjects.put(fs.getPath().toString(), fs); } prevObj = obj; } boolean isTruncated = objectList.isTruncated(); if (isTruncated) { objectList = mClient.listNextBatchOfObjects(objectList); objectSummaries = objectList.getObjectSummaries(); } else { objectScanContinue = false; } } if (prevObj != null) { FileStatus fs = createFileStatus(prevObj, hostName, path); LOG.debug("Adding the last object from the list {}", fs.getPath()); if (fs.getLen() > 0 || fullListing) { LOG.debug("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen()); if (filter == null) { memoryCache.putFileStatus(fs.getPath().toString(), fs); tmpResult.add(fs); } else if (filter != null && filter.accept(fs.getPath())) { memoryCache.putFileStatus(fs.getPath().toString(), fs); tmpResult.add(fs); } else { LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter); } } else if (!fs.getPath().getName().equals(HADOOP_SUCCESS)) { emptyObjects.put(fs.getPath().toString(), fs); } } // get common prefixes for (String comPrefix : commonPrefixes) { LOG.debug("Common prefix is {}", comPrefix); if (emptyObjects.containsKey(keyToQualifiedPath(hostName, comPrefix).toString()) || emptyObjects.isEmpty()) { FileStatus status = new COSFileStatus(true, false, keyToQualifiedPath(hostName, comPrefix)); LOG.debug("Match between common prefix and empty object {}. Adding to result", comPrefix); if (filter == null) { memoryCache.putFileStatus(status.getPath().toString(), status); tmpResult.add(status); } else if (filter != null && filter.accept(status.getPath())) { memoryCache.putFileStatus(status.getPath().toString(), status); tmpResult.add(status); } else { LOG.trace("Common prefix {} rejected by path filter during list. Filter {}", status.getPath(), filter); } } } return tmpResult.toArray(new FileStatus[tmpResult.size()]); }
From source file:com.jeet.s3.AmazonS3ClientWrapper.java
License:Open Source License
public List<String> getListOfFiles(String basepath) { try {//from ww w . ja v a2 s .c o m List<String> keys = new ArrayList(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(Constants.BUCKET_NAME); // listObjectsRequest.setDelimiter(basepath); listObjectsRequest.setPrefix(basepath); ObjectListing objectListing; do { objectListing = s3Client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // System.out.println(" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); keys.add(objectSummary.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); return keys; } catch (Exception e) { e.printStackTrace(); } return new ArrayList<>(); }