List of usage examples for com.amazonaws.services.s3.model ListObjectsRequest setMaxKeys
public void setMaxKeys(Integer maxKeys)
maxKeys parameter indicating the maximum number of keys to include in the response. From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java
License:Open Source License
@Override public ListBucketResponseType listBucket(ListBucketType request) throws S3Exception { ListBucketResponseType reply = request.getReply(); User requestUser = getRequestUser(request); OsgInternalS3Client internalS3Client = null; try {// w ww . j a va2 s . c o m internalS3Client = getS3Client(requestUser); AmazonS3Client s3Client = internalS3Client.getS3Client(); ListObjectsRequest listRequest = new ListObjectsRequest(); listRequest.setBucketName(request.getBucket()); listRequest.setDelimiter(Strings.isNullOrEmpty(request.getDelimiter()) ? null : request.getDelimiter()); listRequest.setMarker(Strings.isNullOrEmpty(request.getMarker()) ? null : request.getMarker()); listRequest.setMaxKeys((request.getMaxKeys() == null ? null : Integer.parseInt(request.getMaxKeys()))); listRequest.setPrefix(Strings.isNullOrEmpty(request.getPrefix()) ? null : request.getPrefix()); ObjectListing response = s3Client.listObjects(listRequest); /* Non-optional, must have non-null values */ reply.setName(request.getBucket()); reply.setMaxKeys(response.getMaxKeys()); reply.setMarker(response.getMarker() == null ? "" : response.getMarker()); reply.setPrefix(response.getPrefix() == null ? "" : response.getPrefix()); reply.setIsTruncated(response.isTruncated()); /* Optional */ reply.setNextMarker(response.getNextMarker()); reply.setDelimiter(response.getDelimiter()); if (reply.getContents() == null) { reply.setContents(new ArrayList<ListEntry>()); } if (reply.getCommonPrefixesList() == null) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); } for (S3ObjectSummary obj : response.getObjectSummaries()) { //Add entry, note that the canonical user is set based on requesting user, not returned user reply.getContents() .add(new ListEntry(obj.getKey(), DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(), obj.getSize(), getCanonicalUser(requestUser), obj.getStorageClass())); } if (response.getCommonPrefixes() != null && response.getCommonPrefixes().size() > 0) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); for (String s : response.getCommonPrefixes()) { reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s)); } } return reply; } catch (AmazonServiceException e) { LOG.debug("Error from backend", e); throw S3ExceptionMapper.fromAWSJavaSDK(e); } }
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
@Override public FileStatus getFileStatus(String hostName, Path path, String msg) throws IOException, FileNotFoundException { FileStatus res = null;/* www . j a va2s .com*/ FileStatus cached = memoryCache.getFileStatus(path.toString()); if (cached != null) { return cached; } LOG.trace("getFileStatus(start) for {}, hostname: {}", path, hostName); /* * The requested path is equal to hostName. HostName is equal to * hostNameScheme, thus the container. Therefore we have no object to look * for and we return the FileStatus as a directory. Containers have to * lastModified. */ if (path.toString().equals(hostName) || (path.toString().length() + 1 == hostName.length())) { LOG.trace("getFileStatus(completed) {}", path); res = new FileStatus(0L, true, 1, mBlockSize, 0L, path); memoryCache.putFileStatus(path.toString(), res); return res; } if (path.toString().contains(HADOOP_TEMPORARY)) { LOG.debug("getFileStatus on temp object {}. Return not found", path.toString()); throw new FileNotFoundException("Not found " + path.toString()); } String key = pathToKey(hostName, path); LOG.debug("getFileStatus: on original key {}", key); try { FileStatus fileStatus = null; try { fileStatus = getFileStatusKeyBased(key, path); } catch (AmazonS3Exception e) { if (e.getStatusCode() != 404) { throw new IOException(e); } } if (fileStatus != null) { LOG.trace("getFileStatus(completed) {}", path); memoryCache.putFileStatus(path.toString(), fileStatus); return fileStatus; } // means key returned not found. Trying to call get file status on key/ // probably not needed this call if (!key.endsWith("/")) { String newKey = key + "/"; try { LOG.debug("getFileStatus: original key not found. Alternative key {}", key); fileStatus = getFileStatusKeyBased(newKey, path); } catch (AmazonS3Exception e) { if (e.getStatusCode() != 404) { throw new IOException(e); } } if (fileStatus != null) { LOG.trace("getFileStatus(completed) {}", path); memoryCache.putFileStatus(path.toString(), fileStatus); return fileStatus; } else { // if here: both key and key/ returned not found. // trying to see if pseudo directory of the form // a/b/key/d/e (a/b/key/ doesn't exists by itself) // perform listing on the key LOG.debug("getFileStatus: Modifined key {} not found. Trying to lisr", key); key = maybeAddTrailingSlash(key); ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(mBucket); request.setPrefix(key); request.setDelimiter("/"); request.setMaxKeys(1); ObjectListing objects = mClient.listObjects(request); if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) { LOG.trace("getFileStatus(completed) {}", path); res = new FileStatus(0, true, 1, 0, 0, path); memoryCache.putFileStatus(path.toString(), res); return res; } else if (key.isEmpty()) { LOG.debug("Found root directory"); LOG.trace("getFileStatus(completed) {}", path); res = new FileStatus(0, true, 1, 0, 0, path); memoryCache.putFileStatus(path.toString(), res); return res; } } } } catch (AmazonS3Exception e) { if (e.getStatusCode() == 403) { throw new IOException(e); } } catch (Exception e) { LOG.debug("Not found {}", path.toString()); LOG.warn(e.getMessage()); throw new FileNotFoundException("Not found " + path.toString()); } throw new FileNotFoundException("Not found " + path.toString()); }
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
/** * {@inheritDoc}/*from w ww . j a va 2 s . co m*/ * * Prefix based * Return everything that starts with the prefix * Fill listing * Return all objects, even zero size * If fileStatus is null means the path is part of some name, neither object * or pseudo directory. Was called by Globber * * @param hostName hostName * @param path path * @param fullListing Return all objects, even zero size * @param prefixBased Return everything that starts with the prefix * @return list * @throws IOException if error */ /* public FileStatus[] list(String hostName, Path path, boolean fullListing, boolean prefixBased) throws IOException { String key = pathToKey(hostName, path); ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>(); ListObjectsRequest request = new ListObjectsRequest().withBucketName(mBucket).withPrefix(key); String curObj; if (path.toString().equals(mBucket)) { curObj = ""; } else if (path.toString().startsWith(mBucket + "/")) { curObj = path.toString().substring(mBucket.length() + 1); } else if (path.toString().startsWith(hostName)) { curObj = path.toString().substring(hostName.length()); } else { curObj = path.toString(); } ObjectListing objectList = mClient.listObjects(request); List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries(); if (objectSummaries.size() == 0) { FileStatus[] emptyRes = {}; LOG.debug("List for bucket {} is empty", mBucket); return emptyRes; } boolean objectScanContinue = true; S3ObjectSummary prevObj = null; while (objectScanContinue) { for (S3ObjectSummary obj : objectSummaries) { if (prevObj == null) { prevObj = obj; continue; } String objKey = obj.getKey(); String unifiedObjectName = extractUnifiedObjectName(objKey); if (!prefixBased && !curObj.equals("") && !path.toString().endsWith("/") && !unifiedObjectName.equals(curObj) && !unifiedObjectName.startsWith(curObj + "/")) { LOG.trace("{} does not match {}. Skipped", unifiedObjectName, curObj); continue; } if (isSparkOrigin(unifiedObjectName) && !fullListing) { LOG.trace("{} created by Spark", unifiedObjectName); if (!isJobSuccessful(unifiedObjectName)) { LOG.trace("{} created by failed Spark job. Skipped", unifiedObjectName); if (fModeAutomaticDelete) { delete(hostName, new Path(objKey), true); } continue; } else { // if we here - data created by spark and job completed // successfully // however there be might parts of failed tasks that // were not aborted // we need to make sure there are no failed attempts if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) { // found failed that was not aborted. LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey); if (prevObj.getSize() < obj.getSize()) { LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey()); prevObj = obj; } continue; } } } if (prevObj.getSize() > 0 || fullListing) { FileStatus fs = getFileStatusObjSummaryBased(prevObj, hostName, path); tmpResult.add(fs); } prevObj = obj; } boolean isTruncated = objectList.isTruncated(); if (isTruncated) { objectList = mClient.listNextBatchOfObjects(objectList); objectSummaries = objectList.getObjectSummaries(); } else { objectScanContinue = false; } } if (prevObj != null && (prevObj.getSize() > 0 || fullListing)) { FileStatus fs = getFileStatusObjSummaryBased(prevObj, hostName, path); tmpResult.add(fs); } if (LOG.isTraceEnabled()) { LOG.trace("COS List to return length {}", tmpResult.size()); for (FileStatus fs: tmpResult) { LOG.trace("{}", fs.getPath()); } } return tmpResult.toArray(new FileStatus[tmpResult.size()]); } */ @Override public FileStatus[] list(String hostName, Path path, boolean fullListing, boolean prefixBased, Boolean isDirectory, boolean flatListing, PathFilter filter) throws FileNotFoundException, IOException { LOG.debug("Native direct list status for {}", path); ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>(); String key = pathToKey(hostName, path); if (isDirectory != null && isDirectory.booleanValue() && !key.endsWith("/")) { key = key + "/"; LOG.debug("listNativeDirect modify key to {}", key); } Map<String, FileStatus> emptyObjects = new HashMap<String, FileStatus>(); ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(mBucket); request.setMaxKeys(5000); request.setPrefix(key); if (!flatListing) { request.setDelimiter("/"); } ObjectListing objectList = mClient.listObjects(request); List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries(); List<String> commonPrefixes = objectList.getCommonPrefixes(); boolean objectScanContinue = true; S3ObjectSummary prevObj = null; // start FTA logic boolean stocatorOrigin = isSparkOrigin(key, path.toString()); if (stocatorOrigin) { LOG.debug("Stocator origin is true for {}", key); if (!isJobSuccessful(key)) { LOG.debug("{} created by failed Spark job. Skipped", key); if (fModeAutomaticDelete) { delete(hostName, new Path(key), true); } return new FileStatus[0]; } } while (objectScanContinue) { for (S3ObjectSummary obj : objectSummaries) { if (prevObj == null) { prevObj = obj; continue; } String objKey = obj.getKey(); String unifiedObjectName = extractUnifiedObjectName(objKey); LOG.debug("list candidate {}, unified name {}", objKey, unifiedObjectName); if (stocatorOrigin && !fullListing) { LOG.trace("{} created by Spark", unifiedObjectName); // if we here - data created by spark and job completed // successfully // however there be might parts of failed tasks that // were not aborted // we need to make sure there are no failed attempts if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) { // found failed that was not aborted. LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey); if (prevObj.getSize() < obj.getSize()) { LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey()); prevObj = obj; } continue; } } FileStatus fs = createFileStatus(prevObj, hostName, path); if (fs.getLen() > 0 || fullListing) { LOG.debug("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen()); if (filter == null) { tmpResult.add(fs); } else if (filter != null && filter.accept(fs.getPath())) { tmpResult.add(fs); } else { LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter); } } else { emptyObjects.put(fs.getPath().toString(), fs); } prevObj = obj; } boolean isTruncated = objectList.isTruncated(); if (isTruncated) { objectList = mClient.listNextBatchOfObjects(objectList); objectSummaries = objectList.getObjectSummaries(); } else { objectScanContinue = false; } } if (prevObj != null) { FileStatus fs = createFileStatus(prevObj, hostName, path); LOG.debug("Adding the last object from the list {}", fs.getPath()); if (fs.getLen() > 0 || fullListing) { LOG.debug("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen()); if (filter == null) { memoryCache.putFileStatus(fs.getPath().toString(), fs); tmpResult.add(fs); } else if (filter != null && filter.accept(fs.getPath())) { memoryCache.putFileStatus(fs.getPath().toString(), fs); tmpResult.add(fs); } else { LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter); } } else if (!fs.getPath().getName().equals(HADOOP_SUCCESS)) { emptyObjects.put(fs.getPath().toString(), fs); } } // get common prefixes for (String comPrefix : commonPrefixes) { LOG.debug("Common prefix is {}", comPrefix); if (emptyObjects.containsKey(keyToQualifiedPath(hostName, comPrefix).toString()) || emptyObjects.isEmpty()) { FileStatus status = new COSFileStatus(true, false, keyToQualifiedPath(hostName, comPrefix)); LOG.debug("Match between common prefix and empty object {}. Adding to result", comPrefix); if (filter == null) { memoryCache.putFileStatus(status.getPath().toString(), status); tmpResult.add(status); } else if (filter != null && filter.accept(status.getPath())) { memoryCache.putFileStatus(status.getPath().toString(), status); tmpResult.add(status); } else { LOG.trace("Common prefix {} rejected by path filter during list. Filter {}", status.getPath(), filter); } } } return tmpResult.toArray(new FileStatus[tmpResult.size()]); }
From source file:com.upplication.s3fs.util.S3ObjectSummaryLookup.java
License:Open Source License
/** * Get the {@link com.amazonaws.services.s3.model.S3ObjectSummary} that represent this Path or her first child if this path not exists * @param s3Path {@link com.upplication.s3fs.S3Path} * @return {@link com.amazonaws.services.s3.model.S3ObjectSummary} * @throws java.nio.file.NoSuchFileException if not found the path and any child */// w w w . jav a 2 s . c om public S3ObjectSummary lookup(S3Path s3Path) throws NoSuchFileException { /* * check is object summary has been cached */ S3ObjectSummary summary = s3Path.fetchObjectSummary(); if (summary != null) { return summary; } final AmazonS3Client client = s3Path.getFileSystem().getClient(); /* * when `key` is an empty string retrieve the object meta-data of the bucket */ if ("".equals(s3Path.getKey())) { ObjectMetadata meta = client.getObjectMetadata(s3Path.getBucket(), ""); if (meta == null) throw new NoSuchFileException("s3://" + s3Path.getBucket()); summary = new S3ObjectSummary(); summary.setBucketName(s3Path.getBucket()); summary.setETag(meta.getETag()); summary.setKey(s3Path.getKey()); summary.setLastModified(meta.getLastModified()); summary.setSize(meta.getContentLength()); // TODO summary.setOwner(?); // TODO summary.setStorageClass(?); return summary; } /* * Lookup for the object summary for the specified object key * by using a `listObjects` request */ String marker = null; while (true) { ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(s3Path.getBucket()); request.setPrefix(s3Path.getKey()); request.setMaxKeys(250); if (marker != null) request.setMarker(marker); ObjectListing listing = client.listObjects(request); List<S3ObjectSummary> results = listing.getObjectSummaries(); if (results.isEmpty()) { break; } for (S3ObjectSummary item : results) { if (matchName(s3Path.getKey(), item)) { return item; } } if (listing.isTruncated()) marker = listing.getNextMarker(); else break; } throw new NoSuchFileException("s3://" + s3Path.getBucket() + "/" + s3Path.toString()); }
From source file:io.konig.camel.aws.s3.DeleteObjectConsumer.java
License:Apache License
@Override protected int poll() throws Exception { // must reset for each poll shutdownRunningTask = null;//from w w w. ja va 2s . c om pendingExchanges = 0; String fileName = getConfiguration().getFileName(); String bucketName = getConfiguration().getBucketName(); Queue<Exchange> exchanges; if (fileName != null) { LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName); S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName)); exchanges = createExchanges(s3Object); } else { LOG.trace("Queueing objects in bucket [{}]...", bucketName); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(bucketName); listObjectsRequest.setPrefix(getConfiguration().getPrefix()); if (maxMessagesPerPoll > 0) { listObjectsRequest.setMaxKeys(maxMessagesPerPoll); } // if there was a marker from previous poll then use that to continue from where we left last time if (marker != null) { LOG.trace("Resuming from marker: {}", marker); listObjectsRequest.setMarker(marker); } ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest); if (listObjects.isTruncated()) { marker = listObjects.getNextMarker(); LOG.trace("Returned list is truncated, so setting next marker: {}", marker); } else { // no more data so clear marker marker = null; } if (LOG.isTraceEnabled()) { LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(), bucketName); } exchanges = createExchanges(listObjects.getObjectSummaries()); } return processBatch(CastUtils.cast(exchanges)); }
From source file:org.apache.camel.component.aws.s3.S3Consumer.java
License:Apache License
@Override protected int poll() throws Exception { // must reset for each poll shutdownRunningTask = null;//from w w w .j av a 2 s. c o m pendingExchanges = 0; String fileName = getConfiguration().getFileName(); String bucketName = getConfiguration().getBucketName(); Queue<Exchange> exchanges = null; if (fileName != null) { LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName); S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName)); exchanges = createExchanges(s3Object); } else { LOG.trace("Queueing objects in bucket [{}]...", bucketName); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(bucketName); listObjectsRequest.setPrefix(getConfiguration().getPrefix()); listObjectsRequest.setMaxKeys(maxMessagesPerPoll); ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest); if (LOG.isTraceEnabled()) { LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(), bucketName); } exchanges = createExchanges(listObjects.getObjectSummaries()); } return processBatch(CastUtils.cast(exchanges)); }
From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java
License:Apache License
/** * Renames Path src to Path dst. Can take place on local fs * or remote DFS./* ww w . j a va 2s. co m*/ * * Warning: S3 does not support renames. This method does a copy which can take S3 some time to execute with large * files and directories. Since there is no Progressable passed in, this can time out jobs. * * Note: This implementation differs with other S3 drivers. Specifically: * Fails if src is a file and dst is a directory. * Fails if src is a directory and dst is a file. * Fails if the parent of dst does not exist or is a file. * Fails if dst is a directory that is not empty. * * @param src path to be renamed * @param dst new path after rename * @throws IOException on failure * @return true if rename is successful */ public boolean rename(Path src, Path dst) throws IOException { LOG.info("Rename path " + src + " to " + dst); String srcKey = pathToKey(src); String dstKey = pathToKey(dst); if (srcKey.length() == 0 || dstKey.length() == 0) { LOG.info("rename: src or dst are empty"); return false; } if (srcKey.equals(dstKey)) { LOG.info("rename: src and dst refer to the same file"); return true; } S3AFileStatus srcStatus; try { srcStatus = getFileStatus(src); } catch (FileNotFoundException e) { LOG.info("rename: src not found " + src); return false; } S3AFileStatus dstStatus = null; try { dstStatus = getFileStatus(dst); if (srcStatus.isFile() && dstStatus.isDirectory()) { LOG.info("rename: src is a file and dst is a directory"); return false; } if (srcStatus.isDirectory() && dstStatus.isFile()) { LOG.info("rename: src is a directory and dst is a file"); return false; } if (dstStatus.isDirectory() && !dstStatus.isEmptyDirectory()) { return false; } } catch (FileNotFoundException e) { // Parent must exist Path parent = dst.getParent(); if (!pathToKey(parent).isEmpty()) { try { S3AFileStatus dstParentStatus = getFileStatus(dst.getParent()); if (!dstParentStatus.isDirectory()) { return false; } } catch (FileNotFoundException e2) { return false; } } } // Ok! Time to start if (srcStatus.isFile()) { if (LOG.isDebugEnabled()) { LOG.debug("rename: renaming file " + src + " to " + dst); } copyFile(srcKey, dstKey); delete(src, false); } else { if (LOG.isDebugEnabled()) { LOG.debug("rename: renaming directory " + src + " to " + dst); } // This is a directory to directory copy if (!dstKey.endsWith("/")) { dstKey = dstKey + "/"; } if (!srcKey.endsWith("/")) { srcKey = srcKey + "/"; } List<DeleteObjectsRequest.KeyVersion> keysToDelete = new ArrayList<DeleteObjectsRequest.KeyVersion>(); if (dstStatus != null && dstStatus.isEmptyDirectory()) { copyFile(srcKey, dstKey); statistics.incrementWriteOps(1); keysToDelete.add(new DeleteObjectsRequest.KeyVersion(srcKey)); } ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(bucket); request.setPrefix(srcKey); request.setMaxKeys(maxKeys); ObjectListing objects = s3.listObjects(request); statistics.incrementReadOps(1); while (true) { for (S3ObjectSummary summary : objects.getObjectSummaries()) { keysToDelete.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); String newDstKey = dstKey + summary.getKey().substring(srcKey.length()); copyFile(summary.getKey(), newDstKey); if (keysToDelete.size() == MAX_ENTRIES_TO_DELETE) { DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket) .withKeys(keysToDelete); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1); keysToDelete.clear(); } } if (objects.isTruncated()) { objects = s3.listNextBatchOfObjects(objects); statistics.incrementReadOps(1); } else { break; } } if (!keysToDelete.isEmpty()) { DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket); deleteRequest.setKeys(keysToDelete); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1); } } if (src.getParent() != dst.getParent()) { deleteUnnecessaryFakeDirectories(dst.getParent()); createFakeDirectoryIfNecessary(src.getParent()); } return true; }
From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java
License:Apache License
/** Delete a file. * * @param f the path to delete.//from w w w . j a va2s . com * @param recursive if path is a directory and set to * true, the directory is deleted else throws an exception. In * case of a file the recursive can be set to either true or false. * @return true if delete is successful else false. * @throws IOException */ public boolean delete(Path f, boolean recursive) throws IOException { LOG.info("Delete path " + f + " - recursive " + recursive); S3AFileStatus status; try { status = getFileStatus(f); } catch (FileNotFoundException e) { if (LOG.isDebugEnabled()) { LOG.debug("Couldn't delete " + f + " - does not exist"); } return false; } String key = pathToKey(f); if (status.isDirectory()) { if (LOG.isDebugEnabled()) { LOG.debug("delete: Path is a directory"); } if (!recursive) { throw new IOException("Path is a folder: " + f); } if (!key.endsWith("/")) { key = key + "/"; } if (status.isEmptyDirectory()) { if (LOG.isDebugEnabled()) { LOG.debug("Deleting fake empty directory"); } s3.deleteObject(bucket, key); statistics.incrementWriteOps(1); } else { if (LOG.isDebugEnabled()) { LOG.debug("Getting objects for directory prefix " + key + " to delete"); } ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(bucket); request.setPrefix(key); // Hopefully not setting a delimiter will cause this to find everything //request.setDelimiter("/"); request.setMaxKeys(maxKeys); List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<DeleteObjectsRequest.KeyVersion>(); ObjectListing objects = s3.listObjects(request); statistics.incrementReadOps(1); while (true) { for (S3ObjectSummary summary : objects.getObjectSummaries()) { keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); if (LOG.isDebugEnabled()) { LOG.debug("Got object to delete " + summary.getKey()); } if (keys.size() == MAX_ENTRIES_TO_DELETE) { DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1); keys.clear(); } } if (objects.isTruncated()) { objects = s3.listNextBatchOfObjects(objects); statistics.incrementReadOps(1); } else { break; } } if (!keys.isEmpty()) { DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1); } } } else { if (LOG.isDebugEnabled()) { LOG.debug("delete: Path is a file"); } s3.deleteObject(bucket, key); statistics.incrementWriteOps(1); } createFakeDirectoryIfNecessary(f.getParent()); return true; }
From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java
License:Apache License
/** * List the statuses of the files/directories in the given path if the path is * a directory./* ww w.ja v a 2 s . c om*/ * * @param f given path * @return the statuses of the files/directories in the given patch * @throws FileNotFoundException when the path does not exist; * IOException see specific implementation */ public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException { String key = pathToKey(f); LOG.info("List status for path: " + f); final List<FileStatus> result = new ArrayList<FileStatus>(); final FileStatus fileStatus = getFileStatus(f); if (fileStatus.isDirectory()) { if (!key.isEmpty()) { key = key + "/"; } ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(bucket); request.setPrefix(key); request.setDelimiter("/"); request.setMaxKeys(maxKeys); if (LOG.isDebugEnabled()) { LOG.debug("listStatus: doing listObjects for directory " + key); } ObjectListing objects = s3.listObjects(request); statistics.incrementReadOps(1); while (true) { for (S3ObjectSummary summary : objects.getObjectSummaries()) { Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir); // Skip over keys that are ourselves and old S3N _$folder$ files if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) { if (LOG.isDebugEnabled()) { LOG.debug("Ignoring: " + keyPath); } continue; } if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) { result.add(new S3AFileStatus(true, true, keyPath)); if (LOG.isDebugEnabled()) { LOG.debug("Adding: fd: " + keyPath); } } else { result.add(new S3AFileStatus(summary.getSize(), dateToLong(summary.getLastModified()), keyPath)); if (LOG.isDebugEnabled()) { LOG.debug("Adding: fi: " + keyPath); } } } for (String prefix : objects.getCommonPrefixes()) { Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir); if (keyPath.equals(f)) { continue; } result.add(new S3AFileStatus(true, false, keyPath)); if (LOG.isDebugEnabled()) { LOG.debug("Adding: rd: " + keyPath); } } if (objects.isTruncated()) { if (LOG.isDebugEnabled()) { LOG.debug("listStatus: list truncated - getting next batch"); } objects = s3.listNextBatchOfObjects(objects); statistics.incrementReadOps(1); } else { break; } } } else { if (LOG.isDebugEnabled()) { LOG.debug("Adding: rd (not a dir): " + f); } result.add(fileStatus); } return result.toArray(new FileStatus[result.size()]); }
From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java
License:Apache License
/** * Return a file status object that represents the path. * @param f The path we want information from * @return a FileStatus object// w ww.j a va2s . c o m * @throws java.io.FileNotFoundException when the path does not exist; * IOException see specific implementation */ public S3AFileStatus getFileStatus(Path f) throws IOException { String key = pathToKey(f); LOG.info("Getting path status for " + f + " (" + key + ")"); if (!key.isEmpty()) { try { ObjectMetadata meta = s3.getObjectMetadata(bucket, key); statistics.incrementReadOps(1); if (objectRepresentsDirectory(key, meta.getContentLength())) { if (LOG.isDebugEnabled()) { LOG.debug("Found exact file: fake directory"); } return new S3AFileStatus(true, true, f.makeQualified(uri, workingDir)); } else { if (LOG.isDebugEnabled()) { LOG.debug("Found exact file: normal file"); } return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), f.makeQualified(uri, workingDir)); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { printAmazonServiceException(e); throw e; } } catch (AmazonClientException e) { printAmazonClientException(e); throw e; } // Necessary? if (!key.endsWith("/")) { try { String newKey = key + "/"; ObjectMetadata meta = s3.getObjectMetadata(bucket, newKey); statistics.incrementReadOps(1); if (objectRepresentsDirectory(newKey, meta.getContentLength())) { if (LOG.isDebugEnabled()) { LOG.debug("Found file (with /): fake directory"); } return new S3AFileStatus(true, true, f.makeQualified(uri, workingDir)); } else { LOG.warn("Found file (with /): real file? should not happen: " + key); return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), f.makeQualified(uri, workingDir)); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { printAmazonServiceException(e); throw e; } } catch (AmazonClientException e) { printAmazonClientException(e); throw e; } } } try { if (!key.isEmpty() && !key.endsWith("/")) { key = key + "/"; } ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(bucket); request.setPrefix(key); request.setDelimiter("/"); request.setMaxKeys(1); ObjectListing objects = s3.listObjects(request); statistics.incrementReadOps(1); if (objects.getCommonPrefixes().size() > 0 || objects.getObjectSummaries().size() > 0) { if (LOG.isDebugEnabled()) { LOG.debug("Found path as directory (with /): " + objects.getCommonPrefixes().size() + "/" + objects.getObjectSummaries().size()); for (S3ObjectSummary summary : objects.getObjectSummaries()) { LOG.debug("Summary: " + summary.getKey() + " " + summary.getSize()); } for (String prefix : objects.getCommonPrefixes()) { LOG.debug("Prefix: " + prefix); } } return new S3AFileStatus(true, false, f.makeQualified(uri, workingDir)); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { printAmazonServiceException(e); throw e; } } catch (AmazonClientException e) { printAmazonClientException(e); throw e; } if (LOG.isDebugEnabled()) { LOG.debug("Not Found: " + f); } throw new FileNotFoundException("No such file or directory: " + f); }