Example usage for com.amazonaws.services.s3.model ListObjectsRequest setDelimiter

List of usage examples for com.amazonaws.services.s3.model ListObjectsRequest setDelimiter

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. ListObjectsRequest
  6. setDelimiter

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ListObjectsRequest setDelimiter.

Prototype

public void setDelimiter(String delimiter)

Source Link

Document

Sets the optional delimiter parameter that causes keys that contain the same string between the prefix and the first occurrence of the delimiter to be combined into a single result element in the ObjectListing#getCommonPrefixes() list.

Usage

From source file:c3.ops.priam.aws.S3PrefixIterator.java

License:Apache License

private void initListing() {
    ListObjectsRequest listReq = new ListObjectsRequest();
    // Get list of tokens
    listReq.setBucketName(bucket);// ww w  . ja  v  a  2s. com
    listReq.setPrefix(clusterPath);
    listReq.setDelimiter(String.valueOf(AbstractBackupPath.PATH_SEP));
    logger.info("Using cluster prefix for searching tokens: " + clusterPath);
    objectListing = s3Client.listObjects(listReq);

}

From source file:cloudtrailviewer.components.S3FileChooserDialog.java

License:Open Source License

private void reloadContents() {

    List<String> tmp = new ArrayList<String>();
    this.files.setAll(tmp);

    String bucketName = PropertiesSingleton.getInstance().getProperty("Bucket");

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(bucketName);
    listObjectsRequest.setPrefix(prefix);
    listObjectsRequest.setDelimiter("/");

    AWSCredentials credentials = new BasicAWSCredentials(PropertiesSingleton.getInstance().getProperty("Key"),
            PropertiesSingleton.getInstance().getProperty("Secret"));

    AmazonS3 s3Client = new AmazonS3Client(credentials);

    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

    // these are directories
    List<String> directories = objectListing.getCommonPrefixes();
    for (String directory : directories) {

        tmp.add(stripPrefix(directory));
    }// w  w w . ja va 2  s  .  c om

    // these are files
    List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
    for (final S3ObjectSummary objectSummary : objectSummaries) {

        tmp.add(stripPrefix(objectSummary.getKey()));
    }

    this.files.setAll(tmp);
}

From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java

License:Open Source License

@Override
public ListBucketResponseType listBucket(ListBucketType request) throws S3Exception {
    ListBucketResponseType reply = request.getReply();
    User requestUser = getRequestUser(request);
    OsgInternalS3Client internalS3Client = null;
    try {//  ww w  .jav a 2  s  .  c o  m
        internalS3Client = getS3Client(requestUser);
        AmazonS3Client s3Client = internalS3Client.getS3Client();
        ListObjectsRequest listRequest = new ListObjectsRequest();
        listRequest.setBucketName(request.getBucket());
        listRequest.setDelimiter(Strings.isNullOrEmpty(request.getDelimiter()) ? null : request.getDelimiter());
        listRequest.setMarker(Strings.isNullOrEmpty(request.getMarker()) ? null : request.getMarker());
        listRequest.setMaxKeys((request.getMaxKeys() == null ? null : Integer.parseInt(request.getMaxKeys())));
        listRequest.setPrefix(Strings.isNullOrEmpty(request.getPrefix()) ? null : request.getPrefix());

        ObjectListing response = s3Client.listObjects(listRequest);

        /* Non-optional, must have non-null values */
        reply.setName(request.getBucket());
        reply.setMaxKeys(response.getMaxKeys());
        reply.setMarker(response.getMarker() == null ? "" : response.getMarker());
        reply.setPrefix(response.getPrefix() == null ? "" : response.getPrefix());
        reply.setIsTruncated(response.isTruncated());

        /* Optional */
        reply.setNextMarker(response.getNextMarker());
        reply.setDelimiter(response.getDelimiter());
        if (reply.getContents() == null) {
            reply.setContents(new ArrayList<ListEntry>());
        }
        if (reply.getCommonPrefixesList() == null) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());
        }

        for (S3ObjectSummary obj : response.getObjectSummaries()) {
            //Add entry, note that the canonical user is set based on requesting user, not returned user
            reply.getContents()
                    .add(new ListEntry(obj.getKey(),
                            DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(),
                            obj.getSize(), getCanonicalUser(requestUser), obj.getStorageClass()));
        }

        if (response.getCommonPrefixes() != null && response.getCommonPrefixes().size() > 0) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());

            for (String s : response.getCommonPrefixes()) {
                reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s));
            }
        }

        return reply;
    } catch (AmazonServiceException e) {
        LOG.debug("Error from backend", e);
        throw S3ExceptionMapper.fromAWSJavaSDK(e);
    }
}

From source file:com.haskins.cloudtrailviewer.dialog.s3filechooser.S3FileList.java

License:Open Source License

private ObjectListing s3ListObjects(String pathPrefix, String delimiter) {

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(currentAccount.getBucket());
    listObjectsRequest.setPrefix(pathPrefix);

    if (delimiter != null) {
        listObjectsRequest.setDelimiter(delimiter);
    }//from   w  w w  .  j  a  v a 2s  . c  o  m

    AmazonS3 s3Client = getS3Client();

    return s3Client.listObjects(listObjectsRequest);
}

From source file:com.haskins.cloudtrailviewer.dialog.S3FileChooser.java

License:Open Source License

private void reloadContents() {

    loadingLabel.setVisible(true);/*from ww  w .jav a 2  s.co m*/
    this.s3ListModel.clear();

    String bucketName = currentAccount.getBucket();

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(bucketName);
    listObjectsRequest.setPrefix(prefix);
    listObjectsRequest.setDelimiter("/");

    AWSCredentials credentials = new BasicAWSCredentials(currentAccount.getKey(), currentAccount.getSecret());

    AmazonS3 s3Client = new AmazonS3Client(credentials);

    try {
        ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

        // Add .. if not at root
        if (prefix.trim().length() != 0) {
            S3ListModel model = new S3ListModel(MOVE_BACK, MOVE_BACK, S3ListModel.FILE_BACK);
            this.s3ListModel.addElement(model);
        }

        // these are directories
        List<String> directories = objectListing.getCommonPrefixes();
        for (String directory : directories) {

            String dir = stripPrefix(directory);
            int lastSlash = dir.lastIndexOf("/");
            String strippeDir = dir.substring(0, lastSlash);

            String alias = dir;
            if (isAccountNumber(strippeDir)) {
                if (aliasMap.containsKey(strippeDir)) {
                    alias = aliasMap.get(strippeDir);
                }
            }

            S3ListModel model = new S3ListModel(dir, alias, S3ListModel.FILE_DIR);
            this.s3ListModel.addElement(model);
        }

        // these are files
        List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
        for (final S3ObjectSummary objectSummary : objectSummaries) {

            String file = stripPrefix(objectSummary.getKey());

            S3ListModel model = new S3ListModel(file, file, S3ListModel.FILE_DOC);
            this.s3ListModel.addElement(model);
        }

        loadingLabel.setVisible(false);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

@Override
public FileStatus getFileStatus(String hostName, Path path, String msg)
        throws IOException, FileNotFoundException {
    FileStatus res = null;/* w ww.  j a  v  a2  s.  co  m*/
    FileStatus cached = memoryCache.getFileStatus(path.toString());
    if (cached != null) {
        return cached;
    }
    LOG.trace("getFileStatus(start) for {}, hostname: {}", path, hostName);
    /*
     * The requested path is equal to hostName. HostName is equal to
     * hostNameScheme, thus the container. Therefore we have no object to look
     * for and we return the FileStatus as a directory. Containers have to
     * lastModified.
     */
    if (path.toString().equals(hostName) || (path.toString().length() + 1 == hostName.length())) {
        LOG.trace("getFileStatus(completed) {}", path);
        res = new FileStatus(0L, true, 1, mBlockSize, 0L, path);
        memoryCache.putFileStatus(path.toString(), res);
        return res;
    }
    if (path.toString().contains(HADOOP_TEMPORARY)) {
        LOG.debug("getFileStatus on temp object {}. Return not found", path.toString());
        throw new FileNotFoundException("Not found " + path.toString());
    }
    String key = pathToKey(hostName, path);
    LOG.debug("getFileStatus: on original key {}", key);
    try {
        FileStatus fileStatus = null;
        try {
            fileStatus = getFileStatusKeyBased(key, path);
        } catch (AmazonS3Exception e) {
            if (e.getStatusCode() != 404) {
                throw new IOException(e);
            }
        }
        if (fileStatus != null) {
            LOG.trace("getFileStatus(completed) {}", path);
            memoryCache.putFileStatus(path.toString(), fileStatus);
            return fileStatus;
        }
        // means key returned not found. Trying to call get file status on key/
        // probably not needed this call
        if (!key.endsWith("/")) {
            String newKey = key + "/";
            try {
                LOG.debug("getFileStatus: original key not found. Alternative key {}", key);
                fileStatus = getFileStatusKeyBased(newKey, path);
            } catch (AmazonS3Exception e) {
                if (e.getStatusCode() != 404) {
                    throw new IOException(e);
                }
            }

            if (fileStatus != null) {
                LOG.trace("getFileStatus(completed) {}", path);
                memoryCache.putFileStatus(path.toString(), fileStatus);
                return fileStatus;
            } else {
                // if here: both key and key/ returned not found.
                // trying to see if pseudo directory of the form
                // a/b/key/d/e (a/b/key/ doesn't exists by itself)
                // perform listing on the key
                LOG.debug("getFileStatus: Modifined key {} not found. Trying to lisr", key);
                key = maybeAddTrailingSlash(key);
                ListObjectsRequest request = new ListObjectsRequest();
                request.setBucketName(mBucket);
                request.setPrefix(key);
                request.setDelimiter("/");
                request.setMaxKeys(1);

                ObjectListing objects = mClient.listObjects(request);
                if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) {
                    LOG.trace("getFileStatus(completed) {}", path);
                    res = new FileStatus(0, true, 1, 0, 0, path);
                    memoryCache.putFileStatus(path.toString(), res);
                    return res;
                } else if (key.isEmpty()) {
                    LOG.debug("Found root directory");
                    LOG.trace("getFileStatus(completed) {}", path);
                    res = new FileStatus(0, true, 1, 0, 0, path);
                    memoryCache.putFileStatus(path.toString(), res);
                    return res;
                }
            }
        }
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == 403) {
            throw new IOException(e);
        }
    } catch (Exception e) {
        LOG.debug("Not found {}", path.toString());
        LOG.warn(e.getMessage());
        throw new FileNotFoundException("Not found " + path.toString());
    }
    throw new FileNotFoundException("Not found " + path.toString());
}

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

/**
 * {@inheritDoc}/*from  ww w. j a  v  a  2  s.  c o  m*/
 *
 * Prefix based
 * Return everything that starts with the prefix
 * Fill listing
 * Return all objects, even zero size
 * If fileStatus is null means the path is part of some name, neither object
 * or pseudo directory. Was called by Globber
 *
 * @param hostName hostName
 * @param path path
 * @param fullListing Return all objects, even zero size
 * @param prefixBased Return everything that starts with the prefix
 * @return list
 * @throws IOException if error
 */
/*
public FileStatus[] list(String hostName, Path path, boolean fullListing,
    boolean prefixBased) throws IOException {
  String key = pathToKey(hostName, path);
  ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>();
  ListObjectsRequest request = new ListObjectsRequest().withBucketName(mBucket).withPrefix(key);
        
  String curObj;
  if (path.toString().equals(mBucket)) {
    curObj = "";
  } else if (path.toString().startsWith(mBucket + "/")) {
    curObj = path.toString().substring(mBucket.length() + 1);
  } else if (path.toString().startsWith(hostName)) {
    curObj = path.toString().substring(hostName.length());
  } else {
    curObj = path.toString();
  }
        
  ObjectListing objectList = mClient.listObjects(request);
  List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries();
  if (objectSummaries.size() == 0) {
    FileStatus[] emptyRes = {};
    LOG.debug("List for bucket {} is empty", mBucket);
    return emptyRes;
  }
  boolean objectScanContinue = true;
  S3ObjectSummary prevObj = null;
  while (objectScanContinue) {
    for (S3ObjectSummary obj : objectSummaries) {
if (prevObj == null) {
  prevObj = obj;
  continue;
}
String objKey = obj.getKey();
String unifiedObjectName = extractUnifiedObjectName(objKey);
if (!prefixBased && !curObj.equals("") && !path.toString().endsWith("/")
    && !unifiedObjectName.equals(curObj) && !unifiedObjectName.startsWith(curObj + "/")) {
  LOG.trace("{} does not match {}. Skipped", unifiedObjectName, curObj);
  continue;
}
if (isSparkOrigin(unifiedObjectName) && !fullListing) {
  LOG.trace("{} created by Spark", unifiedObjectName);
  if (!isJobSuccessful(unifiedObjectName)) {
    LOG.trace("{} created by failed Spark job. Skipped", unifiedObjectName);
    if (fModeAutomaticDelete) {
      delete(hostName, new Path(objKey), true);
    }
    continue;
  } else {
    // if we here - data created by spark and job completed
    // successfully
    // however there be might parts of failed tasks that
    // were not aborted
    // we need to make sure there are no failed attempts
    if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) {
      // found failed that was not aborted.
      LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey);
      if (prevObj.getSize() < obj.getSize()) {
        LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey());
        prevObj = obj;
      }
      continue;
    }
  }
}
if (prevObj.getSize() > 0 || fullListing) {
  FileStatus fs = getFileStatusObjSummaryBased(prevObj, hostName, path);
  tmpResult.add(fs);
}
prevObj = obj;
    }
    boolean isTruncated = objectList.isTruncated();
    if (isTruncated) {
objectList = mClient.listNextBatchOfObjects(objectList);
objectSummaries = objectList.getObjectSummaries();
    } else {
objectScanContinue = false;
    }
  }
  if (prevObj != null && (prevObj.getSize() > 0 || fullListing)) {
    FileStatus fs = getFileStatusObjSummaryBased(prevObj, hostName, path);
    tmpResult.add(fs);
  }
  if (LOG.isTraceEnabled()) {
    LOG.trace("COS List to return length {}", tmpResult.size());
    for (FileStatus fs: tmpResult) {
LOG.trace("{}", fs.getPath());
    }
  }
  return tmpResult.toArray(new FileStatus[tmpResult.size()]);
}
*/
@Override
public FileStatus[] list(String hostName, Path path, boolean fullListing, boolean prefixBased,
        Boolean isDirectory, boolean flatListing, PathFilter filter) throws FileNotFoundException, IOException {
    LOG.debug("Native direct list status for {}", path);
    ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>();
    String key = pathToKey(hostName, path);
    if (isDirectory != null && isDirectory.booleanValue() && !key.endsWith("/")) {
        key = key + "/";
        LOG.debug("listNativeDirect modify key to {}", key);
    }

    Map<String, FileStatus> emptyObjects = new HashMap<String, FileStatus>();
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(mBucket);
    request.setMaxKeys(5000);
    request.setPrefix(key);
    if (!flatListing) {
        request.setDelimiter("/");
    }

    ObjectListing objectList = mClient.listObjects(request);

    List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries();
    List<String> commonPrefixes = objectList.getCommonPrefixes();

    boolean objectScanContinue = true;
    S3ObjectSummary prevObj = null;
    // start FTA logic
    boolean stocatorOrigin = isSparkOrigin(key, path.toString());
    if (stocatorOrigin) {
        LOG.debug("Stocator origin is true for {}", key);
        if (!isJobSuccessful(key)) {
            LOG.debug("{} created by failed Spark job. Skipped", key);
            if (fModeAutomaticDelete) {
                delete(hostName, new Path(key), true);
            }
            return new FileStatus[0];
        }
    }
    while (objectScanContinue) {
        for (S3ObjectSummary obj : objectSummaries) {
            if (prevObj == null) {
                prevObj = obj;
                continue;
            }
            String objKey = obj.getKey();
            String unifiedObjectName = extractUnifiedObjectName(objKey);
            LOG.debug("list candidate {}, unified name {}", objKey, unifiedObjectName);
            if (stocatorOrigin && !fullListing) {
                LOG.trace("{} created by Spark", unifiedObjectName);
                // if we here - data created by spark and job completed
                // successfully
                // however there be might parts of failed tasks that
                // were not aborted
                // we need to make sure there are no failed attempts
                if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) {
                    // found failed that was not aborted.
                    LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey);
                    if (prevObj.getSize() < obj.getSize()) {
                        LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey());
                        prevObj = obj;
                    }
                    continue;
                }
            }
            FileStatus fs = createFileStatus(prevObj, hostName, path);
            if (fs.getLen() > 0 || fullListing) {
                LOG.debug("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen());
                if (filter == null) {
                    tmpResult.add(fs);
                } else if (filter != null && filter.accept(fs.getPath())) {
                    tmpResult.add(fs);
                } else {
                    LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter);
                }
            } else {
                emptyObjects.put(fs.getPath().toString(), fs);
            }
            prevObj = obj;
        }
        boolean isTruncated = objectList.isTruncated();
        if (isTruncated) {
            objectList = mClient.listNextBatchOfObjects(objectList);
            objectSummaries = objectList.getObjectSummaries();
        } else {
            objectScanContinue = false;
        }
    }

    if (prevObj != null) {
        FileStatus fs = createFileStatus(prevObj, hostName, path);
        LOG.debug("Adding the last object from the list {}", fs.getPath());
        if (fs.getLen() > 0 || fullListing) {
            LOG.debug("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen());
            if (filter == null) {
                memoryCache.putFileStatus(fs.getPath().toString(), fs);
                tmpResult.add(fs);
            } else if (filter != null && filter.accept(fs.getPath())) {
                memoryCache.putFileStatus(fs.getPath().toString(), fs);
                tmpResult.add(fs);
            } else {
                LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter);
            }
        } else if (!fs.getPath().getName().equals(HADOOP_SUCCESS)) {
            emptyObjects.put(fs.getPath().toString(), fs);
        }
    }

    // get common prefixes
    for (String comPrefix : commonPrefixes) {
        LOG.debug("Common prefix is {}", comPrefix);
        if (emptyObjects.containsKey(keyToQualifiedPath(hostName, comPrefix).toString())
                || emptyObjects.isEmpty()) {
            FileStatus status = new COSFileStatus(true, false, keyToQualifiedPath(hostName, comPrefix));
            LOG.debug("Match between common prefix and empty object {}. Adding to result", comPrefix);
            if (filter == null) {
                memoryCache.putFileStatus(status.getPath().toString(), status);
                tmpResult.add(status);
            } else if (filter != null && filter.accept(status.getPath())) {
                memoryCache.putFileStatus(status.getPath().toString(), status);
                tmpResult.add(status);
            } else {
                LOG.trace("Common prefix {} rejected by path filter during list. Filter {}", status.getPath(),
                        filter);
            }
        }
    }
    return tmpResult.toArray(new FileStatus[tmpResult.size()]);
}

From source file:com.upplication.s3fs.S3Iterator.java

License:Open Source License

private ListObjectsRequest buildRequest() {

    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucket);//from   ww  w  .ja v a  2  s . com
    request.setPrefix(key);
    request.setMarker(key);
    request.setDelimiter("/");
    return request;
}

From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java

License:Apache License

/**
 * List the statuses of the files/directories in the given path if the path is
 * a directory./*  ww w.  j  av  a 2 s  .  co  m*/
 *
 * @param f given path
 * @return the statuses of the files/directories in the given patch
 * @throws FileNotFoundException when the path does not exist;
 *         IOException see specific implementation
 */
public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException {
    String key = pathToKey(f);
    LOG.info("List status for path: " + f);

    final List<FileStatus> result = new ArrayList<FileStatus>();
    final FileStatus fileStatus = getFileStatus(f);

    if (fileStatus.isDirectory()) {
        if (!key.isEmpty()) {
            key = key + "/";
        }

        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(key);
        request.setDelimiter("/");
        request.setMaxKeys(maxKeys);

        if (LOG.isDebugEnabled()) {
            LOG.debug("listStatus: doing listObjects for directory " + key);
        }

        ObjectListing objects = s3.listObjects(request);
        statistics.incrementReadOps(1);

        while (true) {
            for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir);
                // Skip over keys that are ourselves and old S3N _$folder$ files
                if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Ignoring: " + keyPath);
                    }
                    continue;
                }

                if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
                    result.add(new S3AFileStatus(true, true, keyPath));
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: fd: " + keyPath);
                    }
                } else {
                    result.add(new S3AFileStatus(summary.getSize(), dateToLong(summary.getLastModified()),
                            keyPath));
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: fi: " + keyPath);
                    }
                }
            }

            for (String prefix : objects.getCommonPrefixes()) {
                Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir);
                if (keyPath.equals(f)) {
                    continue;
                }
                result.add(new S3AFileStatus(true, false, keyPath));
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Adding: rd: " + keyPath);
                }
            }

            if (objects.isTruncated()) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("listStatus: list truncated - getting next batch");
                }

                objects = s3.listNextBatchOfObjects(objects);
                statistics.incrementReadOps(1);
            } else {
                break;
            }
        }
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding: rd (not a dir): " + f);
        }
        result.add(fileStatus);
    }

    return result.toArray(new FileStatus[result.size()]);
}

From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java

License:Apache License

/**
 * Return a file status object that represents the path.
 * @param f The path we want information from
 * @return a FileStatus object/*from w w  w  . j av  a2  s .co m*/
 * @throws java.io.FileNotFoundException when the path does not exist;
 *         IOException see specific implementation
 */
public S3AFileStatus getFileStatus(Path f) throws IOException {
    String key = pathToKey(f);

    LOG.info("Getting path status for " + f + " (" + key + ")");

    if (!key.isEmpty()) {
        try {
            ObjectMetadata meta = s3.getObjectMetadata(bucket, key);
            statistics.incrementReadOps(1);

            if (objectRepresentsDirectory(key, meta.getContentLength())) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Found exact file: fake directory");
                }
                return new S3AFileStatus(true, true, f.makeQualified(uri, workingDir));
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Found exact file: normal file");
                }
                return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()),
                        f.makeQualified(uri, workingDir));
            }
        } catch (AmazonServiceException e) {
            if (e.getStatusCode() != 404) {
                printAmazonServiceException(e);
                throw e;
            }
        } catch (AmazonClientException e) {
            printAmazonClientException(e);
            throw e;
        }

        // Necessary?
        if (!key.endsWith("/")) {
            try {
                String newKey = key + "/";
                ObjectMetadata meta = s3.getObjectMetadata(bucket, newKey);
                statistics.incrementReadOps(1);

                if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Found file (with /): fake directory");
                    }
                    return new S3AFileStatus(true, true, f.makeQualified(uri, workingDir));
                } else {
                    LOG.warn("Found file (with /): real file? should not happen: " + key);

                    return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()),
                            f.makeQualified(uri, workingDir));
                }
            } catch (AmazonServiceException e) {
                if (e.getStatusCode() != 404) {
                    printAmazonServiceException(e);
                    throw e;
                }
            } catch (AmazonClientException e) {
                printAmazonClientException(e);
                throw e;
            }
        }
    }

    try {
        if (!key.isEmpty() && !key.endsWith("/")) {
            key = key + "/";
        }
        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(key);
        request.setDelimiter("/");
        request.setMaxKeys(1);

        ObjectListing objects = s3.listObjects(request);
        statistics.incrementReadOps(1);

        if (objects.getCommonPrefixes().size() > 0 || objects.getObjectSummaries().size() > 0) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Found path as directory (with /): " + objects.getCommonPrefixes().size() + "/"
                        + objects.getObjectSummaries().size());

                for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                    LOG.debug("Summary: " + summary.getKey() + " " + summary.getSize());
                }
                for (String prefix : objects.getCommonPrefixes()) {
                    LOG.debug("Prefix: " + prefix);
                }
            }

            return new S3AFileStatus(true, false, f.makeQualified(uri, workingDir));
        }
    } catch (AmazonServiceException e) {
        if (e.getStatusCode() != 404) {
            printAmazonServiceException(e);
            throw e;
        }
    } catch (AmazonClientException e) {
        printAmazonClientException(e);
        throw e;
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug("Not Found: " + f);
    }
    throw new FileNotFoundException("No such file or directory: " + f);
}