Example usage for com.amazonaws.services.s3.model ListObjectsRequest setPrefix

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ListObjectsRequest setPrefix.

Prototype

public void setPrefix(String prefix) 

Source Link

Document

Sets the optional prefix parameter, restricting the response to keys that begin with the specified prefix.

Usage

From source file:c3.ops.priam.aws.S3FileIterator.java

License:Apache License

public S3FileIterator(Provider<AbstractBackupPath> pathProvider, AmazonS3 s3Client, String path, Date start,
        Date till) {//from w  w  w  .  j  a  va  2s  .  c  o m
    this.start = start;
    this.till = till;
    this.pathProvider = pathProvider;
    ListObjectsRequest listReq = new ListObjectsRequest();
    String[] paths = path.split(String.valueOf(S3BackupPath.PATH_SEP));
    listReq.setBucketName(paths[0]);
    listReq.setPrefix(pathProvider.get().remotePrefix(start, till, path));
    this.s3Client = s3Client;
    objectListing = s3Client.listObjects(listReq);
    iterator = createIterator();
}

---

From source file:c3.ops.priam.aws.S3PrefixIterator.java

License:Apache License

private void initListing() {
    ListObjectsRequest listReq = new ListObjectsRequest();
    // Get list of tokens
    listReq.setBucketName(bucket);/*from   w  w w .  j  a v a2s .co  m*/
    listReq.setPrefix(clusterPath);
    listReq.setDelimiter(String.valueOf(AbstractBackupPath.PATH_SEP));
    logger.info("Using cluster prefix for searching tokens: " + clusterPath);
    objectListing = s3Client.listObjects(listReq);

}

---

From source file:c3.ops.priam.aws.S3PrefixIterator.java

License:Apache License

/**
 * Check to see if the path exists for the date
 *///from www  .  j a va 2 s .  c om
private boolean pathExistsForDate(String tprefix, String datestr) {
    ListObjectsRequest listReq = new ListObjectsRequest();
    // Get list of tokens
    listReq.setBucketName(bucket);
    listReq.setPrefix(tprefix + datestr);
    ObjectListing listing;
    listing = s3Client.listObjects(listReq);
    if (listing.getObjectSummaries().size() > 0)
        return true;
    return false;
}

---

From source file:cloudtrailviewer.components.S3FileChooserDialog.java

License:Open Source License

private void reloadContents() {

    List<String> tmp = new ArrayList<String>();
    this.files.setAll(tmp);

    String bucketName = PropertiesSingleton.getInstance().getProperty("Bucket");

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(bucketName);
    listObjectsRequest.setPrefix(prefix);
    listObjectsRequest.setDelimiter("/");

    AWSCredentials credentials = new BasicAWSCredentials(PropertiesSingleton.getInstance().getProperty("Key"),
            PropertiesSingleton.getInstance().getProperty("Secret"));

    AmazonS3 s3Client = new AmazonS3Client(credentials);

    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

    // these are directories
    List<String> directories = objectListing.getCommonPrefixes();
    for (String directory : directories) {

        tmp.add(stripPrefix(directory));
    }// w  w w . java2  s .c  o  m

    // these are files
    List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
    for (final S3ObjectSummary objectSummary : objectSummaries) {

        tmp.add(stripPrefix(objectSummary.getKey()));
    }

    this.files.setAll(tmp);
}

---

From source file:com.aegeus.aws.SimpleStorageService.java

License:Apache License

/**
 * List all files under the current directory based on bucket and prefix parameters
 *
 * @param bucket Bucket name//from   w w w.jav  a 2s. co  m
 * @param prefix Prefix name
 * @return File list
 */
public List<S3ObjectSummary> listObjects(String bucket, String prefix) {
    ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket);
    if (prefix != null) {
        request.setPrefix(prefix);
    }
    List<S3ObjectSummary> summaries = new ArrayList<>();
    ObjectListing listing;

    do {
        listing = s3.listObjects(request);
        summaries.addAll(listing.getObjectSummaries());
        request.setMarker(listing.getNextMarker());
    } while (listing.isTruncated());

    return summaries;
}

---

From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java

License:Open Source License

@Override
public ListBucketResponseType listBucket(ListBucketType request) throws S3Exception {
    ListBucketResponseType reply = request.getReply();
    User requestUser = getRequestUser(request);
    OsgInternalS3Client internalS3Client = null;
    try {//from  w  w w .  j  av  a 2 s.co  m
        internalS3Client = getS3Client(requestUser);
        AmazonS3Client s3Client = internalS3Client.getS3Client();
        ListObjectsRequest listRequest = new ListObjectsRequest();
        listRequest.setBucketName(request.getBucket());
        listRequest.setDelimiter(Strings.isNullOrEmpty(request.getDelimiter()) ? null : request.getDelimiter());
        listRequest.setMarker(Strings.isNullOrEmpty(request.getMarker()) ? null : request.getMarker());
        listRequest.setMaxKeys((request.getMaxKeys() == null ? null : Integer.parseInt(request.getMaxKeys())));
        listRequest.setPrefix(Strings.isNullOrEmpty(request.getPrefix()) ? null : request.getPrefix());

        ObjectListing response = s3Client.listObjects(listRequest);

        /* Non-optional, must have non-null values */
        reply.setName(request.getBucket());
        reply.setMaxKeys(response.getMaxKeys());
        reply.setMarker(response.getMarker() == null ? "" : response.getMarker());
        reply.setPrefix(response.getPrefix() == null ? "" : response.getPrefix());
        reply.setIsTruncated(response.isTruncated());

        /* Optional */
        reply.setNextMarker(response.getNextMarker());
        reply.setDelimiter(response.getDelimiter());
        if (reply.getContents() == null) {
            reply.setContents(new ArrayList<ListEntry>());
        }
        if (reply.getCommonPrefixesList() == null) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());
        }

        for (S3ObjectSummary obj : response.getObjectSummaries()) {
            //Add entry, note that the canonical user is set based on requesting user, not returned user
            reply.getContents()
                    .add(new ListEntry(obj.getKey(),
                            DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(),
                            obj.getSize(), getCanonicalUser(requestUser), obj.getStorageClass()));
        }

        if (response.getCommonPrefixes() != null && response.getCommonPrefixes().size() > 0) {
            reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>());

            for (String s : response.getCommonPrefixes()) {
                reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s));
            }
        }

        return reply;
    } catch (AmazonServiceException e) {
        LOG.debug("Error from backend", e);
        throw S3ExceptionMapper.fromAWSJavaSDK(e);
    }
}

---

From source file:com.haskins.cloudtrailviewer.dialog.s3filechooser.S3FileList.java

License:Open Source License

private ObjectListing s3ListObjects(String pathPrefix, String delimiter) {

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(currentAccount.getBucket());
    listObjectsRequest.setPrefix(pathPrefix);

    if (delimiter != null) {
        listObjectsRequest.setDelimiter(delimiter);
    }//  w w w .j  av  a  2s.c  o  m

    AmazonS3 s3Client = getS3Client();

    return s3Client.listObjects(listObjectsRequest);
}

---

From source file:com.haskins.cloudtrailviewer.dialog.S3FileChooser.java

License:Open Source License

private void reloadContents() {

    loadingLabel.setVisible(true);/*from  w  w w .j ava 2 s.  c  o m*/
    this.s3ListModel.clear();

    String bucketName = currentAccount.getBucket();

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(bucketName);
    listObjectsRequest.setPrefix(prefix);
    listObjectsRequest.setDelimiter("/");

    AWSCredentials credentials = new BasicAWSCredentials(currentAccount.getKey(), currentAccount.getSecret());

    AmazonS3 s3Client = new AmazonS3Client(credentials);

    try {
        ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

        // Add .. if not at root
        if (prefix.trim().length() != 0) {
            S3ListModel model = new S3ListModel(MOVE_BACK, MOVE_BACK, S3ListModel.FILE_BACK);
            this.s3ListModel.addElement(model);
        }

        // these are directories
        List<String> directories = objectListing.getCommonPrefixes();
        for (String directory : directories) {

            String dir = stripPrefix(directory);
            int lastSlash = dir.lastIndexOf("/");
            String strippeDir = dir.substring(0, lastSlash);

            String alias = dir;
            if (isAccountNumber(strippeDir)) {
                if (aliasMap.containsKey(strippeDir)) {
                    alias = aliasMap.get(strippeDir);
                }
            }

            S3ListModel model = new S3ListModel(dir, alias, S3ListModel.FILE_DIR);
            this.s3ListModel.addElement(model);
        }

        // these are files
        List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
        for (final S3ObjectSummary objectSummary : objectSummaries) {

            String file = stripPrefix(objectSummary.getKey());

            S3ListModel model = new S3ListModel(file, file, S3ListModel.FILE_DOC);
            this.s3ListModel.addElement(model);
        }

        loadingLabel.setVisible(false);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

---

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

@Override
public FileStatus getFileStatus(String hostName, Path path, String msg)
        throws IOException, FileNotFoundException {
    FileStatus res = null;/*from w  w w  .j  ava2 s. co  m*/
    FileStatus cached = memoryCache.getFileStatus(path.toString());
    if (cached != null) {
        return cached;
    }
    LOG.trace("getFileStatus(start) for {}, hostname: {}", path, hostName);
    /*
     * The requested path is equal to hostName. HostName is equal to
     * hostNameScheme, thus the container. Therefore we have no object to look
     * for and we return the FileStatus as a directory. Containers have to
     * lastModified.
     */
    if (path.toString().equals(hostName) || (path.toString().length() + 1 == hostName.length())) {
        LOG.trace("getFileStatus(completed) {}", path);
        res = new FileStatus(0L, true, 1, mBlockSize, 0L, path);
        memoryCache.putFileStatus(path.toString(), res);
        return res;
    }
    if (path.toString().contains(HADOOP_TEMPORARY)) {
        LOG.debug("getFileStatus on temp object {}. Return not found", path.toString());
        throw new FileNotFoundException("Not found " + path.toString());
    }
    String key = pathToKey(hostName, path);
    LOG.debug("getFileStatus: on original key {}", key);
    try {
        FileStatus fileStatus = null;
        try {
            fileStatus = getFileStatusKeyBased(key, path);
        } catch (AmazonS3Exception e) {
            if (e.getStatusCode() != 404) {
                throw new IOException(e);
            }
        }
        if (fileStatus != null) {
            LOG.trace("getFileStatus(completed) {}", path);
            memoryCache.putFileStatus(path.toString(), fileStatus);
            return fileStatus;
        }
        // means key returned not found. Trying to call get file status on key/
        // probably not needed this call
        if (!key.endsWith("/")) {
            String newKey = key + "/";
            try {
                LOG.debug("getFileStatus: original key not found. Alternative key {}", key);
                fileStatus = getFileStatusKeyBased(newKey, path);
            } catch (AmazonS3Exception e) {
                if (e.getStatusCode() != 404) {
                    throw new IOException(e);
                }
            }

            if (fileStatus != null) {
                LOG.trace("getFileStatus(completed) {}", path);
                memoryCache.putFileStatus(path.toString(), fileStatus);
                return fileStatus;
            } else {
                // if here: both key and key/ returned not found.
                // trying to see if pseudo directory of the form
                // a/b/key/d/e (a/b/key/ doesn't exists by itself)
                // perform listing on the key
                LOG.debug("getFileStatus: Modifined key {} not found. Trying to lisr", key);
                key = maybeAddTrailingSlash(key);
                ListObjectsRequest request = new ListObjectsRequest();
                request.setBucketName(mBucket);
                request.setPrefix(key);
                request.setDelimiter("/");
                request.setMaxKeys(1);

                ObjectListing objects = mClient.listObjects(request);
                if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) {
                    LOG.trace("getFileStatus(completed) {}", path);
                    res = new FileStatus(0, true, 1, 0, 0, path);
                    memoryCache.putFileStatus(path.toString(), res);
                    return res;
                } else if (key.isEmpty()) {
                    LOG.debug("Found root directory");
                    LOG.trace("getFileStatus(completed) {}", path);
                    res = new FileStatus(0, true, 1, 0, 0, path);
                    memoryCache.putFileStatus(path.toString(), res);
                    return res;
                }
            }
        }
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == 403) {
            throw new IOException(e);
        }
    } catch (Exception e) {
        LOG.debug("Not found {}", path.toString());
        LOG.warn(e.getMessage());
        throw new FileNotFoundException("Not found " + path.toString());
    }
    throw new FileNotFoundException("Not found " + path.toString());
}

---

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

/**
 * {@inheritDoc}/* w w  w  .  j  a v a 2 s .  co  m*/
 *
 * Prefix based
 * Return everything that starts with the prefix
 * Fill listing
 * Return all objects, even zero size
 * If fileStatus is null means the path is part of some name, neither object
 * or pseudo directory. Was called by Globber
 *
 * @param hostName hostName
 * @param path path
 * @param fullListing Return all objects, even zero size
 * @param prefixBased Return everything that starts with the prefix
 * @return list
 * @throws IOException if error
 */
/*
public FileStatus[] list(String hostName, Path path, boolean fullListing,
    boolean prefixBased) throws IOException {
  String key = pathToKey(hostName, path);
  ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>();
  ListObjectsRequest request = new ListObjectsRequest().withBucketName(mBucket).withPrefix(key);
        
  String curObj;
  if (path.toString().equals(mBucket)) {
    curObj = "";
  } else if (path.toString().startsWith(mBucket + "/")) {
    curObj = path.toString().substring(mBucket.length() + 1);
  } else if (path.toString().startsWith(hostName)) {
    curObj = path.toString().substring(hostName.length());
  } else {
    curObj = path.toString();
  }
        
  ObjectListing objectList = mClient.listObjects(request);
  List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries();
  if (objectSummaries.size() == 0) {
    FileStatus[] emptyRes = {};
    LOG.debug("List for bucket {} is empty", mBucket);
    return emptyRes;
  }
  boolean objectScanContinue = true;
  S3ObjectSummary prevObj = null;
  while (objectScanContinue) {
    for (S3ObjectSummary obj : objectSummaries) {
if (prevObj == null) {
  prevObj = obj;
  continue;
}
String objKey = obj.getKey();
String unifiedObjectName = extractUnifiedObjectName(objKey);
if (!prefixBased && !curObj.equals("") && !path.toString().endsWith("/")
    && !unifiedObjectName.equals(curObj) && !unifiedObjectName.startsWith(curObj + "/")) {
  LOG.trace("{} does not match {}. Skipped", unifiedObjectName, curObj);
  continue;
}
if (isSparkOrigin(unifiedObjectName) && !fullListing) {
  LOG.trace("{} created by Spark", unifiedObjectName);
  if (!isJobSuccessful(unifiedObjectName)) {
    LOG.trace("{} created by failed Spark job. Skipped", unifiedObjectName);
    if (fModeAutomaticDelete) {
      delete(hostName, new Path(objKey), true);
    }
    continue;
  } else {
    // if we here - data created by spark and job completed
    // successfully
    // however there be might parts of failed tasks that
    // were not aborted
    // we need to make sure there are no failed attempts
    if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) {
      // found failed that was not aborted.
      LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey);
      if (prevObj.getSize() < obj.getSize()) {
        LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey());
        prevObj = obj;
      }
      continue;
    }
  }
}
if (prevObj.getSize() > 0 || fullListing) {
  FileStatus fs = getFileStatusObjSummaryBased(prevObj, hostName, path);
  tmpResult.add(fs);
}
prevObj = obj;
    }
    boolean isTruncated = objectList.isTruncated();
    if (isTruncated) {
objectList = mClient.listNextBatchOfObjects(objectList);
objectSummaries = objectList.getObjectSummaries();
    } else {
objectScanContinue = false;
    }
  }
  if (prevObj != null && (prevObj.getSize() > 0 || fullListing)) {
    FileStatus fs = getFileStatusObjSummaryBased(prevObj, hostName, path);
    tmpResult.add(fs);
  }
  if (LOG.isTraceEnabled()) {
    LOG.trace("COS List to return length {}", tmpResult.size());
    for (FileStatus fs: tmpResult) {
LOG.trace("{}", fs.getPath());
    }
  }
  return tmpResult.toArray(new FileStatus[tmpResult.size()]);
}
*/
@Override
public FileStatus[] list(String hostName, Path path, boolean fullListing, boolean prefixBased,
        Boolean isDirectory, boolean flatListing, PathFilter filter) throws FileNotFoundException, IOException {
    LOG.debug("Native direct list status for {}", path);
    ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>();
    String key = pathToKey(hostName, path);
    if (isDirectory != null && isDirectory.booleanValue() && !key.endsWith("/")) {
        key = key + "/";
        LOG.debug("listNativeDirect modify key to {}", key);
    }

    Map<String, FileStatus> emptyObjects = new HashMap<String, FileStatus>();
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(mBucket);
    request.setMaxKeys(5000);
    request.setPrefix(key);
    if (!flatListing) {
        request.setDelimiter("/");
    }

    ObjectListing objectList = mClient.listObjects(request);

    List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries();
    List<String> commonPrefixes = objectList.getCommonPrefixes();

    boolean objectScanContinue = true;
    S3ObjectSummary prevObj = null;
    // start FTA logic
    boolean stocatorOrigin = isSparkOrigin(key, path.toString());
    if (stocatorOrigin) {
        LOG.debug("Stocator origin is true for {}", key);
        if (!isJobSuccessful(key)) {
            LOG.debug("{} created by failed Spark job. Skipped", key);
            if (fModeAutomaticDelete) {
                delete(hostName, new Path(key), true);
            }
            return new FileStatus[0];
        }
    }
    while (objectScanContinue) {
        for (S3ObjectSummary obj : objectSummaries) {
            if (prevObj == null) {
                prevObj = obj;
                continue;
            }
            String objKey = obj.getKey();
            String unifiedObjectName = extractUnifiedObjectName(objKey);
            LOG.debug("list candidate {}, unified name {}", objKey, unifiedObjectName);
            if (stocatorOrigin && !fullListing) {
                LOG.trace("{} created by Spark", unifiedObjectName);
                // if we here - data created by spark and job completed
                // successfully
                // however there be might parts of failed tasks that
                // were not aborted
                // we need to make sure there are no failed attempts
                if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) {
                    // found failed that was not aborted.
                    LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey);
                    if (prevObj.getSize() < obj.getSize()) {
                        LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey());
                        prevObj = obj;
                    }
                    continue;
                }
            }
            FileStatus fs = createFileStatus(prevObj, hostName, path);
            if (fs.getLen() > 0 || fullListing) {
                LOG.debug("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen());
                if (filter == null) {
                    tmpResult.add(fs);
                } else if (filter != null && filter.accept(fs.getPath())) {
                    tmpResult.add(fs);
                } else {
                    LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter);
                }
            } else {
                emptyObjects.put(fs.getPath().toString(), fs);
            }
            prevObj = obj;
        }
        boolean isTruncated = objectList.isTruncated();
        if (isTruncated) {
            objectList = mClient.listNextBatchOfObjects(objectList);
            objectSummaries = objectList.getObjectSummaries();
        } else {
            objectScanContinue = false;
        }
    }

    if (prevObj != null) {
        FileStatus fs = createFileStatus(prevObj, hostName, path);
        LOG.debug("Adding the last object from the list {}", fs.getPath());
        if (fs.getLen() > 0 || fullListing) {
            LOG.debug("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen());
            if (filter == null) {
                memoryCache.putFileStatus(fs.getPath().toString(), fs);
                tmpResult.add(fs);
            } else if (filter != null && filter.accept(fs.getPath())) {
                memoryCache.putFileStatus(fs.getPath().toString(), fs);
                tmpResult.add(fs);
            } else {
                LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter);
            }
        } else if (!fs.getPath().getName().equals(HADOOP_SUCCESS)) {
            emptyObjects.put(fs.getPath().toString(), fs);
        }
    }

    // get common prefixes
    for (String comPrefix : commonPrefixes) {
        LOG.debug("Common prefix is {}", comPrefix);
        if (emptyObjects.containsKey(keyToQualifiedPath(hostName, comPrefix).toString())
                || emptyObjects.isEmpty()) {
            FileStatus status = new COSFileStatus(true, false, keyToQualifiedPath(hostName, comPrefix));
            LOG.debug("Match between common prefix and empty object {}. Adding to result", comPrefix);
            if (filter == null) {
                memoryCache.putFileStatus(status.getPath().toString(), status);
                tmpResult.add(status);
            } else if (filter != null && filter.accept(status.getPath())) {
                memoryCache.putFileStatus(status.getPath().toString(), status);
                tmpResult.add(status);
            } else {
                LOG.trace("Common prefix {} rejected by path filter during list. Filter {}", status.getPath(),
                        filter);
            }
        }
    }
    return tmpResult.toArray(new FileStatus[tmpResult.size()]);
}

---

• «
• 1
• 2
• 3
• 4
• »