Example usage for com.amazonaws.services.s3 AmazonS3Client listObjects

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 AmazonS3Client listObjects.

Prototype

@Override
    public ObjectListing listObjects(String bucketName, String prefix)
            throws SdkClientException, AmazonServiceException 

Source Link

Usage

From source file:com.netflix.ice.basic.BasicManagers.java

License:Apache License

private void doWork() {

    logger.info("trying to find new tag group and data managers...");
    Set<Product> products = Sets.newHashSet(this.products);
    Map<Product, BasicTagGroupManager> tagGroupManagers = Maps.newHashMap(this.tagGroupManagers);
    TreeMap<Key, BasicDataManager> costManagers = Maps.newTreeMap(this.costManagers);
    TreeMap<Key, BasicDataManager> usageManagers = Maps.newTreeMap(this.usageManagers);

    Set<Product> newProducts = Sets.newHashSet();
    AmazonS3Client s3Client = AwsUtils.getAmazonS3Client();
    for (S3ObjectSummary s3ObjectSummary : s3Client
            .listObjects(config.workS3BucketName, config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX)
            .getObjectSummaries()) {// ww w  . j a  v  a2  s .com
        String key = s3ObjectSummary.getKey();
        Product product;
        if (key.endsWith("_all")) {
            product = null;
        } else {
            String name = key.substring((config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).length());
            product = config.productService.getProductByName(name);
        }
        if (!products.contains(product)) {
            products.add(product);
            newProducts.add(product);
        }
    }

    for (Product product : newProducts) {
        tagGroupManagers.put(product, new BasicTagGroupManager(product));
        for (ConsolidateType consolidateType : ConsolidateType.values()) {
            Key key = new Key(product, consolidateType);
            costManagers.put(key, new BasicDataManager(product, consolidateType, true));
            usageManagers.put(key, new BasicDataManager(product, consolidateType, false));
        }
    }

    if (newProducts.size() > 0) {
        this.costManagers = costManagers;
        this.usageManagers = usageManagers;
        this.tagGroupManagers = tagGroupManagers;
        this.products = products;
    }
}

---

From source file:com.netflix.ice.basic.MapDb.java

License:Apache License

MapDb(String name) {
    this.config = ProcessorConfig.getInstance();

    this.dbName = "db_" + name;
    File file = new File(config.localDir, dbName);
    if (!file.exists()) {
        AmazonS3Client s3Client = AwsUtils.getAmazonS3Client();
        for (S3ObjectSummary s3ObjectSummary : s3Client
                .listObjects(config.workS3BucketName, config.workS3BucketPrefix + this.dbName)
                .getObjectSummaries()) {
            File dbFile = new File(config.localDir,
                    s3ObjectSummary.getKey().substring(config.workS3BucketPrefix.length()));
            AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, dbFile);
        }//from  w  w  w. ja  v a  2 s .co  m
    }
    this.db = DBMaker.newFileDB(new File(config.localDir, this.dbName)).make();
    try {
        this.items = db.createHashMap(name, false, null, null);
    } catch (IllegalArgumentException e) {
        this.items = db.getHashMap(name);
        logger.info("found " + this.items.size() + " items from mapdb for " + name);
    }
}

---

From source file:com.shareplaylearn.models.UserItemManager.java

License:Open Source License

public HashMap<String, HashMap<ItemSchema.PresentationType, List<String>>> getItemLocations() {
    HashMap<String, HashMap<ItemSchema.PresentationType, List<String>>> itemLocations = new HashMap<>();

    AmazonS3Client s3Client = new AmazonS3Client(
            new BasicAWSCredentials(SecretsService.amazonClientId, SecretsService.amazonClientSecret));

    for (String contentType : ItemSchema.CONTENT_TYPES) {
        for (ItemSchema.PresentationType presentationType : ItemSchema.PRESENTATION_TYPES) {

            ObjectListing listing = s3Client.listObjects(ItemSchema.S3_BUCKET,
                    this.getItemDirectory(contentType, presentationType));

            HashSet<String> locations = getExternalItemListing(listing);
            String curDirectory = makeExternalLocation(getItemDirectory(contentType, presentationType));
            for (String location : locations) {
                //it would be nice if s3 didn't return stuff that doesn't technically match the prefix
                //(due to trailing /), but it looks like it might
                if (curDirectory.endsWith(location)) {
                    log.debug("Skipping location: " + location + " because it looks like a group (folder)"
                            + ", not an object");
                    continue;
                }/*  w  w  w.  jav a  2  s.  c o  m*/
                if (!itemLocations.containsKey(contentType)) {
                    itemLocations.put(contentType, new HashMap<>());
                }
                if (!itemLocations.get(contentType).containsKey(presentationType)) {
                    itemLocations.get(contentType).put(presentationType, new ArrayList<>());
                }
                itemLocations.get(contentType).get(presentationType).add(location);
            }
        }
    }
    return itemLocations;
}

---

From source file:com.shareplaylearn.models.UserItemManager.java

License:Open Source License

private Response checkQuota() {
    AmazonS3Client s3Client = new AmazonS3Client(
            new BasicAWSCredentials(SecretsService.amazonClientId, SecretsService.amazonClientSecret));
    ObjectListing curList = s3Client.listObjects(ItemSchema.S3_BUCKET, this.getUserDir());
    Response listCheck;/*from  w  ww .ja  v  a  2s  . c o  m*/
    if ((listCheck = this.checkObjectListingSize(curList, Limits.MAX_NUM_FILES_PER_USER))
            .getStatus() != Response.Status.OK.getStatusCode()) {
        return listCheck;
    }
    ObjectListing userList = s3Client.listObjects(ItemSchema.S3_BUCKET, "/");
    if ((listCheck = this.checkObjectListingSize(userList, Limits.MAX_TOTAL_FILES))
            .getStatus() != Response.Status.OK.getStatusCode()) {
        return listCheck;
    }
    return Response.status(Response.Status.OK).build();
}

---

From source file:n3phele.storage.s3.CloudStorageImpl.java

License:Open Source License

public boolean deleteFolder(Repository repo, String filename) {
    boolean result = false;
    Credential credential = repo.getCredential().decrypt();
    int retry = 3;

    setPermissions(repo, filename, false);
    if (!filename.endsWith("/")) {
        filename += "/";
    }/*from   w  w  w  .j ava 2s  .  c  om*/

    AmazonS3Client s3 = new AmazonS3Client(
            new BasicAWSCredentials(credential.getAccount(), credential.getSecret()));
    s3.setEndpoint(repo.getTarget().toString());
    while (retry-- > 0) {
        try {
            ObjectListing objects = s3.listObjects(repo.getRoot(), filename);
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                log.info("Delete " + repo.getRoot() + ":" + objectSummary.getKey());
                s3.deleteObject(repo.getRoot(), objectSummary.getKey());
            }
            if (objects.isTruncated()) {
                retry++;
                log.info("Doing next portion");
                continue;
            }
            result = true;
            break;
        } catch (AmazonServiceException e) {
            log.log(Level.WARNING, "Service Error processing " + repo, e);
        } catch (AmazonClientException e) {
            log.log(Level.SEVERE, "Client Error processing " + repo, e);
        }
    }
    return result;
}

---