List of usage examples for com.amazonaws.services.s3 AmazonS3Client listObjects
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
throws SdkClientException, AmazonServiceException
From source file:com.qubole.qds.sdk.java.client.ResultStreamer.java
License:Apache License
@VisibleForTesting protected S3Client newS3Client() throws Exception { Account account = getAccount();// www. j ava 2 s . co m AWSCredentials awsCredentials = new BasicAWSCredentials(account.getStorage_access_key(), account.getStorage_secret_key()); final AmazonS3Client client = new AmazonS3Client(awsCredentials); return new S3Client() { @Override public void shutdown() { client.shutdown(); } @Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) { return client.listObjects(listObjectsRequest); } @Override public S3Object getObject(String bucket, String key) { return client.getObject(bucket, key); } }; }
From source file:com.tango.BucketSyncer.KeyListers.S3KeyLister.java
License:Apache License
private ObjectListing s3getFirstBatch(AmazonS3Client client, ListObjectsRequest request) { final MirrorOptions options = context.getOptions(); final boolean verbose = options.isVerbose(); final int maxRetries = options.getMaxRetries(); ObjectListing listing = null;// w w w.ja v a2 s. c o m Exception lastException = null; for (int tries = 0; tries < maxRetries; tries++) { try { context.getStats().getCount.incrementAndGet(); listing = client.listObjects(request); if (verbose) { log.info("successfully got first batch of objects (on try # {})", tries); } break; } catch (AmazonS3Exception e) { if (e.getStatusCode() == HttpStatus.SC_FORBIDDEN) { if (verbose) { log.error("Please provide valid AWS credentials: {}", e); } } System.exit(1); } catch (Exception e) { lastException = e; log.warn("s3getFirstBatch: error listing (try # {}): {}", tries, e); if (Sleep.sleep(50)) { log.info("s3getFirstBatch: interrupted while waiting for next try"); break; } } } if (listing != null) { return listing; } else { throw new IllegalStateException("s3getFirstBatch: error listing: " + lastException, lastException); } }
From source file:com.treasure_data.td_import.source.S3Source.java
License:Apache License
static List<S3ObjectSummary> getSources(AmazonS3Client client, String bucket, String basePath) { String prefix;// w ww .j a va 2 s . c o m int index = basePath.indexOf('*'); if (index >= 0) { prefix = basePath.substring(0, index); } else { ObjectMetadata om = client.getObjectMetadata(bucket, basePath); S3ObjectSummary s3object = new S3ObjectSummary(); s3object.setBucketName(bucket); s3object.setKey(basePath); s3object.setSize(om.getContentLength()); return Arrays.asList(s3object); } LOG.info(String.format("list s3 files by client %s: bucket=%s, basePath=%s, prefix=%s", client, bucket, basePath, prefix)); List<S3ObjectSummary> s3objects = new ArrayList<S3ObjectSummary>(); String lastKey = prefix; do { ObjectListing listing = client.listObjects(new ListObjectsRequest(bucket, prefix, lastKey, null, 1024)); for (S3ObjectSummary s3object : listing.getObjectSummaries()) { s3objects.add(s3object); } lastKey = listing.getNextMarker(); } while (lastKey != null); return filterSources(s3objects, basePath); }
From source file:dashboard.AmazonLogs.java
License:Open Source License
public int readAmazonLogs(int n, String AWS_USER, String AWS_PASS, String IPfile, String ERRfile, String bucketName, String DELETE_PROCESSED_LOGS, String API_KEY, String TOKEN, String apiuser, String apipass) throws Exception { if (n < 1) return 0; int eventsNumber = 0; String line = null;//w w w .j a va 2 s. co m int begin = 0; int zips = 0; int deletedZips = 0; int mixpanelStatus = 0; String registrant = ""; String ip = ""; String prevIP = ""; Mixpanel mix = new Mixpanel(); Whois w = new Whois(apiuser, apipass); int index = -1; Registrant r; ArrayList<Registrant> rList = new ArrayList<Registrant>(); ArrayList<Registrant> eList = new ArrayList<Registrant>(); IPList ipl = new IPList(); IPList errl = new IPList(); // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; else { ipl.loadList(rList, IPfile); ipl.printList(rList, 30); } while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; ip = values[4]; if (ip != prevIP) { prevIP = ip; index = ipl.ipInList(ip, rList); if (index >= 0) { r = rList.get(index); registrant = r.name; // Update counter for this IP r.counter = r.counter + 1; rList.set(index, r); } else { // WHOIS - Check registrant of this IP address registrant = w.whoisIP(ip); // if there was an error, try again if (registrant.equals("ERROR")) registrant = w.whoisIP(ip); // if there was a second error, add it to errors list if (registrant.equals("ERROR")) { eList.add(new Registrant(ip, registrant, 1)); } else { // If name includes a comma, exclude the comma registrant = registrant.replace(",", ""); rList.add(new Registrant(ip, registrant, 1)); } } } String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip + " " + registrant); // ==================================================== // Track the event in Mixpanel (using the POST import) // ==================================================== mixpanelStatus = mix.postCDNEventToMixpanel(API_KEY, TOKEN, ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode, registrant); } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("========= Deleted Zip " + zips + " ===== List Size " + rList.size() + " =========="); deletedZips++; } } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 100); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 50); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; }
From source file:dashboard.ImportCDN.java
License:Open Source License
public static int readAmazonLogs(int n) throws Exception { if (n < 1) return 0; int eventsNumber = 0; int begin = 0; int zips = 0; int postFailures = 0; int deletedZips = 0; int mixpanelStatus = 0; String line = null;/*from w w w.j a v a 2s . c o m*/ // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); // Set MARKER - from which Log File to start reading // listObjectsRequest.setMarker("E2DXXJR0N8BXOK.2013-03-18-10.ICK6IvaY.gz"); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; String ip = values[4]; String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip); // Track the event in Mixpanel (using the POST import) mixpanelStatus = postCDNEventToMixpanel(ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode); // If failed if (mixpanelStatus != 1) { postFailures++; System.out.println(" >>> POST event to Mixpanel Failed!! " + postFailures); } } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("============ Deleted Zip " + zips + " ============"); deletedZips++; } else System.out.println("============ Zip " + zips + " (not deleted) ============"); } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; }
From source file:de.fischer.thotti.s3.clients.S3FileUploader.java
License:Apache License
private void preparePut(String bucketName) { AmazonS3Client client = getAmazonS3Client(); /*//from www . j a va2s.co m * Obtain the list of already existing objects in the * bucket to check if the object already exists. */ ObjectListing bucketObjects = client.listObjects(bucketName); setObjectsInBucket(bucketObjects); }
From source file:imperial.modaclouds.monitoring.datacollectors.monitors.DetailedCostMonitor.java
License:BSD License
@Override public void run() { String accessKeyId = null;/*from w ww. j a v a 2 s. c o m*/ String secretKey = null; ObjectListing objects = null; AmazonS3Client s3Client = null; String key = null; long startTime = 0; while (!dcmt.isInterrupted()) { if (System.currentTimeMillis() - startTime > 10000) { cost_nonspot = new HashMap<String, Double>(); cost_spot = new HashMap<String, Double>(); for (String metric : getProvidedMetrics()) { try { VM resource = new VM(Config.getInstance().getVmType(), Config.getInstance().getVmId()); if (dcAgent.shouldMonitor(resource, metric)) { Map<String, String> parameters = dcAgent.getParameters(resource, metric); accessKeyId = parameters.get("accessKey"); secretKey = parameters.get("secretKey"); bucketName = parameters.get("bucketName"); filePath = parameters.get("filePath"); period = Integer.valueOf(parameters.get("samplingTime")) * 1000; } } catch (NumberFormatException e) { e.printStackTrace(); } catch (ConfigurationException e) { e.printStackTrace(); } } startTime = System.currentTimeMillis(); AWSCredentials credentials = new BasicAWSCredentials(accessKeyId, secretKey); s3Client = new AmazonS3Client(credentials); objects = s3Client.listObjects(bucketName); key = "aws-billing-detailed-line-items-with-resources-and-tags-"; SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM"); String date = sdf.format(new Date()); key = key + date + ".csv.zip"; } String fileName = null; do { for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { System.out.println(objectSummary.getKey() + "\t" + objectSummary.getSize() + "\t" + StringUtils.fromDate(objectSummary.getLastModified())); if (objectSummary.getKey().contains(key)) { fileName = objectSummary.getKey(); s3Client.getObject(new GetObjectRequest(bucketName, fileName), new File(filePath + fileName)); break; } } objects = s3Client.listNextBatchOfObjects(objects); } while (objects.isTruncated()); try { ZipFile zipFile = new ZipFile(filePath + fileName); zipFile.extractAll(filePath); } catch (ZipException e) { e.printStackTrace(); } String csvFileName = fileName.replace(".zip", ""); AnalyseFile(filePath + csvFileName); try { Thread.sleep(period); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } } }
From source file:n3phele.storage.s3.CloudStorageImpl.java
License:Open Source License
public boolean createBucket(Repository repo) throws ForbiddenException { Credential credential = repo.getCredential().decrypt(); AmazonS3Client s3 = new AmazonS3Client( new BasicAWSCredentials(credential.getAccount(), credential.getSecret())); s3.setEndpoint(repo.getTarget().toString()); try {/*w w w .j a v a2 s .c om*/ try { s3.listObjects(new ListObjectsRequest(repo.getRoot(), null, null, null, 0)); // it exists and the current account owns it return false; } catch (AmazonServiceException ase) { switch (ase.getStatusCode()) { case 403: /* * A permissions error means the bucket exists, but is owned by * another account. */ throw new ForbiddenException( "Bucket " + repo.getRoot() + " has already been created by another user."); case 404: Bucket bucket = s3.createBucket(repo.getRoot()); log.info("Bucket created " + bucket.getName()); return true; default: throw ase; } } } catch (AmazonServiceException e) { log.log(Level.WARNING, "Service Error processing " + repo, e); } catch (AmazonClientException e) { log.log(Level.SEVERE, "Client Error processing " + repo, e); } return false; }
From source file:n3phele.storage.s3.CloudStorageImpl.java
License:Open Source License
public FileNode getMetadata(Repository repo, String filename) { FileNode f = new FileNode(); UriBuilder result = UriBuilder.fromUri(repo.getTarget()); result.path(repo.getRoot()).path(filename); f.setName(result.build().toString()); Credential credential = repo.getCredential().decrypt(); log.info("Get info on " + repo.getRoot() + " " + filename); AmazonS3Client s3 = new AmazonS3Client( new BasicAWSCredentials(credential.getAccount(), credential.getSecret())); s3.setEndpoint(repo.getTarget().toString()); try {/*from w w w. j av a 2 s .c o m*/ ObjectListing metadata = s3.listObjects( new ListObjectsRequest().withBucketName(repo.getRoot()).withPrefix(filename).withMaxKeys(1)); List<S3ObjectSummary> metadataList = metadata.getObjectSummaries(); if (metadataList == null || metadataList.size() != 1) { throw new NotFoundException(filename); } f.setModified(metadataList.get(0).getLastModified()); f.setSize(metadataList.get(0).getSize()); log.info(filename + " " + f.getModified() + " " + f.getSize()); } catch (AmazonServiceException e) { log.log(Level.WARNING, "Service Error processing " + f + repo, e); throw new NotFoundException("Retrieve " + filename + " fails " + e.toString()); } catch (AmazonClientException e) { log.log(Level.SEVERE, "Client Error processing " + f + repo, e); throw new NotFoundException("Retrieve " + filename + " fails " + e.toString()); } return f; }
From source file:n3phele.storage.s3.CloudStorageImpl.java
License:Open Source License
public List<FileNode> getFileList(Repository repo, String prefix, int max) { List<FileNode> result = new ArrayList<FileNode>(); Credential credential = repo.getCredential().decrypt(); AmazonS3Client s3 = new AmazonS3Client( new BasicAWSCredentials(credential.getAccount(), credential.getSecret())); s3.setEndpoint(repo.getTarget().toString()); Policy p = null;//from www. j ava 2s . c o m try { BucketPolicy bp = s3.getBucketPolicy(repo.getRoot()); log.info("Policy text " + bp.getPolicyText()); if (bp != null && bp.getPolicyText() != null) { p = PolicyHelper.parse(bp.getPolicyText()); log.info("Policy object is " + (p == null ? null : p.toJson())); } } catch (Exception e) { log.log(Level.WARNING, "Policy not supported", e); } try { ObjectListing s3Objects = s3 .listObjects(new ListObjectsRequest(repo.getRoot(), prefix, null, "/", max)); if (s3Objects.getCommonPrefixes() != null) { for (String dirName : s3Objects.getCommonPrefixes()) { String name = dirName; if (dirName.endsWith("/")) { name = dirName.substring(0, dirName.length() - 1); } boolean isPublic = isPublicFolder(p, repo.getRoot(), name); name = name.substring(name.lastIndexOf("/") + 1); FileNode folder = FileNode.newFolder(name, prefix, repo, isPublic); log.info("Folder:" + folder); result.add(folder); } } if (s3Objects.getObjectSummaries() != null) { for (S3ObjectSummary fileSummary : s3Objects.getObjectSummaries()) { String key = fileSummary.getKey(); if (key != null && !key.equals(prefix)) { String name = key.substring(key.lastIndexOf("/") + 1); UriBuilder builder = UriBuilder.fromUri(repo.getTarget()).path(repo.getRoot()); if (prefix != null && !prefix.isEmpty()) { builder = builder.path(prefix); } FileNode file = FileNode.newFile(name, prefix, repo, fileSummary.getLastModified(), fileSummary.getSize(), builder.path(name).toString()); log.info("File:" + file); result.add(file); } } } } catch (AmazonServiceException e) { log.log(Level.WARNING, "Service Error processing " + repo, e); } catch (AmazonClientException e) { log.log(Level.SEVERE, "Client Error processing " + repo, e); } return result; }