List of usage examples for com.amazonaws.services.s3 AmazonS3Client listNextBatchOfObjects
@Override
public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest)
throws SdkClientException, AmazonServiceException
From source file:com.appdynamics.monitors.s3.AWSS3Monitor.java
License:Apache License
/** * This method calls Amazon WS to get required S3 statistics, set values * based on configured unit, and returns the result back * /*w ww. ja v a2s.c om*/ * @param buckets * @param amazonS3Client * @return Map<String, String> * @throws TaskExecutionException */ private Map<String, String> getS3Result(List<Bucket> buckets, AmazonS3Client amazonS3Client) throws TaskExecutionException { // Declaring result variables with default values long size = 0; long count = 0; Date lastModified = new Date(0); try { // Fetching all bucket names if passed buckets is null if (buckets == null) { logger.debug("Calling Webservice to list all buckets"); buckets = amazonS3Client.listBuckets(); } // Looping over all buckets for (Bucket bucket : buckets) { logger.debug("Getting data for bucket: " + bucket.getName()); ObjectListing objectListing = null; do { // Getting objectListing while calling it for the first time if (objectListing == null) { logger.debug("Calling Webservice to get objectlisting for first time"); objectListing = amazonS3Client.listObjects(bucket.getName()); } else { // Calling listNextBatchOfObjects if previous response // is truncated logger.debug("Calling Webservice to get objectlisting subsequent time"); objectListing = amazonS3Client.listNextBatchOfObjects(objectListing); } // Incrementing the count count += objectListing.getObjectSummaries().size(); // Looping over all objects for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) { // Incrementing size size += s3ObjectSummary.getSize(); // Setting last modified if lastModifiedDate is latest if (lastModified.before(s3ObjectSummary.getLastModified())) { lastModified = s3ObjectSummary.getLastModified(); } } } // Continuing till objectListing is complete while (objectListing.isTruncated()); } } catch (AmazonS3Exception exception) { logger.error("AmazonS3Exception occurred", exception); throw new TaskExecutionException("Sending S3 metric failed due to AmazonS3Exception"); } return getResultWithRequiredUnit(size, count, lastModified); }
From source file:com.netflix.exhibitor.core.s3.S3ClientFactoryImpl.java
License:Apache License
@Override public S3Client makeNewClient(final S3Credential credentials) throws Exception { return new S3Client() { private final AtomicReference<RefCountedClient> client = new AtomicReference<RefCountedClient>(null); {//w ww .j a v a 2s. co m changeCredentials(credentials); } @Override public void changeCredentials(S3Credential credential) throws Exception { RefCountedClient newRefCountedClient = (credential != null) ? new RefCountedClient( new AmazonS3Client(new BasicAWSCredentials(credentials.getAccessKeyId(), credentials.getSecretAccessKey()))) : null; RefCountedClient oldRefCountedClient = client.getAndSet(newRefCountedClient); if (oldRefCountedClient != null) { oldRefCountedClient.markForDelete(); } } @Override public void close() throws IOException { try { changeCredentials(null); } catch (Exception e) { throw new IOException(e); } } @Override public InitiateMultipartUploadResult initiateMultipartUpload(InitiateMultipartUploadRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.initiateMultipartUpload(request); } finally { holder.release(); } } @Override public PutObjectResult putObject(PutObjectRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.putObject(request); } finally { holder.release(); } } @Override public S3Object getObject(String bucket, String key) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.getObject(bucket, key); } finally { holder.release(); } } @Override public ObjectListing listObjects(ListObjectsRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.listObjects(request); } finally { holder.release(); } } @Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.listNextBatchOfObjects(previousObjectListing); } finally { holder.release(); } } @Override public void deleteObject(String bucket, String key) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { amazonS3Client.deleteObject(bucket, key); } finally { holder.release(); } } @Override public UploadPartResult uploadPart(UploadPartRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.uploadPart(request); } finally { holder.release(); } } @Override public void completeMultipartUpload(CompleteMultipartUploadRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { amazonS3Client.completeMultipartUpload(request); } finally { holder.release(); } } @Override public void abortMultipartUpload(AbortMultipartUploadRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { amazonS3Client.abortMultipartUpload(request); } finally { holder.release(); } } }; }
From source file:com.netflix.exhibitor.core.s3.S3ClientImpl.java
License:Apache License
@Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try {//from w ww . j ava 2 s.c o m return amazonS3Client.listNextBatchOfObjects(previousObjectListing); } finally { holder.release(); } }
From source file:dashboard.AmazonLogs.java
License:Open Source License
public int readAmazonLogs(int n, String AWS_USER, String AWS_PASS, String IPfile, String ERRfile, String bucketName, String DELETE_PROCESSED_LOGS, String API_KEY, String TOKEN, String apiuser, String apipass) throws Exception { if (n < 1) return 0; int eventsNumber = 0; String line = null;/* www.j a v a 2s . c om*/ int begin = 0; int zips = 0; int deletedZips = 0; int mixpanelStatus = 0; String registrant = ""; String ip = ""; String prevIP = ""; Mixpanel mix = new Mixpanel(); Whois w = new Whois(apiuser, apipass); int index = -1; Registrant r; ArrayList<Registrant> rList = new ArrayList<Registrant>(); ArrayList<Registrant> eList = new ArrayList<Registrant>(); IPList ipl = new IPList(); IPList errl = new IPList(); // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; else { ipl.loadList(rList, IPfile); ipl.printList(rList, 30); } while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; ip = values[4]; if (ip != prevIP) { prevIP = ip; index = ipl.ipInList(ip, rList); if (index >= 0) { r = rList.get(index); registrant = r.name; // Update counter for this IP r.counter = r.counter + 1; rList.set(index, r); } else { // WHOIS - Check registrant of this IP address registrant = w.whoisIP(ip); // if there was an error, try again if (registrant.equals("ERROR")) registrant = w.whoisIP(ip); // if there was a second error, add it to errors list if (registrant.equals("ERROR")) { eList.add(new Registrant(ip, registrant, 1)); } else { // If name includes a comma, exclude the comma registrant = registrant.replace(",", ""); rList.add(new Registrant(ip, registrant, 1)); } } } String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip + " " + registrant); // ==================================================== // Track the event in Mixpanel (using the POST import) // ==================================================== mixpanelStatus = mix.postCDNEventToMixpanel(API_KEY, TOKEN, ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode, registrant); } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("========= Deleted Zip " + zips + " ===== List Size " + rList.size() + " =========="); deletedZips++; } } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 100); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 50); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; }
From source file:dashboard.ImportCDN.java
License:Open Source License
public static int readAmazonLogs(int n) throws Exception { if (n < 1) return 0; int eventsNumber = 0; int begin = 0; int zips = 0; int postFailures = 0; int deletedZips = 0; int mixpanelStatus = 0; String line = null;/*from w w w .ja v a 2 s . c o m*/ // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); // Set MARKER - from which Log File to start reading // listObjectsRequest.setMarker("E2DXXJR0N8BXOK.2013-03-18-10.ICK6IvaY.gz"); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; String ip = values[4]; String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip); // Track the event in Mixpanel (using the POST import) mixpanelStatus = postCDNEventToMixpanel(ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode); // If failed if (mixpanelStatus != 1) { postFailures++; System.out.println(" >>> POST event to Mixpanel Failed!! " + postFailures); } } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("============ Deleted Zip " + zips + " ============"); deletedZips++; } else System.out.println("============ Zip " + zips + " (not deleted) ============"); } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; }
From source file:imperial.modaclouds.monitoring.datacollectors.monitors.DetailedCostMonitor.java
License:BSD License
@Override public void run() { String accessKeyId = null;/*w ww. j a v a 2s .co m*/ String secretKey = null; ObjectListing objects = null; AmazonS3Client s3Client = null; String key = null; long startTime = 0; while (!dcmt.isInterrupted()) { if (System.currentTimeMillis() - startTime > 10000) { cost_nonspot = new HashMap<String, Double>(); cost_spot = new HashMap<String, Double>(); for (String metric : getProvidedMetrics()) { try { VM resource = new VM(Config.getInstance().getVmType(), Config.getInstance().getVmId()); if (dcAgent.shouldMonitor(resource, metric)) { Map<String, String> parameters = dcAgent.getParameters(resource, metric); accessKeyId = parameters.get("accessKey"); secretKey = parameters.get("secretKey"); bucketName = parameters.get("bucketName"); filePath = parameters.get("filePath"); period = Integer.valueOf(parameters.get("samplingTime")) * 1000; } } catch (NumberFormatException e) { e.printStackTrace(); } catch (ConfigurationException e) { e.printStackTrace(); } } startTime = System.currentTimeMillis(); AWSCredentials credentials = new BasicAWSCredentials(accessKeyId, secretKey); s3Client = new AmazonS3Client(credentials); objects = s3Client.listObjects(bucketName); key = "aws-billing-detailed-line-items-with-resources-and-tags-"; SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM"); String date = sdf.format(new Date()); key = key + date + ".csv.zip"; } String fileName = null; do { for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { System.out.println(objectSummary.getKey() + "\t" + objectSummary.getSize() + "\t" + StringUtils.fromDate(objectSummary.getLastModified())); if (objectSummary.getKey().contains(key)) { fileName = objectSummary.getKey(); s3Client.getObject(new GetObjectRequest(bucketName, fileName), new File(filePath + fileName)); break; } } objects = s3Client.listNextBatchOfObjects(objects); } while (objects.isTruncated()); try { ZipFile zipFile = new ZipFile(filePath + fileName); zipFile.extractAll(filePath); } catch (ZipException e) { e.printStackTrace(); } String csvFileName = fileName.replace(".zip", ""); AnalyseFile(filePath + csvFileName); try { Thread.sleep(period); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } } }
From source file:org.commoncrawl.service.parser.ec2.EC2ParserMaster.java
License:Open Source License
private boolean doScan(boolean initialScan) throws IOException { try {//from ww w . j a va 2 s . c o m LOG.info("Scanner Thread Starting"); AmazonS3Client s3Client = new AmazonS3Client(new BasicAWSCredentials(s3AccessKeyId, s3SecretKey)); ObjectListing response = s3Client.listObjects(new ListObjectsRequest() .withBucketName("aws-publicdatasets").withPrefix(CC_BUCKET_ROOT + CC_CRAWLLOG_SOURCE)); do { LOG.info("Response Key Count:" + response.getObjectSummaries().size()); for (S3ObjectSummary entry : response.getObjectSummaries()) { Matcher matcher = crawlLogPattern.matcher(entry.getKey()); if (matcher.matches()) { ParseCandidate candidate = ParseCandidate.candidateFromBucketEntry(entry.getKey()); if (candidate == null) { LOG.error("Failed to Parse Candidate for:" + entry.getKey()); } else { LOG.info("Candidate is:" + candidate); synchronized (this) { if (_complete.contains(candidate._crawlLogName)) { LOG.info("Skipping completed Candidate:" + candidate); } else { if (!_candidates.containsEntry(candidate._timestamp, candidate) && !_active.containsKey(candidate)) { // update candidate size here ... candidate._size = entry.getSize(); LOG.info("New Candidate:" + candidate._crawlLogName + " Found"); _candidates.put(candidate._timestamp, candidate); } else { LOG.info("Skipping Existing Candidate:" + candidate._crawlLogName); } } } } } } if (response.isTruncated()) { response = s3Client.listNextBatchOfObjects(response); } else { break; } } while (!shutdownFlag.get()); if (initialScan) { // search for completions synchronized (this) { scanForCompletions(); } } return true; } catch (IOException e) { LOG.error(CCStringUtils.stringifyException(e)); return false; } }
From source file:org.commoncrawl.service.parser.ec2.EC2ParserMaster.java
License:Open Source License
public void scanForCompletions() throws IOException { AmazonS3Client s3Client = new AmazonS3Client(new BasicAWSCredentials(s3AccessKeyId, s3SecretKey)); ObjectListing response = s3Client.listObjects(new ListObjectsRequest().withBucketName("aws-publicdatasets") .withPrefix(CC_BUCKET_ROOT + CC_PARSER_INTERMEDIATE)); do {//w w w .jav a 2 s.co m LOG.info("Response Key Count:" + response.getObjectSummaries().size()); for (S3ObjectSummary entry : response.getObjectSummaries()) { Matcher matcher = doneFilePattern.matcher(entry.getKey()); if (matcher.matches()) { ParseCandidate candidate = ParseCandidate.candidateFromBucketEntry(entry.getKey()); if (candidate == null) { LOG.error("Failed to Parse Candidate for:" + entry.getKey()); } else { long partialTimestamp = Long.parseLong(matcher.group(2)); long position = Long.parseLong(matcher.group(3)); LOG.info("Found completion for Log:" + candidate._crawlLogName + " TS:" + partialTimestamp + " Pos:" + position); candidate._lastValidPos = position; // ok lookup existing entry if present ... ParseCandidate existingCandidate = Iterables.find(_candidates.get(candidate._timestamp), Predicates.equalTo(candidate)); // if existing candidate found if (existingCandidate != null) { LOG.info("Found existing candidate with last pos:" + existingCandidate._lastValidPos); if (candidate._lastValidPos > existingCandidate._lastValidPos) { existingCandidate._lastValidPos = candidate._lastValidPos; if (candidate._lastValidPos == candidate._size) { LOG.info("Found last pos == size for candidate:" + candidate._crawlLogName + ".REMOVING FROM ACTIVE - MOVING TO COMPLETE"); _candidates.remove(candidate._timestamp, candidate); _complete.add(candidate._crawlLogName); } } } else { LOG.info("Skipping Completion for CrawlLog:" + candidate._crawlLogName + " because existing candidate was not found."); } } } } if (response.isTruncated()) { response = s3Client.listNextBatchOfObjects(response); } else { break; } } while (true); }
From source file:org.commoncrawl.util.EC2MetadataTransferUtil.java
License:Open Source License
public static List<S3ObjectSummary> getMetadataPaths(String s3AccessKeyId, String s3SecretKey, String bucketName, String segmentPath) throws IOException { AmazonS3Client s3Client = new AmazonS3Client(new BasicAWSCredentials(s3AccessKeyId, s3SecretKey)); ImmutableList.Builder<S3ObjectSummary> listBuilder = new ImmutableList.Builder<S3ObjectSummary>(); String metadataFilterKey = segmentPath + "metadata-"; LOG.info("Prefix Search Key is:" + metadataFilterKey); ObjectListing response = s3Client//from w ww.j a v a 2s .c om .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(metadataFilterKey)); do { LOG.info("Response Key Count:" + response.getObjectSummaries().size()); for (S3ObjectSummary entry : response.getObjectSummaries()) { listBuilder.add(entry); } if (response.isTruncated()) { response = s3Client.listNextBatchOfObjects(response); } else { break; } } while (true); return listBuilder.build(); }
From source file:org.commoncrawl.util.S3BulkTransferUtil.java
License:Open Source License
public static List<S3ObjectSummary> getPaths(String s3AccessKeyId, String s3SecretKey, String bucketName, String segmentPath) throws IOException { AmazonS3Client s3Client = new AmazonS3Client(new BasicAWSCredentials(s3AccessKeyId, s3SecretKey)); ImmutableList.Builder<S3ObjectSummary> listBuilder = new ImmutableList.Builder<S3ObjectSummary>(); ObjectListing response = s3Client/* ww w .ja v a2 s .co m*/ .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(segmentPath)); do { LOG.info("Response Key Count:" + response.getObjectSummaries().size()); for (S3ObjectSummary entry : response.getObjectSummaries()) { listBuilder.add(entry); } if (response.isTruncated()) { response = s3Client.listNextBatchOfObjects(response); } else { break; } } while (true); return listBuilder.build(); }