List of usage examples for com.amazonaws.services.s3.model GetObjectRequest GetObjectRequest
public GetObjectRequest(String bucketName, String key)
From source file:com.universal.storage.UniversalS3Storage.java
License:Open Source License
/** * This method retrieves a file from the storage as InputStream. * The method will retrieve the file according to the passed path. * A file will be stored within the settings' tmp folder. * //from www. jav a 2s . c o m * @param path in context. * @returns an InputStream pointing to the retrieved file. */ public InputStream retrieveFileAsStream(String path) throws UniversalIOException { PathValidator.validatePath(path); if ("".equals(path.trim())) { return null; } if (path.trim().endsWith("/")) { UniversalIOException error = new UniversalIOException( "Invalid path. Looks like you're trying to retrieve a folder."); this.triggerOnErrorListeners(error); throw error; } try { S3Object object = s3client.getObject(new GetObjectRequest(this.settings.getRoot(), path)); return object.getObjectContent(); } catch (Exception e) { UniversalIOException error = new UniversalIOException(e.getMessage()); this.triggerOnErrorListeners(error); throw error; } }
From source file:com.vmware.photon.controller.model.adapters.awsadapter.AWSCostStatsService.java
License:Open Source License
private void downloadAndParse(AWSCostStatsCreationContext statsData, String awsBucketname, int year, int month, AWSCostStatsCreationStages next) throws IOException { // Creating a working directory for downloanding and processing the bill final Path workingDirPath = Paths.get(System.getProperty(TEMP_DIR_LOCATION), UUID.randomUUID().toString()); Files.createDirectories(workingDirPath); String accountId = statsData.computeDesc.customProperties.getOrDefault(AWSConstants.AWS_ACCOUNT_ID_KEY, null);/* w w w . ja v a 2s.c om*/ AWSCsvBillParser parser = new AWSCsvBillParser(); final String csvBillZipFileName = parser.getCsvBillFileName(month, year, accountId, true); Path csvBillZipFilePath = Paths.get(workingDirPath.toString(), csvBillZipFileName); GetObjectRequest getObjectRequest = new GetObjectRequest(awsBucketname, csvBillZipFileName); Download download = statsData.s3Client.download(getObjectRequest, csvBillZipFilePath.toFile()); final StatelessService service = this; download.addProgressListener(new ProgressListener() { @Override public void progressChanged(ProgressEvent progressEvent) { try { ProgressEventType eventType = progressEvent.getEventType(); if (ProgressEventType.TRANSFER_COMPLETED_EVENT.equals(eventType)) { LocalDate monthDate = new LocalDate(year, month, 1); statsData.accountDetailsMap = parser.parseDetailedCsvBill(statsData.ignorableInvoiceCharge, csvBillZipFilePath, monthDate); deleteTempFiles(); OperationContext.restoreOperationContext(statsData.opContext); statsData.stage = next; handleCostStatsCreationRequest(statsData); } else if (ProgressEventType.TRANSFER_FAILED_EVENT.equals(eventType)) { deleteTempFiles(); throw new IOException("Download of AWS CSV Bill '" + csvBillZipFileName + "' failed."); } } catch (IOException e) { logSevere(e); AdapterUtils.sendFailurePatchToProvisioningTask(service, statsData.statsRequest.taskReference, e); } } private void deleteTempFiles() { try { Files.deleteIfExists(csvBillZipFilePath); Files.deleteIfExists(workingDirPath); } catch (IOException e) { // Ignore IO exception while cleaning files. } } }); }
From source file:com.yahoo.athenz.zts.store.s3.S3ChangeLogStore.java
License:Apache License
SignedDomain getSignedDomain(AmazonS3 s3, String domainName) {
SignedDomain signedDomain = null;//from w w w . j ava 2 s. c o m
try {
S3Object object = s3.getObject(new GetObjectRequest(s3BucketName, domainName));
if (object == null) {
LOGGER.error("AWSS3ChangeLog: getSignedDomain - domain not found " + domainName);
return null;
}
BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
StringBuilder data = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
data.append(line);
}
reader.close();
signedDomain = JSON.fromString(data.toString(), SignedDomain.class);
} catch (Exception ex) {
LOGGER.error("AWSS3ChangeLog: getSignedDomain - unable to get domain " + domainName + " error: "
+ ex.getMessage());
}
return signedDomain;
}
From source file:com.yahoo.ycsb.db.S3Client.java
License:Open Source License
private Map.Entry<S3Object, ObjectMetadata> getS3ObjectAndMetadata(String bucket, String key, SSECustomerKey ssecLocal) {/*from w ww . ja va2 s . c om*/ GetObjectRequest getObjectRequest; GetObjectMetadataRequest getObjectMetadataRequest; if (ssecLocal != null) { getObjectRequest = new GetObjectRequest(bucket, key).withSSECustomerKey(ssecLocal); getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key).withSSECustomerKey(ssecLocal); } else { getObjectRequest = new GetObjectRequest(bucket, key); getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key); } return new AbstractMap.SimpleEntry<>(s3Client.getObject(getObjectRequest), s3Client.getObjectMetadata(getObjectMetadataRequest)); }
From source file:com.yahoo.ycsb.utils.connection.S3Connection.java
License:Open Source License
public byte[] read(String key) throws InterruptedException { //long starttime = System.currentTimeMillis(); byte[] bytes = null; try {/* w ww . ja va2 s . com*/ GetObjectRequest getObjectRequest = null; GetObjectMetadataRequest getObjectMetadataRequest = null; if (ssecKey != null) { getObjectRequest = new GetObjectRequest(bucket, key).withSSECustomerKey(ssecKey); getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key).withSSECustomerKey(ssecKey); } else { getObjectRequest = new GetObjectRequest(bucket, key); getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key); } //System.out.println("Get object " + key + " from " + bucket); S3Object object = awsClient.getObject(getObjectRequest); ObjectMetadata objectMetadata = awsClient.getObjectMetadata(getObjectMetadataRequest); //System.out.println("Get object " + key + " from " + bucket + " OK"); InputStream objectData = object.getObjectContent(); //consuming the stream // writing the stream to bytes and to results int sizeOfFile = (int) objectMetadata.getContentLength(); bytes = new byte[sizeOfFile]; int offset = 0; while (offset < sizeOfFile) { int chunk_size; //read in 4k chunks chunk_size = sizeOfFile - offset > 4096 ? 4096 : sizeOfFile - offset; int nr_bytes_read = objectData.read(bytes, offset, sizeOfFile - offset); offset = offset + nr_bytes_read; if (Thread.interrupted()) { //System.out.println("interrupt " + key); objectData.close(); throw new InterruptedException(); } } //int nr_bytes_read = objectData.read(bytes, 0, sizeOfFile); objectData.close(); } catch (IOException e) { logger.warn("Not possible to get the object " + key); } //long endtime = System.currentTimeMillis(); //System.out.println("ReadS3: " + key + " " + (endtime - starttime) + " " + region); return bytes; }
From source file:common.S3Processor.java
License:Open Source License
public static RestaurantDO getObject(String key) { GetObjectRequest request = new GetObjectRequest(BUCKET_NAME, key); S3Object object = s3.getObject(request); ObjectMapper mapper = new ObjectMapper(); RestaurantDO actualObj;//from w w w . jav a2s. c o m try { actualObj = mapper.readValue(object.getObjectContent(), RestaurantDO.class); return actualObj; } catch (Exception e) { } return null; }
From source file:ConnectionUtils.AWSS3Utils.java
public static ImageDTO loadImage(String bucketName, ImageDTO imageDTO, IMAGE_TYPE imageType) { Properties awsCredentialsProperties = new Properties(); try {// w ww . j av a 2 s . c o m awsCredentialsProperties .load(AWSS3Utils.class.getClassLoader().getResourceAsStream("prefs.properties")); } catch (Exception ex) { Logger.getLogger(AWSS3Utils.class.getName()).log(Level.SEVERE, null, ex); } String awsAccessKey = awsCredentialsProperties.getProperty("AWSACCESSKEY"); String awsSecretKey = awsCredentialsProperties.getProperty("SECRETACCESSKEY"); try { BasicAWSCredentials awsCredentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey); AmazonS3 s3Client = new AmazonS3Client(awsCredentials); GetObjectRequest request = null; if (imageType.equals(IMAGE_TYPE.FULL)) { request = new GetObjectRequest(bucketName, imageDTO.getIID() + ".jpg"); } else if (imageType.equals(IMAGE_TYPE.THUMB)) { request = new GetObjectRequest(bucketName, imageDTO.getIID() + "_" + IMAGE_TYPE.THUMB.toString() + ".jpg"); } S3Object object = s3Client.getObject(request); S3ObjectInputStream objectContent = object.getObjectContent(); if (imageType.equals(IMAGE_TYPE.FULL)) { imageDTO.setImageBlob(objectContent); } else if (imageType.equals(IMAGE_TYPE.THUMB)) { imageDTO.setThumbnailBlob(objectContent); } return (imageDTO); } catch (Exception ex) { Logger.getLogger(AWSS3Utils.class.getName()).log(Level.SEVERE, null, ex); return null; } }
From source file:core.connector.s3.sync.S3Connector.java
License:GNU General Public License
@Override public byte[] get(String path, long size) throws ConnectorException { try {//from ww w.j a v a 2 s . c o m GetObjectRequest request = new GetObjectRequest(info.getBucketName(), path); if (size >= 0) request.withRange(0, size); return Streams.readFullyBytes(s3.getObject(request).getObjectContent()); } catch (Exception e) { throw new ConnectorException(e); } }
From source file:cz.pichlik.goodsentiment.server.repository.S3RepositoryBase.java
License:Apache License
public InputStream load(String bucket, String key) { GetObjectRequest getReq = new GetObjectRequest(bucket, key); S3Object object = s3Client.getObject(getReq); return object.getObjectContent(); }
From source file:dashboard.AmazonLogs.java
License:Open Source License
public int readAmazonLogs(int n, String AWS_USER, String AWS_PASS, String IPfile, String ERRfile, String bucketName, String DELETE_PROCESSED_LOGS, String API_KEY, String TOKEN, String apiuser, String apipass) throws Exception { if (n < 1) return 0; int eventsNumber = 0; String line = null;//from ww w .ja va 2 s . co m int begin = 0; int zips = 0; int deletedZips = 0; int mixpanelStatus = 0; String registrant = ""; String ip = ""; String prevIP = ""; Mixpanel mix = new Mixpanel(); Whois w = new Whois(apiuser, apipass); int index = -1; Registrant r; ArrayList<Registrant> rList = new ArrayList<Registrant>(); ArrayList<Registrant> eList = new ArrayList<Registrant>(); IPList ipl = new IPList(); IPList errl = new IPList(); // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; else { ipl.loadList(rList, IPfile); ipl.printList(rList, 30); } while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; ip = values[4]; if (ip != prevIP) { prevIP = ip; index = ipl.ipInList(ip, rList); if (index >= 0) { r = rList.get(index); registrant = r.name; // Update counter for this IP r.counter = r.counter + 1; rList.set(index, r); } else { // WHOIS - Check registrant of this IP address registrant = w.whoisIP(ip); // if there was an error, try again if (registrant.equals("ERROR")) registrant = w.whoisIP(ip); // if there was a second error, add it to errors list if (registrant.equals("ERROR")) { eList.add(new Registrant(ip, registrant, 1)); } else { // If name includes a comma, exclude the comma registrant = registrant.replace(",", ""); rList.add(new Registrant(ip, registrant, 1)); } } } String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip + " " + registrant); // ==================================================== // Track the event in Mixpanel (using the POST import) // ==================================================== mixpanelStatus = mix.postCDNEventToMixpanel(API_KEY, TOKEN, ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode, registrant); } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("========= Deleted Zip " + zips + " ===== List Size " + rList.size() + " =========="); deletedZips++; } } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 100); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 50); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; }