List of usage examples for com.amazonaws.services.s3.model GetObjectRequest GetObjectRequest
public GetObjectRequest(String bucketName, String key)
From source file:dashboard.ImportCDN.java
License:Open Source License
public static int readAmazonLogs(int n) throws Exception { if (n < 1) return 0; int eventsNumber = 0; int begin = 0; int zips = 0; int postFailures = 0; int deletedZips = 0; int mixpanelStatus = 0; String line = null;/* ww w .j a v a 2 s. c o m*/ // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); // Set MARKER - from which Log File to start reading // listObjectsRequest.setMarker("E2DXXJR0N8BXOK.2013-03-18-10.ICK6IvaY.gz"); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; String ip = values[4]; String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip); // Track the event in Mixpanel (using the POST import) mixpanelStatus = postCDNEventToMixpanel(ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode); // If failed if (mixpanelStatus != 1) { postFailures++; System.out.println(" >>> POST event to Mixpanel Failed!! " + postFailures); } } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("============ Deleted Zip " + zips + " ============"); deletedZips++; } else System.out.println("============ Zip " + zips + " (not deleted) ============"); } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; }
From source file:dbs.Getfroms3.java
public static void main(String[] args) throws IOException { AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(Regions.AP_SOUTH_1) .withCredentials(new AWSStaticCredentialsProvider(credentials)).build(); //AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider("inhack2hire28")); try {/*from w ww. java2 s . c o m*/ System.out.println("Downloading an object"); S3Object s3object = s3Client.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + s3object.getObjectMetadata().getContentType()); displayTextInputStream(s3object.getObjectContent()); // Get a range of bytes from an object. GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, key); rangeObjectRequest.setRange(0, 10); S3Object objectPortion = s3Client.getObject(rangeObjectRequest); System.out.println("Printing bytes retrieved."); displayTextInputStream(objectPortion.getObjectContent()); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which" + " means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means" + " the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:edu.harvard.hms.dbmi.bd2k.irct.aws.event.result.S3AfterGetResult.java
License:Mozilla Public License
@Override public void fire(Result result) { if (result.getResultStatus() != ResultStatus.AVAILABLE) { return;/*from w w w .j a v a 2 s . c o m*/ } if (!result.getResultSetLocation().startsWith("S3://")) { File temp = new File(result.getResultSetLocation()); if (temp.exists()) { return; } else { result.setResultSetLocation( "S3://" + s3Folder + result.getResultSetLocation().replaceAll(irctSaveLocation + "/", "")); } } String location = result.getResultSetLocation().substring(5); // List the files in that bucket path try { final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName) .withPrefix(location); // Loop Through all the files ListObjectsV2Result s3Files; do { s3Files = s3client.listObjectsV2(req); for (S3ObjectSummary objectSummary : s3Files.getObjectSummaries()) { // Download the files to the directory specified String keyName = objectSummary.getKey(); String fileName = irctSaveLocation + keyName.replace(location, ""); log.info("Downloading: " + keyName + " --> " + fileName); s3client.getObject(new GetObjectRequest(bucketName, keyName), new File(fileName)); } req.setContinuationToken(s3Files.getNextContinuationToken()); } while (s3Files.isTruncated() == true); // Update the result set id result.setResultSetLocation(irctSaveLocation + "/" + location.replace(s3Folder, "")); } catch (AmazonServiceException ase) { log.warn("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); log.warn("Error Message: " + ase.getMessage()); log.warn("HTTP Status Code: " + ase.getStatusCode()); log.warn("AWS Error Code: " + ase.getErrorCode()); log.warn("Error Type: " + ase.getErrorType()); log.warn("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { log.warn("Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); log.warn("Error Message: " + ace.getMessage()); } }
From source file:edu.iu.finch.core.NexRad.java
License:Apache License
/** * key eg: 2015/05/15/KVWX/KVWX20150515_080737_V06.gz * * @param key//w ww . java 2 s .c o m * @return */ private S3Object getS3Object(String key) { return s3Client.getObject(new GetObjectRequest(BUCKET_NAME, key)); }
From source file:exemplos.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/*from w w w .j a va 2 s.c o m*/ * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:fr.eurecom.hybris.kvs.drivers.AmazonKvs.java
License:Apache License
public byte[] get(String key) throws IOException { try {// ww w. ja v a 2 s. c om S3Object object = this.s3.getObject(new GetObjectRequest(this.rootContainer, key)); return ByteStreams.toByteArray(object.getObjectContent()); } catch (AmazonClientException e) { if (e instanceof AmazonS3Exception) { AmazonS3Exception as3e = (AmazonS3Exception) e; if (as3e.getStatusCode() == HttpStatus.SC_NOT_FOUND) return null; } throw new IOException(e); } }
From source file:fsi_admin.JAwsS3Conn.java
License:Open Source License
private boolean descargarArchivo(HttpServletResponse response, StringBuffer msj, AmazonS3 s3, String S3BUKT, String nombre, String destino) { //System.out.println("AwsConn DescargarArchivo:" + nombre + ":nombre"); try {//from ww w . j a v a2s.c om System.out.println("DESCARGA BUCKET: " + S3BUKT + " OBJETO: " + nombre); S3Object object = s3.getObject(new GetObjectRequest(S3BUKT, nombre)); //out.println("Content-Type: " + object.getObjectMetadata().getContentType()); //System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); byte[] byteArray = IOUtils.toByteArray(object.getObjectContent()); ByteArrayInputStream bais = new ByteArrayInputStream(byteArray); JBajarArchivo fd = new JBajarArchivo(); fd.doDownload(response, getServletConfig().getServletContext(), bais, object.getObjectMetadata().getContentType(), byteArray.length, destino); System.out.println("Content-Length: " + object.getObjectMetadata().getContentLength() + " BA: " + byteArray.length); return true; } catch (AmazonServiceException ase) { ase.printStackTrace(); msj.append("Error de AmazonServiceException al descargar archivo de S3.<br>"); msj.append("Mensaje: " + ase.getMessage() + "<br>"); msj.append("Cdigo de Estatus HTTP: " + ase.getStatusCode() + "<br>"); msj.append("Cdigo de Error AWS: " + ase.getErrorCode() + "<br>"); msj.append("Tipo de Error: " + ase.getErrorType() + "<br>"); msj.append("Request ID: " + ase.getRequestId()); return false; } catch (AmazonClientException ace) { ace.printStackTrace(); msj.append("Error de AmazonClientException al descargar archivo de S3.<br>"); msj.append("Mensaje: " + ace.getMessage()); return false; } catch (IOException ace) { ace.printStackTrace(); msj.append("Error de IOException al descargar archivo de S3.<br>"); msj.append("Mensaje: " + ace.getMessage()); return false; } }
From source file:gobblin.aws.AWSSdkClient.java
License:Apache License
/*** * Download a S3 object to local directory * * @param s3ObjectSummary S3 object summary for the object to download * @param targetDirectory Local target directory to download the object to * @throws IOException If any errors were encountered in downloading the object *//* w w w . java 2s . c o m*/ public void downloadS3Object(S3ObjectSummary s3ObjectSummary, String targetDirectory) throws IOException { final AmazonS3 amazonS3 = getS3Client(); final GetObjectRequest getObjectRequest = new GetObjectRequest(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); final S3Object s3Object = amazonS3.getObject(getObjectRequest); final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey(); FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile)); LOGGER.info("S3 object downloaded to file: " + targetFile); }
From source file:gov.usgs.cida.iplover.util.ImageStorage.java
public static byte[] get(String uuid) throws IOException { AmazonS3 s3 = prepS3Client();//from w w w .j ava 2s.c om String imageKey = KEY_BASE + "/" + uuid + ".jpg"; S3Object object = s3.getObject(new GetObjectRequest(BUCKET_NAME, imageKey)); return IOUtils.toByteArray(object.getObjectContent()); }
From source file:hu.mta.sztaki.lpds.cloud.entice.imageoptimizer.iaashandler.amazontarget.Storage.java
License:Apache License
/** * @param endpoint S3 endpoint URL//from w w w . ja v a 2s. co m * @param accessKey Access key * @param secretKey Secret key * @param bucket Bucket name * @param path Key name of the object to download (path + file name) * @param file Local file to download to * @throws Exception On any error */ public static void download(String endpoint, String accessKey, String secretKey, String bucket, String path, File file) throws Exception { AmazonS3Client amazonS3Client = null; InputStream in = null; OutputStream out = null; try { AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setMaxConnections(MAX_CONNECTIONS); clientConfiguration.setMaxErrorRetry(PredefinedRetryPolicies.DEFAULT_MAX_ERROR_RETRY); clientConfiguration.setConnectionTimeout(ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT); amazonS3Client = new AmazonS3Client(awsCredentials, clientConfiguration); S3ClientOptions clientOptions = new S3ClientOptions().withPathStyleAccess(true); amazonS3Client.setS3ClientOptions(clientOptions); amazonS3Client.setEndpoint(endpoint); S3Object object = amazonS3Client.getObject(new GetObjectRequest(bucket, path)); in = object.getObjectContent(); byte[] buf = new byte[BUFFER_SIZE]; out = new FileOutputStream(file); int count; while ((count = in.read(buf)) != -1) out.write(buf, 0, count); out.close(); in.close(); } catch (AmazonServiceException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } catch (AmazonClientException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } catch (IOException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } finally { if (in != null) { try { in.close(); } catch (Exception e) { } } if (out != null) { try { out.close(); } catch (Exception e) { } } if (amazonS3Client != null) { try { amazonS3Client.shutdown(); } catch (Exception e) { } } } }