List of usage examples for com.amazonaws.services.s3 AmazonS3Client getObject
@Override
public ObjectMetadata getObject(final GetObjectRequest getObjectRequest, File destinationFile)
throws SdkClientException, AmazonServiceException
From source file:com.adobe.people.jedelson.rugsinlambda.helpers.RugWrapper.java
License:Apache License
public RugWrapper(String requestId) throws IOException { tmpRoot = new File(FileUtils.getTempDirectory().getAbsoluteFile(), requestId); tmpRoot.mkdirs();//from ww w . j a va 2 s . co m repo = new File(tmpRoot, "rugs"); File archiveFolder = new File(repo, groupId + "/" + artifactId + "/" + version); archiveFolder.mkdirs(); archive = new File(archiveFolder, artifactId + "-" + version + ".zip"); AmazonS3Client s3Client = new AmazonS3Client(); S3Object rugFile = s3Client.getObject(rugBucketName, rugObjectKey); FileUtils.copyToFile(rugFile.getObjectContent(), archive); log.info("Saved rug zip in {}", archive); }
From source file:com.netflix.exhibitor.core.s3.S3ClientFactoryImpl.java
License:Apache License
@Override public S3Client makeNewClient(final S3Credential credentials) throws Exception { return new S3Client() { private final AtomicReference<RefCountedClient> client = new AtomicReference<RefCountedClient>(null); {// ww w . j a v a 2s . co m changeCredentials(credentials); } @Override public void changeCredentials(S3Credential credential) throws Exception { RefCountedClient newRefCountedClient = (credential != null) ? new RefCountedClient( new AmazonS3Client(new BasicAWSCredentials(credentials.getAccessKeyId(), credentials.getSecretAccessKey()))) : null; RefCountedClient oldRefCountedClient = client.getAndSet(newRefCountedClient); if (oldRefCountedClient != null) { oldRefCountedClient.markForDelete(); } } @Override public void close() throws IOException { try { changeCredentials(null); } catch (Exception e) { throw new IOException(e); } } @Override public InitiateMultipartUploadResult initiateMultipartUpload(InitiateMultipartUploadRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.initiateMultipartUpload(request); } finally { holder.release(); } } @Override public PutObjectResult putObject(PutObjectRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.putObject(request); } finally { holder.release(); } } @Override public S3Object getObject(String bucket, String key) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.getObject(bucket, key); } finally { holder.release(); } } @Override public ObjectListing listObjects(ListObjectsRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.listObjects(request); } finally { holder.release(); } } @Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.listNextBatchOfObjects(previousObjectListing); } finally { holder.release(); } } @Override public void deleteObject(String bucket, String key) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { amazonS3Client.deleteObject(bucket, key); } finally { holder.release(); } } @Override public UploadPartResult uploadPart(UploadPartRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.uploadPart(request); } finally { holder.release(); } } @Override public void completeMultipartUpload(CompleteMultipartUploadRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { amazonS3Client.completeMultipartUpload(request); } finally { holder.release(); } } @Override public void abortMultipartUpload(AbortMultipartUploadRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { amazonS3Client.abortMultipartUpload(request); } finally { holder.release(); } } }; }
From source file:com.netflix.exhibitor.core.s3.S3ClientImpl.java
License:Apache License
@Override public S3Object getObject(String bucket, String key) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try {//from ww w . j a v a 2 s . co m return amazonS3Client.getObject(bucket, key); } finally { holder.release(); } }
From source file:com.netflix.ice.common.AwsUtils.java
License:Apache License
private static boolean download(AmazonS3Client s3Client, String bucketName, String fileKey, File file) { do {/*from w w w.j a v a 2 s.c om*/ S3Object s3Object = s3Client.getObject(bucketName, fileKey); InputStream input = s3Object.getObjectContent(); long targetSize = s3Object.getObjectMetadata().getContentLength(); FileOutputStream output = null; boolean downloaded = false; long size = 0; try { output = new FileOutputStream(file); byte buf[] = new byte[1024000]; int len; while ((len = input.read(buf)) > 0) { output.write(buf, 0, len); size += len; } downloaded = true; } catch (IOException e) { logger.error("error in downloading " + file, e); } finally { if (input != null) try { input.close(); } catch (IOException e) { } if (output != null) try { output.close(); } catch (IOException e) { } } if (downloaded) { long contentLenth = s3Client.getObjectMetadata(bucketName, fileKey).getContentLength(); if (contentLenth != size) { logger.warn("size does not match contentLenth=" + contentLenth + " downloadSize=" + size + "targetSize=" + targetSize + " ... re-downlaoding " + fileKey); } else return true; } try { Thread.sleep(2000L); } catch (Exception e) { } } while (true); }
From source file:com.netflix.ice.processor.BillingFileProcessor.java
License:Apache License
private Long getLastMillis(String filename) { AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); InputStream in = null;//w w w .j ava 2 s. c om try { in = s3Client.getObject(config.workS3BucketName, config.workS3BucketPrefix + filename) .getObjectContent(); return Long.parseLong(IOUtils.toString(in)); } catch (Exception e) { logger.error("Error reading from file " + filename, e); return 0L; } finally { if (in != null) try { in.close(); } catch (Exception e) { } } }
From source file:com.nextdoor.bender.config.BenderConfig.java
License:Apache License
public static BenderConfig load(AmazonS3ClientFactory s3ClientFactory, AmazonS3URI s3Uri) { AmazonS3Client s3 = s3ClientFactory.newInstance(); S3Object s3object = s3.getObject(s3Uri.getBucket(), s3Uri.getKey()); StringWriter writer = new StringWriter(); try {//ww w. j a v a 2 s . c o m IOUtils.copy(s3object.getObjectContent(), writer, "UTF-8"); } catch (IOException e) { throw new ConfigurationException("Unable to read file from s3", e); } BenderConfig config = load(s3Uri.getKey().toString(), writer.toString()); config.setConfigFile(s3Uri.getURI().toString()); return config; }
From source file:com.qubole.qds.sdk.java.client.ResultStreamer.java
License:Apache License
@VisibleForTesting protected S3Client newS3Client() throws Exception { Account account = getAccount();//from w w w. j a v a 2 s.com AWSCredentials awsCredentials = new BasicAWSCredentials(account.getStorage_access_key(), account.getStorage_secret_key()); final AmazonS3Client client = new AmazonS3Client(awsCredentials); return new S3Client() { @Override public void shutdown() { client.shutdown(); } @Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) { return client.listObjects(listObjectsRequest); } @Override public S3Object getObject(String bucket, String key) { return client.getObject(bucket, key); } }; }
From source file:com.shareplaylearn.models.UserItemManager.java
License:Open Source License
public Response getItem(String contentType, ItemSchema.PresentationType presentationType, String name, String encoding) {// ww w . j av a 2s .c o m if (encoding != null && encoding.length() > 0 && !AvailableEncodings.isAvailable(encoding)) { return Response.status(Response.Status.UNSUPPORTED_MEDIA_TYPE) .entity("Inner Encoding Type: " + encoding + " not available").build(); } AmazonS3Client s3Client = new AmazonS3Client( new BasicAWSCredentials(SecretsService.amazonClientId, SecretsService.amazonClientSecret)); try { S3Object object = s3Client.getObject(ItemSchema.S3_BUCKET, getItemLocation(name, contentType, presentationType)); try (S3ObjectInputStream inputStream = object.getObjectContent()) { long contentLength = object.getObjectMetadata().getContentLength(); if (contentLength > Limits.MAX_RETRIEVE_SIZE) { throw new IOException("Object is to large: " + contentLength + " bytes."); } int bufferSize = Math.min((int) contentLength, 10 * 8192); byte[] buffer = new byte[bufferSize]; ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); int bytesRead = 0; int totalBytesRead = 0; while ((bytesRead = inputStream.read(buffer)) > 0) { outputStream.write(buffer, 0, bytesRead); totalBytesRead += bytesRead; } log.debug("GET in file resource read: " + totalBytesRead + " bytes."); if (encoding == null || encoding.length() == 0 || encoding.equals(AvailableEncodings.IDENTITY)) { return Response.status(Response.Status.OK).entity(outputStream.toByteArray()).build(); } else if (encoding.equals(AvailableEncodings.BASE64)) { return Response.status(Response.Status.OK) .entity(Base64.encodeAsString(outputStream.toByteArray())).build(); } else { return Response.status(Response.Status.UNSUPPORTED_MEDIA_TYPE) .entity("Inner Encoding Type: " + encoding + " not available").build(); } } } catch (Exception e) { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); pw.println("\nFailed to retrieve: " + name); e.printStackTrace(pw); log.warn("Failed to retrieve: " + name); log.info(Exceptions.asString(e)); return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(sw.toString()).build(); } }
From source file:com.streamsets.pipeline.stage.origin.s3.AmazonS3Util.java
License:Apache License
static S3Object getObject(AmazonS3Client s3Client, String bucket, String objectKey) throws AmazonClientException { return s3Client.getObject(bucket, objectKey); }
From source file:eu.stratosphere.nephele.fs.s3.S3DataInputStream.java
License:Apache License
/** * Constructs a new input stream which reads its data from the specified S3 object. * /* ww w . j av a 2s. c o m*/ * @param s3Client * the S3 client to connect to Amazon S3. * @param bucket * the name of the S3 bucket the object is stored in * @param object * the name of the S3 object whose content shall be read * @throws IOException * thrown if an error occurs while accessing the specified S3 object */ S3DataInputStream(final AmazonS3Client s3Client, final String bucket, final String object) throws IOException { S3Object s3o = null; try { s3o = s3Client.getObject(bucket, object); } catch (AmazonServiceException e) { throw new IOException(StringUtils.stringifyException(e)); } this.inputStream = s3o.getObjectContent(); }