List of usage examples for com.amazonaws.services.s3 AmazonS3Client getObjectMetadata
@Override
public ObjectMetadata getObjectMetadata(String bucketName, String key)
throws SdkClientException, AmazonServiceException
From source file:com.mweagle.tereus.aws.S3Resource.java
License:Open Source License
public boolean exists() { DefaultAWSCredentialsProviderChain credentialProviderChain = new DefaultAWSCredentialsProviderChain(); final AmazonS3Client awsClient = new AmazonS3Client(credentialProviderChain); try {/* www. ja v a2s . c om*/ awsClient.getObjectMetadata(bucketName, getS3Path()); } catch (AmazonServiceException e) { return false; } return true; }
From source file:com.netflix.exhibitor.core.s3.S3ClientImpl.java
License:Apache License
@Override public ObjectMetadata getObjectMetadata(String bucket, String key) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try {/* w w w. j a v a2s . com*/ return amazonS3Client.getObjectMetadata(bucket, key); } finally { holder.release(); } }
From source file:com.netflix.ice.common.AwsUtils.java
License:Apache License
public static boolean downloadFileIfChangedSince(String bucketName, String bucketFilePrefix, File file, long milles, String accountId, String assumeRole, String externalId) { AmazonS3Client s3Client = AwsUtils.s3Client; try {// ww w.j a va 2 s .co m if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) { Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId); s3Client = new AmazonS3Client( new BasicSessionCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey(), assumedCredentials.getSessionToken()), clientConfig); } ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName()); boolean download = !file.exists() || metadata.getLastModified().getTime() > milles; if (download) { return download(s3Client, bucketName, bucketFilePrefix + file.getName(), file); } else return download; } finally { if (s3Client != AwsUtils.s3Client) s3Client.shutdown(); } }
From source file:com.netflix.ice.common.AwsUtils.java
License:Apache License
private static boolean download(AmazonS3Client s3Client, String bucketName, String fileKey, File file) { do {/* ww w . ja v a 2s . co m*/ S3Object s3Object = s3Client.getObject(bucketName, fileKey); InputStream input = s3Object.getObjectContent(); long targetSize = s3Object.getObjectMetadata().getContentLength(); FileOutputStream output = null; boolean downloaded = false; long size = 0; try { output = new FileOutputStream(file); byte buf[] = new byte[1024000]; int len; while ((len = input.read(buf)) > 0) { output.write(buf, 0, len); size += len; } downloaded = true; } catch (IOException e) { logger.error("error in downloading " + file, e); } finally { if (input != null) try { input.close(); } catch (IOException e) { } if (output != null) try { output.close(); } catch (IOException e) { } } if (downloaded) { long contentLenth = s3Client.getObjectMetadata(bucketName, fileKey).getContentLength(); if (contentLenth != size) { logger.warn("size does not match contentLenth=" + contentLenth + " downloadSize=" + size + "targetSize=" + targetSize + " ... re-downlaoding " + fileKey); } else return true; } try { Thread.sleep(2000L); } catch (Exception e) { } } while (true); }
From source file:com.nextdoor.bender.S3SnsNotifier.java
License:Apache License
public static void main(String[] args) throws ParseException, InterruptedException, IOException { formatter = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZoneUTC(); /*/*from w w w .ja v a2s.c om*/ * Parse cli arguments */ Options options = new Options(); options.addOption(Option.builder().longOpt("bucket").hasArg().required() .desc("Name of S3 bucket to list s3 objects from").build()); options.addOption(Option.builder().longOpt("key-file").hasArg().required() .desc("Local file of S3 keys to process").build()); options.addOption( Option.builder().longOpt("sns-arn").hasArg().required().desc("SNS arn to publish to").build()); options.addOption(Option.builder().longOpt("throttle-ms").hasArg() .desc("Amount of ms to wait between publishing to SNS").build()); options.addOption(Option.builder().longOpt("processed-file").hasArg() .desc("Local file to use to store procssed S3 object names").build()); options.addOption(Option.builder().longOpt("skip-processed").hasArg(false) .desc("Whether to skip S3 objects that have been processed").build()); options.addOption( Option.builder().longOpt("dry-run").hasArg(false).desc("If set do not publish to SNS").build()); CommandLineParser parser = new DefaultParser(); CommandLine cmd = parser.parse(options, args); String bucket = cmd.getOptionValue("bucket"); String keyFile = cmd.getOptionValue("key-file"); String snsArn = cmd.getOptionValue("sns-arn"); String processedFile = cmd.getOptionValue("processed-file", null); boolean skipProcessed = cmd.hasOption("skip-processed"); dryRun = cmd.hasOption("dry-run"); long throttle = Long.parseLong(cmd.getOptionValue("throttle-ms", "-1")); if (processedFile != null) { File file = new File(processedFile); if (!file.exists()) { logger.debug("creating local file to store processed s3 object names: " + processedFile); file.createNewFile(); } } /* * Import S3 keys that have been processed */ if (skipProcessed && processedFile != null) { try (BufferedReader br = new BufferedReader(new FileReader(processedFile))) { String line; while ((line = br.readLine()) != null) { alreadyPublished.add(line.trim()); } } } /* * Setup writer for file containing processed S3 keys */ FileWriter fw = null; BufferedWriter bw = null; if (processedFile != null) { fw = new FileWriter(processedFile, true); bw = new BufferedWriter(fw); } /* * Create clients */ AmazonS3Client s3Client = new AmazonS3Client(); AmazonSNSClient snsClient = new AmazonSNSClient(); /* * Get S3 object list */ try (BufferedReader br = new BufferedReader(new FileReader(keyFile))) { String line; while ((line = br.readLine()) != null) { String key = line.trim(); if (alreadyPublished.contains(key)) { logger.info("skipping " + key); } ObjectMetadata om = s3Client.getObjectMetadata(bucket, key); S3EventNotification s3Notification = getS3Notification(key, bucket, om.getContentLength()); String json = s3Notification.toJson(); /* * Publish to SNS */ if (publish(snsArn, json, snsClient, key) && processedFile != null) { bw.write(key + "\n"); bw.flush(); } if (throttle != -1) { Thread.sleep(throttle); } } } if (processedFile != null) { bw.close(); fw.close(); } }
From source file:com.treasure_data.td_import.source.S3Source.java
License:Apache License
static List<S3ObjectSummary> getSources(AmazonS3Client client, String bucket, String basePath) { String prefix;/*w w w. j av a2s.co m*/ int index = basePath.indexOf('*'); if (index >= 0) { prefix = basePath.substring(0, index); } else { ObjectMetadata om = client.getObjectMetadata(bucket, basePath); S3ObjectSummary s3object = new S3ObjectSummary(); s3object.setBucketName(bucket); s3object.setKey(basePath); s3object.setSize(om.getContentLength()); return Arrays.asList(s3object); } LOG.info(String.format("list s3 files by client %s: bucket=%s, basePath=%s, prefix=%s", client, bucket, basePath, prefix)); List<S3ObjectSummary> s3objects = new ArrayList<S3ObjectSummary>(); String lastKey = prefix; do { ObjectListing listing = client.listObjects(new ListObjectsRequest(bucket, prefix, lastKey, null, 1024)); for (S3ObjectSummary s3object : listing.getObjectSummaries()) { s3objects.add(s3object); } lastKey = listing.getNextMarker(); } while (lastKey != null); return filterSources(s3objects, basePath); }
From source file:n3phele.storage.s3.CloudStorageImpl.java
License:Open Source License
public boolean checkExists(Repository repo, String filename) { boolean result = false; Credential credential = repo.getCredential().decrypt(); AmazonS3Client s3 = new AmazonS3Client( new BasicAWSCredentials(credential.getAccount(), credential.getSecret())); s3.setEndpoint(repo.getTarget().toString()); try {/*from ww w . j a va 2s.c o m*/ ObjectMetadata metadata = s3.getObjectMetadata(repo.getRoot(), filename); log.info("Exists " + metadata.getContentType()); return true; } catch (AmazonServiceException e) { log.log(Level.WARNING, "Service Error processing " + repo + " filename " + filename, e); } catch (AmazonClientException e) { log.log(Level.SEVERE, "Client Error processing " + repo + " filename " + filename, e); } return result; }
From source file:org.finra.dm.dao.impl.S3OperationsImpl.java
License:Apache License
/** * {@inheritDoc}//from www .j ava 2 s .c om */ @Override public ObjectMetadata getObjectMetadata(String sourceBucketName, String filePath, AmazonS3Client s3Client) { return s3Client.getObjectMetadata(sourceBucketName, filePath); }
From source file:org.openflamingo.fs.s3.S3Utils.java
License:Apache License
/** * Bucket ./*from w ww. ja v a 2s . com*/ * * @param client Amazon S3 Client * @param bucketName Bucket Name */ public static Map<String, String> getBucketInfo(AmazonS3Client client, String bucketName) { Bucket bucket = getBucket(client, bucketName); if (bucket == null) { return null; } ObjectMetadata objectMetadata = client.getObjectMetadata(bucketName, ""); Map<String, String> map = new HashMap<String, String>(); map.put("name", bucket.getName()); map.put("ownerName", bucket.getOwner().getDisplayName()); map.put("ownerId", bucket.getOwner().getId()); setValue("create", bucket.getCreationDate(), map); setValue("location", client.getBucketLocation(bucketName), map); setValue("version", objectMetadata.getVersionId(), map); setValue("contentDisposition", objectMetadata.getContentDisposition(), map); setValue("contentType", objectMetadata.getContentType(), map); setValue("etag", objectMetadata.getETag(), map); setValue("contentEncoding", objectMetadata.getContentEncoding(), map); setValue("contentLength", objectMetadata.getContentLength(), map); setValue("lastModified", objectMetadata.getLastModified(), map); return map; }