List of usage examples for com.amazonaws.services.s3.model CopyObjectRequest setAccessControlList
public void setAccessControlList(AccessControlList accessControlList)
From source file:com.tango.BucketSyncer.KeyJobs.S32S3KeyCopyJob.java
License:Apache License
boolean keyCopied(ObjectMetadata sourceMetadata, AccessControlList objectAcl) { boolean copied = false; String key = summary.getKey(); MirrorOptions options = context.getOptions(); boolean verbose = options.isVerbose(); int maxRetries = options.getMaxRetries(); MirrorStats stats = context.getStats(); for (int tries = 0; tries < maxRetries; tries++) { if (verbose) { log.info("copying (try # {}): {} to: {}", new Object[] { tries, key, keydest }); }//from ww w.j av a 2 s.c om final CopyObjectRequest request = new CopyObjectRequest(options.getSourceBucket(), key, options.getDestinationBucket(), keydest); request.setNewObjectMetadata(sourceMetadata); if (options.isCrossAccountCopy()) { request.setCannedAccessControlList(CannedAccessControlList.BucketOwnerFullControl); } else { request.setAccessControlList(objectAcl); } try { stats.copyCount.incrementAndGet(); client.copyObject(request); stats.bytesCopied.addAndGet(sourceMetadata.getContentLength()); if (verbose) { log.info("successfully copied (on try #{}): {} to: {}", new Object[] { tries, key, keydest }); } copied = true; break; } catch (AmazonS3Exception s3e) { //if return with 404 error, problem with bucket name if (s3e.getStatusCode() == HttpStatus.SC_NOT_FOUND) { log.error("Failed to access S3 bucket. Check bucket name: ", s3e); System.exit(1); } log.error("s3 exception copying (try #{}) {} to: {}: {}", new Object[] { tries, key, keydest, s3e }); } catch (Exception e) { log.error("unexpected exception copying (try #{}) {} to: {}: {}", new Object[] { tries, key, keydest, e }); } try { Thread.sleep(10); } catch (InterruptedException e) { log.error("interrupted while waiting to retry key: {}: {}", key, e); return copied; } } return copied; }
From source file:io.druid.storage.s3.S3DataSegmentMover.java
License:Apache License
/** * Copies an object and after that checks that the object is present at the target location, via a separate API call. * If it is not, an exception is thrown, and the object is not deleted at the old location. This "paranoic" check * is added after it was observed that S3 may report a successful move, and the object is not found at the target * location.//from w ww. ja v a 2 s.c o m */ private void selfCheckingMove(String s3Bucket, String targetS3Bucket, String s3Path, String targetS3Path, String copyMsg) throws IOException, SegmentLoadingException { if (s3Bucket.equals(targetS3Bucket) && s3Path.equals(targetS3Path)) { log.info("No need to move file[s3://%s/%s] onto itself", s3Bucket, s3Path); return; } if (s3Client.doesObjectExist(s3Bucket, s3Path)) { final ListObjectsV2Result listResult = s3Client.listObjectsV2( new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3Path).withMaxKeys(1)); if (listResult.getKeyCount() == 0) { // should never happen throw new ISE("Unable to list object [s3://%s/%s]", s3Bucket, s3Path); } final S3ObjectSummary objectSummary = listResult.getObjectSummaries().get(0); if (objectSummary.getStorageClass() != null && StorageClass.fromValue(StringUtils.toUpperCase(objectSummary.getStorageClass())) .equals(StorageClass.Glacier)) { throw new AmazonServiceException(StringUtils.format( "Cannot move file[s3://%s/%s] of storage class glacier, skipping.", s3Bucket, s3Path)); } else { log.info("Moving file %s", copyMsg); final CopyObjectRequest copyRequest = new CopyObjectRequest(s3Bucket, s3Path, targetS3Bucket, targetS3Path); if (!config.getDisableAcl()) { copyRequest .setAccessControlList(S3Utils.grantFullControlToBucketOwner(s3Client, targetS3Bucket)); } s3Client.copyObject(copyRequest); if (!s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) { throw new IOE( "After copy was reported as successful the file doesn't exist in the target location [%s]", copyMsg); } deleteWithRetriesSilent(s3Bucket, s3Path); log.debug("Finished moving file %s", copyMsg); } } else { // ensure object exists in target location if (s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) { log.info("Not moving file [s3://%s/%s], already present in target location [s3://%s/%s]", s3Bucket, s3Path, targetS3Bucket, targetS3Path); } else { throw new SegmentLoadingException( "Unable to move file %s, not present in either source or target location", copyMsg); } } }
From source file:org.apache.druid.storage.s3.S3DataSegmentMover.java
License:Apache License
/** * Copies an object and after that checks that the object is present at the target location, via a separate API call. * If it is not, an exception is thrown, and the object is not deleted at the old location. This "paranoic" check * is added after it was observed that S3 may report a successful move, and the object is not found at the target * location.// ww w . j ava 2s . com */ private void selfCheckingMove(String s3Bucket, String targetS3Bucket, String s3Path, String targetS3Path, String copyMsg) throws IOException, SegmentLoadingException { if (s3Bucket.equals(targetS3Bucket) && s3Path.equals(targetS3Path)) { log.info("No need to move file[s3://%s/%s] onto itself", s3Bucket, s3Path); return; } if (s3Client.doesObjectExist(s3Bucket, s3Path)) { final ListObjectsV2Result listResult = s3Client.listObjectsV2( new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3Path).withMaxKeys(1)); // Using getObjectSummaries().size() instead of getKeyCount as, in some cases // it is observed that even though the getObjectSummaries returns some data // keyCount is still zero. if (listResult.getObjectSummaries().size() == 0) { // should never happen throw new ISE("Unable to list object [s3://%s/%s]", s3Bucket, s3Path); } final S3ObjectSummary objectSummary = listResult.getObjectSummaries().get(0); if (objectSummary.getStorageClass() != null && StorageClass.fromValue(StringUtils.toUpperCase(objectSummary.getStorageClass())) .equals(StorageClass.Glacier)) { throw new AmazonServiceException(StringUtils.format( "Cannot move file[s3://%s/%s] of storage class glacier, skipping.", s3Bucket, s3Path)); } else { log.info("Moving file %s", copyMsg); final CopyObjectRequest copyRequest = new CopyObjectRequest(s3Bucket, s3Path, targetS3Bucket, targetS3Path); if (!config.getDisableAcl()) { copyRequest .setAccessControlList(S3Utils.grantFullControlToBucketOwner(s3Client, targetS3Bucket)); } s3Client.copyObject(copyRequest); if (!s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) { throw new IOE( "After copy was reported as successful the file doesn't exist in the target location [%s]", copyMsg); } deleteWithRetriesSilent(s3Bucket, s3Path); log.debug("Finished moving file %s", copyMsg); } } else { // ensure object exists in target location if (s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) { log.info("Not moving file [s3://%s/%s], already present in target location [s3://%s/%s]", s3Bucket, s3Path, targetS3Bucket, targetS3Path); } else { throw new SegmentLoadingException( "Unable to move file %s, not present in either source or target location", copyMsg); } } }