Example usage for com.amazonaws.services.s3.model CopyObjectRequest CopyObjectRequest

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model CopyObjectRequest CopyObjectRequest.

Prototype

public CopyObjectRequest(String sourceBucketName, String sourceKey, String destinationBucketName,
        String destinationKey) 

Source Link

Document

Constructs with basic options.

Usage

From source file:alluxio.underfs.s3a.S3AUnderFileSystem.java

License:Apache License

/**
 * Copies an object to another key./*from   w  w w . j a  v  a2 s. co m*/
 *
 * @param src the source key to copy
 * @param dst the destination key to copy to
 * @return true if the operation was successful, false otherwise
 */
private boolean copy(String src, String dst) {
    src = stripPrefixIfPresent(src);
    dst = stripPrefixIfPresent(dst);
    LOG.debug("Copying {} to {}", src, dst);
    // Retry copy for a few times, in case some AWS internal errors happened during copy.
    int retries = 3;
    for (int i = 0; i < retries; i++) {
        try {
            CopyObjectRequest request = new CopyObjectRequest(mBucketName, src, mBucketName, dst);
            if (Configuration.getBoolean(PropertyKey.UNDERFS_S3A_SERVER_SIDE_ENCRYPTION_ENABLED)) {
                ObjectMetadata meta = new ObjectMetadata();
                meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
                request.setNewObjectMetadata(meta);
            }
            mManager.copy(request).waitForCopyResult();
            return true;
        } catch (AmazonClientException | InterruptedException e) {
            LOG.error("Failed to copy file {} to {}", src, dst, e);
            if (i != retries - 1) {
                LOG.error("Retrying copying file {} to {}", src, dst);
            }
        }
    }
    LOG.error("Failed to copy file {} to {}, after {} retries", src, dst, retries);
    return false;
}

---

From source file:br.com.ingenieux.mojo.aws.util.BeanstalkerS3Client.java

License:Apache License

@Override
public PutObjectResult putObject(PutObjectRequest req) throws AmazonClientException, AmazonServiceException {
    if (!multipartUpload) {
        return super.putObject(req);
    }//from   w ww  .  j a v a 2  s .  c o m

    final long contentLen = TransferManagerUtils.getContentLength(req);

    String tempFilename = req.getKey() + ".tmp";
    String origFilename = req.getKey();

    req.setKey(tempFilename);

    XProgressListener progressListener = new XProgressListener();

    req.setGeneralProgressListener(new ProgressListenerChain(progressListener));

    progressListener.setContentLen(contentLen);
    progressListener.setUpload(transferManager.upload(req));
    progressListener.setSilentUpload(silentUpload);

    try {
        progressListener.getUpload().waitForCompletion();
    } catch (InterruptedException e) {
        throw new AmazonClientException(e.getMessage(), e);
    }

    CopyObjectRequest copyReq = new CopyObjectRequest(req.getBucketName(), tempFilename, req.getBucketName(),
            origFilename);

    copyObject(copyReq);

    deleteObject(new DeleteObjectRequest(req.getBucketName(), tempFilename));

    return null;
}

---

From source file:ca.pgon.amazons3masscontenttype.App.java

License:Apache License

private static void process(ObjectListing objectListing) {
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        // Show the key
        String key = objectSummary.getKey();
        System.out.println(key);//from w  w w  .ja  v  a 2s  .c  om

        // Get the metadata and check the content type
        ObjectMetadata objectMetadata = amazonS3Client.getObjectMetadata(bucketName, key);
        System.out.println("\tCurrent content type: " + objectMetadata.getContentType());
        if (!contentType.equals(objectMetadata.getContentType())) {
            System.out.println("\tChanging content type for : " + contentType);
            objectMetadata.setContentType(contentType);

            // Get the current ACL
            AccessControlList accessControlList = amazonS3Client.getObjectAcl(bucketName, key);

            // Modify the file
            CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName, key);
            copyObjectRequest.withNewObjectMetadata(objectMetadata);
            copyObjectRequest.withAccessControlList(accessControlList);
            amazonS3Client.copyObject(copyObjectRequest);
        }
        ++count;
    }
}

---

From source file:com.altoukhov.svsync.fileviews.S3FileSpace.java

License:Apache License

@Override
public boolean moveFile(String oldPath, String newPath) {

    try {/*from  w  ww.j av  a  2  s  .  c om*/
        ObjectMetadata meta = s3.getObjectMetadata(bucketName, toAbsoluteFilePath(oldPath));
        CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, toAbsoluteFilePath(oldPath),
                bucketName, toAbsoluteFilePath(newPath));
        copyRequest.setNewObjectMetadata(meta);
        s3.copyObject(copyRequest);
        s3.deleteObject(bucketName, toAbsoluteFilePath(oldPath));
    } catch (AmazonClientException ex) {
        return false;
    }

    return true;
}

---

From source file:com.emc.ecs.sync.target.S3Target.java

License:Open Source License

@Override
public void filter(SyncObject obj) {
    try {//w  ww. ja va  2 s . c  om
        // skip the root of the bucket since it obviously exists
        if ("".equals(rootKey + obj.getRelativePath())) {
            log.debug("Target is bucket root; skipping");
            return;
        }

        // some sync objects lazy-load their metadata (i.e. AtmosSyncObject)
        // since this may be a timed operation, ensure it loads outside of other timed operations
        if (!(obj instanceof S3ObjectVersion) || !((S3ObjectVersion) obj).isDeleteMarker())
            obj.getMetadata();

        // Compute target key
        String targetKey = getTargetKey(obj);
        obj.setTargetIdentifier(AwsS3Util.fullPath(bucketName, targetKey));

        if (includeVersions) {
            ListIterator<S3ObjectVersion> sourceVersions = s3Source.versionIterator((S3SyncObject) obj);
            ListIterator<S3ObjectVersion> targetVersions = versionIterator(obj);

            boolean newVersions = false, replaceVersions = false;
            if (force) {
                replaceVersions = true;
            } else {

                // special workaround for bug where objects are listed, but they have no versions
                if (sourceVersions.hasNext()) {

                    // check count and etag/delete-marker to compare version chain
                    while (sourceVersions.hasNext()) {
                        S3ObjectVersion sourceVersion = sourceVersions.next();

                        if (targetVersions.hasNext()) {
                            S3ObjectVersion targetVersion = targetVersions.next();

                            if (sourceVersion.isDeleteMarker()) {

                                if (!targetVersion.isDeleteMarker())
                                    replaceVersions = true;
                            } else {

                                if (targetVersion.isDeleteMarker())
                                    replaceVersions = true;

                                else if (!sourceVersion.getETag().equals(targetVersion.getETag()))
                                    replaceVersions = true; // different checksum
                            }

                        } else if (!replaceVersions) { // source has new versions, but existing target versions are ok
                            newVersions = true;
                            sourceVersions.previous(); // back up one
                            putIntermediateVersions(sourceVersions, targetKey); // add any new intermediary versions (current is added below)
                        }
                    }

                    if (targetVersions.hasNext())
                        replaceVersions = true; // target has more versions

                    if (!newVersions && !replaceVersions) {
                        log.info("Source and target versions are the same. Skipping {}", obj.getRelativePath());
                        return;
                    }
                }
            }

            // something's off; must delete all versions of the object
            if (replaceVersions) {
                log.info(
                        "[{}]: version history differs between source and target; re-placing target version history with that from source.",
                        obj.getRelativePath());

                // collect versions in target
                List<DeleteObjectsRequest.KeyVersion> deleteVersions = new ArrayList<>();
                while (targetVersions.hasNext())
                    targetVersions.next(); // move cursor to end
                while (targetVersions.hasPrevious()) { // go in reverse order
                    S3ObjectVersion version = targetVersions.previous();
                    deleteVersions.add(new DeleteObjectsRequest.KeyVersion(targetKey, version.getVersionId()));
                }

                // batch delete all versions in target
                log.debug("[{}]: deleting all versions in target", obj.getRelativePath());
                s3.deleteObjects(new DeleteObjectsRequest(bucketName).withKeys(deleteVersions));

                // replay version history in target
                while (sourceVersions.hasPrevious())
                    sourceVersions.previous(); // move cursor to beginning
                putIntermediateVersions(sourceVersions, targetKey);
            }

        } else { // normal sync (no versions)
            Date sourceLastModified = obj.getMetadata().getModificationTime();
            long sourceSize = obj.getMetadata().getContentLength();

            // Get target metadata.
            ObjectMetadata destMeta = null;
            try {
                destMeta = s3.getObjectMetadata(bucketName, targetKey);
            } catch (AmazonS3Exception e) {
                if (e.getStatusCode() != 404)
                    throw new RuntimeException("Failed to check target key '" + targetKey + "' : " + e, e);
            }

            if (!force && obj.getFailureCount() == 0 && destMeta != null) {

                // Check overwrite
                Date destLastModified = destMeta.getLastModified();
                long destSize = destMeta.getContentLength();

                if (destLastModified.equals(sourceLastModified) && sourceSize == destSize) {
                    log.info("Source and target the same.  Skipping {}", obj.getRelativePath());
                    return;
                }
                if (destLastModified.after(sourceLastModified)) {
                    log.info("Target newer than source.  Skipping {}", obj.getRelativePath());
                    return;
                }
            }
        }

        // at this point we know we are going to write the object
        // Put [current object version]
        if (obj instanceof S3ObjectVersion && ((S3ObjectVersion) obj).isDeleteMarker()) {

            // object has version history, but is currently deleted
            log.debug("[{}]: deleting object in target to replicate delete marker in source.",
                    obj.getRelativePath());
            s3.deleteObject(bucketName, targetKey);
        } else {
            putObject(obj, targetKey);

            // if object has new metadata after the stream (i.e. encryption checksum), we must update S3 again
            if (obj.requiresPostStreamMetadataUpdate()) {
                log.debug("[{}]: updating metadata after sync as required", obj.getRelativePath());
                CopyObjectRequest cReq = new CopyObjectRequest(bucketName, targetKey, bucketName, targetKey);
                cReq.setNewObjectMetadata(AwsS3Util.s3MetaFromSyncMeta(obj.getMetadata()));
                s3.copyObject(cReq);
            }
        }
    } catch (Exception e) {
        throw new RuntimeException("Failed to store object: " + e, e);
    }
}

---

From source file:com.liferay.portal.store.s3.S3Store.java

License:Open Source License

protected void moveObjects(String oldPrefix, String newPrefix) throws DuplicateFileException {

    ObjectListing objectListing = _amazonS3.listObjects(_bucketName, newPrefix);

    List<S3ObjectSummary> newS3ObjectSummaries = objectListing.getObjectSummaries();

    if (!newS3ObjectSummaries.isEmpty()) {
        throw new DuplicateFileException(StringBundler
                .concat("Duplicate S3 object found when moving files from ", oldPrefix, " to ", newPrefix));
    }/*from  w  w  w  .  j a va  2s  . c o  m*/

    List<S3ObjectSummary> oldS3ObjectSummaries = getS3ObjectSummaries(oldPrefix);

    for (S3ObjectSummary s3ObjectSummary : oldS3ObjectSummaries) {
        String oldKey = s3ObjectSummary.getKey();

        String newKey = _s3KeyTransformer.moveKey(oldKey, oldPrefix, newPrefix);

        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(_bucketName, oldKey, _bucketName, newKey);

        _amazonS3.copyObject(copyObjectRequest);
    }

    for (S3ObjectSummary objectSummary : oldS3ObjectSummaries) {
        String oldKey = objectSummary.getKey();

        DeleteObjectRequest deleteObjectRequest = new DeleteObjectRequest(_bucketName, oldKey);

        _amazonS3.deleteObject(deleteObjectRequest);
    }
}

---

From source file:com.openkm.util.backup.RepositoryS3Backup.java

License:Open Source License

/**
 * Performs a recursive repository content export with metadata
 *///  w  w  w . ja v a  2 s. c  o  m
private static ImpExpStats backupHelper(String token, String fldPath, AmazonS3 s3, String bucket,
        boolean metadata, Writer out, InfoDecorator deco)
        throws FileNotFoundException, PathNotFoundException, AccessDeniedException, ParseException,
        NoSuchGroupException, RepositoryException, IOException, DatabaseException {
    log.info("backup({}, {}, {}, {}, {}, {})", new Object[] { token, fldPath, bucket, metadata, out, deco });
    ImpExpStats stats = new ImpExpStats();
    DocumentModule dm = ModuleManager.getDocumentModule();
    FolderModule fm = ModuleManager.getFolderModule();
    MetadataAdapter ma = MetadataAdapter.getInstance(token);
    Gson gson = new Gson();

    for (Iterator<Document> it = dm.getChildren(token, fldPath).iterator(); it.hasNext();) {
        File tmpDoc = null;
        InputStream is = null;
        FileOutputStream fos = null;
        boolean upload = true;

        try {
            Document docChild = it.next();
            String path = docChild.getPath().substring(1);
            ObjectMetadata objMeta = new ObjectMetadata();

            if (Config.REPOSITORY_CONTENT_CHECKSUM) {
                if (exists(s3, bucket, path)) {
                    objMeta = s3.getObjectMetadata(bucket, path);

                    if (docChild.getActualVersion().getChecksum().equals(objMeta.getETag())) {
                        upload = false;
                    }
                }
            }

            if (upload) {
                tmpDoc = FileUtils.createTempFileFromMime(docChild.getMimeType());
                fos = new FileOutputStream(tmpDoc);
                is = dm.getContent(token, docChild.getPath(), false);
                IOUtils.copy(is, fos);
                PutObjectRequest request = new PutObjectRequest(bucket, path, tmpDoc);

                if (metadata) {
                    // Metadata
                    DocumentMetadata dmd = ma.getMetadata(docChild);
                    String json = gson.toJson(dmd);
                    objMeta.addUserMetadata("okm", json);
                }

                request.setMetadata(objMeta);
                s3.putObject(request);
                out.write(deco.print(docChild.getPath(), docChild.getActualVersion().getSize(), null));
                out.flush();
            } else {
                if (metadata) {
                    // Metadata
                    DocumentMetadata dmd = ma.getMetadata(docChild);
                    String json = gson.toJson(dmd);
                    objMeta.addUserMetadata("okm", json);

                    // Update object metadata
                    CopyObjectRequest copyObjReq = new CopyObjectRequest(bucket, path, bucket, path);
                    copyObjReq.setNewObjectMetadata(objMeta);
                    s3.copyObject(copyObjReq);
                }

                log.info("Don't need to upload document {}", docChild.getPath());
            }

            // Stats
            stats.setSize(stats.getSize() + docChild.getActualVersion().getSize());
            stats.setDocuments(stats.getDocuments() + 1);
        } finally {
            IOUtils.closeQuietly(is);
            IOUtils.closeQuietly(fos);
            FileUtils.deleteQuietly(tmpDoc);
        }
    }

    for (Iterator<Folder> it = fm.getChildren(token, fldPath).iterator(); it.hasNext();) {
        InputStream is = null;

        try {
            Folder fldChild = it.next();
            String path = fldChild.getPath().substring(1) + "/";
            is = new ByteArrayInputStream(new byte[0]);
            ObjectMetadata objMeta = new ObjectMetadata();
            objMeta.setContentLength(0);
            PutObjectRequest request = new PutObjectRequest(bucket, path, is, objMeta);

            // Metadata
            if (metadata) {
                FolderMetadata fmd = ma.getMetadata(fldChild);
                String json = gson.toJson(fmd);
                objMeta.addUserMetadata("okm", json);
            }

            request.setMetadata(objMeta);
            s3.putObject(request);

            ImpExpStats tmp = backupHelper(token, fldChild.getPath(), s3, bucket, metadata, out, deco);

            // Stats
            stats.setSize(stats.getSize() + tmp.getSize());
            stats.setDocuments(stats.getDocuments() + tmp.getDocuments());
            stats.setFolders(stats.getFolders() + tmp.getFolders() + 1);
            stats.setOk(stats.isOk() && tmp.isOk());
        } finally {
            IOUtils.closeQuietly(is);
        }
    }

    log.debug("backupHelper: {}", stats);
    return stats;
}

---

From source file:com.scoyo.tools.s3cacheenhancer.S3HeaderEnhancer.java

License:Apache License

private void setHeaders(ObjectListing listing, final String maxAgeHeader, ExecutorService executorService) {

    for (final S3ObjectSummary summary : listing.getObjectSummaries()) {
        executorService.submit(new Runnable() {
            @Override/*www .  j  a va 2s  .  com*/
            public void run() {
                String bucket = summary.getBucketName();
                String key = summary.getKey();

                ObjectMetadata metadata = null;
                try {
                    metadata = s3.getObjectMetadata(bucket, key);
                } catch (AmazonS3Exception exception) {
                    System.out.println("Could not update " + key + " [" + exception.getMessage() + "]");
                    return;
                }

                if ("application/x-directory".equals(metadata.getContentType())) {
                    System.out.println("Skipping because content-type " + key);
                    return;
                }

                if (!maxAgeHeader.equals(metadata.getCacheControl())) {
                    metadata.setCacheControl(maxAgeHeader);
                } else {
                    System.out.println("Skipping because header is already correct " + key);
                    return;
                }

                AccessControlList acl = s3.getObjectAcl(summary.getBucketName(), summary.getKey());

                CopyObjectRequest copyReq = new CopyObjectRequest(bucket, key, bucket, key)
                        .withAccessControlList(acl).withNewObjectMetadata(metadata);

                CopyObjectResult result = s3.copyObject(copyReq);

                if (result != null) {
                    System.out.println("Updated " + key);
                } else {
                    System.out.println("Could not update " + key);
                }
            }
        });
    }
}

---

From source file:com.streamsets.pipeline.stage.origin.s3.AmazonS3Util.java

License:Apache License

static void move(AmazonS3Client s3Client, String srcBucket, String sourceKey, String destBucket, String destKey)
        throws AmazonClientException {
    CopyObjectRequest cp = new CopyObjectRequest(srcBucket, sourceKey, destBucket, destKey);
    s3Client.copyObject(cp);//  www .j  a v  a 2s .  c o m
    s3Client.deleteObject(new DeleteObjectRequest(srcBucket, sourceKey));
}

---

From source file:com.tango.BucketSyncer.KeyJobs.S32S3KeyCopyJob.java

License:Apache License

boolean keyCopied(ObjectMetadata sourceMetadata, AccessControlList objectAcl) {
    boolean copied = false;
    String key = summary.getKey();
    MirrorOptions options = context.getOptions();
    boolean verbose = options.isVerbose();
    int maxRetries = options.getMaxRetries();
    MirrorStats stats = context.getStats();
    for (int tries = 0; tries < maxRetries; tries++) {
        if (verbose) {
            log.info("copying (try # {}): {} to: {}", new Object[] { tries, key, keydest });
        }/* ww  w. j  a v a  2  s.c  om*/
        final CopyObjectRequest request = new CopyObjectRequest(options.getSourceBucket(), key,
                options.getDestinationBucket(), keydest);
        request.setNewObjectMetadata(sourceMetadata);
        if (options.isCrossAccountCopy()) {
            request.setCannedAccessControlList(CannedAccessControlList.BucketOwnerFullControl);
        } else {
            request.setAccessControlList(objectAcl);
        }
        try {
            stats.copyCount.incrementAndGet();
            client.copyObject(request);
            stats.bytesCopied.addAndGet(sourceMetadata.getContentLength());
            if (verbose) {
                log.info("successfully copied (on try #{}): {} to: {}", new Object[] { tries, key, keydest });
            }
            copied = true;
            break;
        } catch (AmazonS3Exception s3e) {
            //if return with 404 error, problem with bucket name
            if (s3e.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
                log.error("Failed to access S3 bucket. Check bucket name: ", s3e);
                System.exit(1);
            }
            log.error("s3 exception copying (try #{}) {} to: {}: {}",
                    new Object[] { tries, key, keydest, s3e });
        } catch (Exception e) {
            log.error("unexpected exception copying (try #{}) {} to: {}: {}",
                    new Object[] { tries, key, keydest, e });
        }
        try {
            Thread.sleep(10);
        } catch (InterruptedException e) {
            log.error("interrupted while waiting to retry key: {}: {}", key, e);
            return copied;
        }
    }
    return copied;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• »