Example usage for com.amazonaws.services.s3.model PutObjectRequest setMetadata

List of usage examples for com.amazonaws.services.s3.model PutObjectRequest setMetadata

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. PutObjectRequest
  6. setMetadata

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model PutObjectRequest setMetadata.

Prototype

public void setMetadata(ObjectMetadata metadata)

Source Link

Document

Sets the optional metadata instructing Amazon S3 how to handle the uploaded data (e.g.

Usage

From source file:S3DataManager.java

License:Open Source License

public UploadToS3Output uploadSourceToS3(AbstractBuild build, Launcher launcher, BuildListener listener)
        throws Exception {
    Validation.checkS3SourceUploaderConfig(projectName, workspace);

    SCM scm = build.getProject().getScm();
    if (scm.getType().equals("hudson.scm.NullSCM")) {
        throw new Exception("Select a valid option in Source Code Management.");
    }/*from   w  w  w . j  ava2 s .  com*/
    scm.checkout(build, launcher, workspace, listener, null, null);
    String localfileName = this.projectName + "-" + "source.zip";
    String sourceFilePath = workspace.getRemote();
    String zipFilePath = sourceFilePath.substring(0, sourceFilePath.lastIndexOf("/")) + "/" + localfileName;
    File zipFile = new File(zipFilePath);

    if (!zipFile.getParentFile().exists()) {
        boolean dirMade = zipFile.getParentFile().mkdirs();
        if (!dirMade) {
            throw new Exception("Unable to create directory: " + zipFile.getParentFile().getAbsolutePath());
        }
    }

    ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFilePath));
    try {
        zipSource(sourceFilePath, out, sourceFilePath);
    } finally {
        out.close();
    }

    File sourceZipFile = new File(zipFilePath);
    PutObjectRequest putObjectRequest = new PutObjectRequest(s3InputBucket, s3InputKey, sourceZipFile);

    // Add MD5 checksum as S3 Object metadata
    String zipFileMD5;
    try (FileInputStream fis = new FileInputStream(zipFilePath)) {
        zipFileMD5 = new String(org.apache.commons.codec.binary.Base64.encodeBase64(DigestUtils.md5(fis)),
                "UTF-8");
    }
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentMD5(zipFileMD5);
    objectMetadata.setContentLength(sourceZipFile.length());
    putObjectRequest.setMetadata(objectMetadata);

    LoggingHelper.log(listener, "Uploading code to S3 at location " + putObjectRequest.getBucketName() + "/"
            + putObjectRequest.getKey() + ". MD5 checksum is " + zipFileMD5);
    PutObjectResult putObjectResult = s3Client.putObject(putObjectRequest);

    return new UploadToS3Output(putObjectRequest.getBucketName() + "/" + putObjectRequest.getKey(),
            putObjectResult.getVersionId());
}

From source file:ch.entwine.weblounge.maven.S3DeployMojo.java

License:Open Source License

/**
 * /*w  w w.j a  va  2 s .co m*/
 * {@inheritDoc}
 * 
 * @see org.apache.maven.plugin.Mojo#execute()
 */
public void execute() throws MojoExecutionException, MojoFailureException {

    // Setup AWS S3 client
    AWSCredentials credentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey);
    AmazonS3Client uploadClient = new AmazonS3Client(credentials);
    TransferManager transfers = new TransferManager(credentials);

    // Make sure key prefix does not start with a slash but has one at the
    // end
    if (keyPrefix.startsWith("/"))
        keyPrefix = keyPrefix.substring(1);
    if (!keyPrefix.endsWith("/"))
        keyPrefix = keyPrefix + "/";

    // Keep track of how much data has been transferred
    long totalBytesTransferred = 0L;
    int items = 0;
    Queue<Upload> uploads = new LinkedBlockingQueue<Upload>();

    try {
        // Check if S3 bucket exists
        getLog().debug("Checking whether bucket " + bucket + " exists");
        if (!uploadClient.doesBucketExist(bucket)) {
            getLog().error("Desired bucket '" + bucket + "' does not exist!");
            return;
        }

        getLog().debug("Collecting files to transfer from " + resources.getDirectory());
        List<File> res = getResources();
        for (File file : res) {
            // Make path of resource relative to resources directory
            String filename = file.getName();
            String extension = FilenameUtils.getExtension(filename);
            String path = file.getPath().substring(resources.getDirectory().length());
            String key = concat("/", keyPrefix, path).substring(1);

            // Delete old file version in bucket
            getLog().debug("Removing existing object at " + key);
            uploadClient.deleteObject(bucket, key);

            // Setup meta data
            ObjectMetadata meta = new ObjectMetadata();
            meta.setCacheControl("public, max-age=" + String.valueOf(valid * 3600));

            FileInputStream fis = null;
            GZIPOutputStream gzipos = null;
            final File fileToUpload;

            if (gzip && ("js".equals(extension) || "css".equals(extension))) {
                try {
                    fis = new FileInputStream(file);
                    File gzFile = File.createTempFile(file.getName(), null);
                    gzipos = new GZIPOutputStream(new FileOutputStream(gzFile));
                    IOUtils.copy(fis, gzipos);
                    fileToUpload = gzFile;
                    meta.setContentEncoding("gzip");
                    if ("js".equals(extension))
                        meta.setContentType("text/javascript");
                    if ("css".equals(extension))
                        meta.setContentType("text/css");
                } catch (FileNotFoundException e) {
                    getLog().error(e);
                    continue;
                } catch (IOException e) {
                    getLog().error(e);
                    continue;
                } finally {
                    IOUtils.closeQuietly(fis);
                    IOUtils.closeQuietly(gzipos);
                }
            } else {
                fileToUpload = file;
            }

            // Do a random check for existing errors before starting the next upload
            if (erroneousUpload != null)
                break;

            // Create put object request
            long bytesToTransfer = fileToUpload.length();
            totalBytesTransferred += bytesToTransfer;
            PutObjectRequest request = new PutObjectRequest(bucket, key, fileToUpload);
            request.setProgressListener(new UploadListener(credentials, bucket, key, bytesToTransfer));
            request.setMetadata(meta);

            // Schedule put object request
            getLog().info(
                    "Uploading " + key + " (" + FileUtils.byteCountToDisplaySize((int) bytesToTransfer) + ")");
            Upload upload = transfers.upload(request);
            uploads.add(upload);
            items++;
        }
    } catch (AmazonServiceException e) {
        getLog().error("Uploading resources failed: " + e.getMessage());
    } catch (AmazonClientException e) {
        getLog().error("Uploading resources failed: " + e.getMessage());
    }

    // Wait for uploads to be finished
    String currentUpload = null;
    try {
        Thread.sleep(1000);
        getLog().info("Waiting for " + uploads.size() + " uploads to finish...");
        while (!uploads.isEmpty()) {
            Upload upload = uploads.poll();
            currentUpload = upload.getDescription().substring("Uploading to ".length());
            if (TransferState.InProgress.equals(upload.getState()))
                getLog().debug("Waiting for upload " + currentUpload + " to finish");
            upload.waitForUploadResult();
        }
    } catch (AmazonServiceException e) {
        throw new MojoExecutionException("Error while uploading " + currentUpload);
    } catch (AmazonClientException e) {
        throw new MojoExecutionException("Error while uploading " + currentUpload);
    } catch (InterruptedException e) {
        getLog().debug("Interrupted while waiting for upload to finish");
    }

    // Check for errors that happened outside of the actual uploading
    if (erroneousUpload != null) {
        throw new MojoExecutionException("Error while uploading " + erroneousUpload);
    }

    getLog().info("Deployed " + items + " files ("
            + FileUtils.byteCountToDisplaySize((int) totalBytesTransferred) + ") to s3://" + bucket);
}

From source file:cloudExplorer.Put.java

License:Open Source License

public void run() {
    try {//from  www  .j  a v  a 2  s  . com
        AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
        AmazonS3 s3Client = new AmazonS3Client(credentials,
                new ClientConfiguration().withSignerOverride("S3SignerType"));
        s3Client.setEndpoint(endpoint);
        TransferManager tx = new TransferManager(s3Client);
        File file = new File(what);
        PutObjectRequest putRequest;
        if (!rrs) {
            putRequest = new PutObjectRequest(bucket, ObjectKey, file);
        } else {
            putRequest = new PutObjectRequest(bucket, ObjectKey, file)
                    .withStorageClass(StorageClass.ReducedRedundancy);
        }
        MimetypesFileTypeMap mimeTypesMap = new MimetypesFileTypeMap();
        String mimeType = mimeTypesMap.getContentType(file);
        mimeType = mimeTypesMap.getContentType(file);
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (encrypt) {
            objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
        }
        if ((ObjectKey.contains(".html")) || ObjectKey.contains(".txt")) {
            objectMetadata.setContentType("text/html");
        } else {
            objectMetadata.setContentType(mimeType);
        }
        long t1 = System.currentTimeMillis();
        putRequest.setMetadata(objectMetadata);
        Upload myUpload = tx.upload(putRequest);
        myUpload.waitForCompletion();
        tx.shutdownNow();
        long t2 = System.currentTimeMillis();
        long diff = t2 - t1;

        if (!mainFrame.perf) {
            if (terminal) {
                System.out.print("\nUploaded object: " + ObjectKey + " in " + diff / 1000 + " second(s).\n");
            } else {
                mainFrame.jTextArea1
                        .append("\nUploaded object: " + ObjectKey + " in " + diff / 1000 + " second(s).");
            }
        }
    } catch (AmazonServiceException ase) {
        if (NewJFrame.gui) {
            mainFrame.jTextArea1.append("\n\nError Message:    " + ase.getMessage());
            mainFrame.jTextArea1.append("\nHTTP Status Code: " + ase.getStatusCode());
            mainFrame.jTextArea1.append("\nAWS Error Code:   " + ase.getErrorCode());
            mainFrame.jTextArea1.append("\nError Type:       " + ase.getErrorType());
            mainFrame.jTextArea1.append("\nRequest ID:       " + ase.getRequestId());
            calibrate();
        } else {
            System.out.print("\n\nError Message:    " + ase.getMessage());
            System.out.print("\nHTTP Status Code: " + ase.getStatusCode());
            System.out.print("\nAWS Error Code:   " + ase.getErrorCode());
            System.out.print("\nError Type:       " + ase.getErrorType());
            System.out.print("\nRequest ID:       " + ase.getRequestId());
        }
    } catch (Exception put) {
    }

    calibrate();
}

From source file:com.flipzu.PostProcThread.java

License:Apache License

private boolean uploadToS3(Broadcast bcast, boolean delete) {
    debug.logPostProc("PostProcThread, S3 upload for " + bcast.getFilename());

    if (bcast.getFilename() == null) {
        debug.logPostProc("PostProcThread, uploadToS3, filename is null");
        return false;
    }/*from w w w  .  jav  a 2 s.  c  o  m*/

    File file = new File(bcast.getFilename());
    if (!file.exists()) {
        debug.logPostProc("PostProcThread, uploadToS3, " + bcast.getFilename() + " does not exist");
        return false;
    }

    AmazonS3 s3 = null;

    try {
        InputStream is = new FileInputStream("aws.properties");
        s3 = new AmazonS3Client(new PropertiesCredentials(is));
    } catch (Exception e) {
        Debug.getInstance().logError("uploadToS3 Error ", e);
        return false;
    }

    String bucketName = Config.getInstance().getS3Bucket();
    String dirName = Config.getInstance().getS3dir();
    String objName = dirName + "/" + bcast.getId() + Config.getInstance().getFileWriterExtension();

    PutObjectRequest po = new PutObjectRequest(bucketName, objName, file);

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentType("audio/mpeg");
    po.setMetadata(metadata);
    po.setCannedAcl(CannedAccessControlList.PublicRead);

    try {
        s3.putObject(po);
    } catch (AmazonServiceException ase) {
        debug.logPostProc("Caught an AmazonServiceException, which means your request made it "
                + "to Amazon S3, but was rejected with an error response for some reason.");
        debug.logPostProc("Error Message:    " + ase.getMessage());
        debug.logPostProc("HTTP Status Code: " + ase.getStatusCode());
        debug.logPostProc("AWS Error Code:   " + ase.getErrorCode());
        debug.logPostProc("Error Type:       " + ase.getErrorType());
        debug.logPostProc("Request ID:       " + ase.getRequestId());
        return false;

    } catch (AmazonClientException ace) {
        debug.logPostProc("Caught an AmazonClientException, which means the client encountered "
                + "a serious internal problem while trying to communicate with S3, "
                + "such as not being able to access the network.");
        debug.logPostProc("Error Message: " + ace.getMessage());
        return false;
    }

    if (delete) {
        if (Config.getInstance().deleteSmallBcasts())
            /* check and remove empty/short broadcasts */
            cleanCrappyBroadcasts(bcast.getKey(), file);

        debug.logPostProc("uploadToS3, deleting file " + bcast.getFilename());
        file.delete();
    }

    return true;

}

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

/**
 * Create a putObject request./* w  ww .j  ava  2  s .  com*/
 * Adds the ACL and metadata
 * @param key key of object
 * @param metadata metadata header
 * @param srcfile source file
 * @return the request
 */
public PutObjectRequest newPutObjectRequest(String key, ObjectMetadata metadata, File srcfile) {
    PutObjectRequest putObjectRequest = new PutObjectRequest(mBucket, key, srcfile);
    putObjectRequest.setMetadata(metadata);
    return putObjectRequest;
}

From source file:com.ibm.stocator.fs.cos.COSBlockOutputStream.java

License:Apache License

/**
 * Upload the current block as a single PUT request; if the buffer is empty a
 * 0-byte PUT will be invoked, as it is needed to create an entry at the far
 * end.//from   ww  w.  j  av  a  2 s. c o m
 *
 * @throws IOException any problem
 */
private void putObject() throws IOException {
    LOG.debug("Executing regular upload for {}", writeOperationHelper);

    final COSDataBlocks.DataBlock block = getActiveBlock();
    int size = block.dataSize();
    final COSDataBlocks.BlockUploadData uploadData = block.startUpload();
    final PutObjectRequest putObjectRequest = uploadData.hasFile()
            ? writeOperationHelper.newPutRequest(uploadData.getFile())
            : writeOperationHelper.newPutRequest(uploadData.getUploadStream(), size);

    final ObjectMetadata om = new ObjectMetadata();
    om.setUserMetadata(mMetadata);
    if (contentType != null && !contentType.isEmpty()) {
        om.setContentType(contentType);
    } else {
        om.setContentType("application/octet-stream");
    }
    putObjectRequest.setMetadata(om);
    ListenableFuture<PutObjectResult> putObjectResult = executorService.submit(new Callable<PutObjectResult>() {
        @Override
        public PutObjectResult call() throws Exception {
            PutObjectResult result;
            try {
                // the putObject call automatically closes the input
                // stream afterwards.
                result = writeOperationHelper.putObject(putObjectRequest);
            } finally {
                closeAll(LOG, uploadData, block);
            }
            return result;
        }
    });
    clearActiveBlock();
    // wait for completion
    try {
        putObjectResult.get();
    } catch (InterruptedException ie) {
        LOG.warn("Interrupted object upload", ie);
        Thread.currentThread().interrupt();
    } catch (ExecutionException ee) {
        throw extractException("regular upload", key, ee);
    }
}

From source file:com.ibm.stocator.fs.cos.COSOutputStream.java

License:Apache License

@Override
public void close() throws IOException {
    if (closed.getAndSet(true)) {
        return;/*from   w w w  .  ja  v a2  s  . c  o m*/
    }
    mBackupOutputStream.close();
    LOG.debug("OutputStream for key '{}' closed. Now beginning upload", mKey);
    try {
        final ObjectMetadata om = new ObjectMetadata();
        om.setContentLength(mBackupFile.length());
        om.setContentType(mContentType);
        om.setUserMetadata(mMetadata);

        PutObjectRequest putObjectRequest = new PutObjectRequest(mBucketName, mKey, mBackupFile);
        putObjectRequest.setMetadata(om);

        Upload upload = transfers.upload(putObjectRequest);

        upload.waitForUploadResult();
    } catch (InterruptedException e) {
        throw (InterruptedIOException) new InterruptedIOException(e.toString()).initCause(e);
    } catch (AmazonClientException e) {
        throw new IOException(String.format("saving output %s %s", mKey, e));
    } finally {
        if (!mBackupFile.delete()) {
            LOG.warn("Could not delete temporary cos file: {}", mBackupOutputStream);
        }
        super.close();
    }
    LOG.debug("OutputStream for key '{}' upload complete", mKey);
}

From source file:com.intuit.s3encrypt.S3Encrypt.java

License:Open Source License

private static void putS3Object(CommandLine cmd, AmazonS3EncryptionClient s3, String bucket, String filename,
        String keyname, String key) {
    String[] searchArgs = cmd.getOptionValues("put");
    System.out.println("Uploading a new object to S3 BUCKET = " + bucket + " FILENAME = " + filename);
    File file = new File(filename);
    PutObjectRequest request = new PutObjectRequest(bucket, filename, file);
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.addUserMetadata(keyname, key);
    request.setMetadata(metadata);
    s3.putObject(request);/*from w  w  w .ja va  2 s.c o  m*/
    System.out.println();
}

From source file:com.meteotester.util.S3Util.java

License:Open Source License

public static void saveFileToS3(File file) {
    try {//www. j  a va 2s  . com
        AWSCredentials myCredentials = new BasicAWSCredentials(Config.AWS_ACCESS_KEY, Config.AWS_SECRET_KEY);
        AmazonS3 s3client = new AmazonS3Client(myCredentials);

        s3client.setRegion(Region.getRegion(Regions.EU_WEST_1));
        String filename = file.getName();
        String path = file.getPath();

        PutObjectRequest req = new PutObjectRequest(Config.S3_BUCKETNAME, path, file);
        ObjectMetadata metadata = new ObjectMetadata();
        metadata.setContentLength(file.length());
        String contentType = (filename.contains("json")) ? "application/json" : "text/csv";
        metadata.setContentType(contentType);
        req.setMetadata(metadata);

        s3client.putObject(req);

        log.info(filename + " stored in S3");
    } catch (Exception e) {
        e.printStackTrace();
    }
}

From source file:com.openkm.util.backup.RepositoryS3Backup.java

License:Open Source License

/**
 * Performs a recursive repository content export with metadata
 *///w ww. ja  va2 s  .  c o  m
private static ImpExpStats backupHelper(String token, String fldPath, AmazonS3 s3, String bucket,
        boolean metadata, Writer out, InfoDecorator deco)
        throws FileNotFoundException, PathNotFoundException, AccessDeniedException, ParseException,
        NoSuchGroupException, RepositoryException, IOException, DatabaseException {
    log.info("backup({}, {}, {}, {}, {}, {})", new Object[] { token, fldPath, bucket, metadata, out, deco });
    ImpExpStats stats = new ImpExpStats();
    DocumentModule dm = ModuleManager.getDocumentModule();
    FolderModule fm = ModuleManager.getFolderModule();
    MetadataAdapter ma = MetadataAdapter.getInstance(token);
    Gson gson = new Gson();

    for (Iterator<Document> it = dm.getChildren(token, fldPath).iterator(); it.hasNext();) {
        File tmpDoc = null;
        InputStream is = null;
        FileOutputStream fos = null;
        boolean upload = true;

        try {
            Document docChild = it.next();
            String path = docChild.getPath().substring(1);
            ObjectMetadata objMeta = new ObjectMetadata();

            if (Config.REPOSITORY_CONTENT_CHECKSUM) {
                if (exists(s3, bucket, path)) {
                    objMeta = s3.getObjectMetadata(bucket, path);

                    if (docChild.getActualVersion().getChecksum().equals(objMeta.getETag())) {
                        upload = false;
                    }
                }
            }

            if (upload) {
                tmpDoc = FileUtils.createTempFileFromMime(docChild.getMimeType());
                fos = new FileOutputStream(tmpDoc);
                is = dm.getContent(token, docChild.getPath(), false);
                IOUtils.copy(is, fos);
                PutObjectRequest request = new PutObjectRequest(bucket, path, tmpDoc);

                if (metadata) {
                    // Metadata
                    DocumentMetadata dmd = ma.getMetadata(docChild);
                    String json = gson.toJson(dmd);
                    objMeta.addUserMetadata("okm", json);
                }

                request.setMetadata(objMeta);
                s3.putObject(request);
                out.write(deco.print(docChild.getPath(), docChild.getActualVersion().getSize(), null));
                out.flush();
            } else {
                if (metadata) {
                    // Metadata
                    DocumentMetadata dmd = ma.getMetadata(docChild);
                    String json = gson.toJson(dmd);
                    objMeta.addUserMetadata("okm", json);

                    // Update object metadata
                    CopyObjectRequest copyObjReq = new CopyObjectRequest(bucket, path, bucket, path);
                    copyObjReq.setNewObjectMetadata(objMeta);
                    s3.copyObject(copyObjReq);
                }

                log.info("Don't need to upload document {}", docChild.getPath());
            }

            // Stats
            stats.setSize(stats.getSize() + docChild.getActualVersion().getSize());
            stats.setDocuments(stats.getDocuments() + 1);
        } finally {
            IOUtils.closeQuietly(is);
            IOUtils.closeQuietly(fos);
            FileUtils.deleteQuietly(tmpDoc);
        }
    }

    for (Iterator<Folder> it = fm.getChildren(token, fldPath).iterator(); it.hasNext();) {
        InputStream is = null;

        try {
            Folder fldChild = it.next();
            String path = fldChild.getPath().substring(1) + "/";
            is = new ByteArrayInputStream(new byte[0]);
            ObjectMetadata objMeta = new ObjectMetadata();
            objMeta.setContentLength(0);
            PutObjectRequest request = new PutObjectRequest(bucket, path, is, objMeta);

            // Metadata
            if (metadata) {
                FolderMetadata fmd = ma.getMetadata(fldChild);
                String json = gson.toJson(fmd);
                objMeta.addUserMetadata("okm", json);
            }

            request.setMetadata(objMeta);
            s3.putObject(request);

            ImpExpStats tmp = backupHelper(token, fldChild.getPath(), s3, bucket, metadata, out, deco);

            // Stats
            stats.setSize(stats.getSize() + tmp.getSize());
            stats.setDocuments(stats.getDocuments() + tmp.getDocuments());
            stats.setFolders(stats.getFolders() + tmp.getFolders() + 1);
            stats.setOk(stats.isOk() && tmp.isOk());
        } finally {
            IOUtils.closeQuietly(is);
        }
    }

    log.debug("backupHelper: {}", stats);
    return stats;
}