Example usage for com.amazonaws.services.s3.transfer TransferManager upload

List of usage examples for com.amazonaws.services.s3.transfer TransferManager upload

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.transfer.*
  5. TransferManager
  6. upload

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.transfer TransferManager upload.

Prototype

public Upload upload(final PutObjectRequest putObjectRequest)
        throws AmazonServiceException, AmazonClientException

Source Link

Document

Schedules a new transfer to upload data to Amazon S3.

Usage

From source file:br.puc_rio.ele.lvc.interimage.common.udf.ROIStorage.java

License:Apache License

/**
  * Method invoked on every tuple during foreach evaluation.
  * @param input tuple<br>//from  w  w w. j  av  a  2 s  .c o  m
  * first column is assumed to have the geometry<br>
  * second column is assumed to have the class name<br>
  * third column is assumed to have the output path
  * @exception java.io.IOException
  * @return true if successful, false otherwise
  */
@Override
public Boolean exec(Tuple input) throws IOException {
    if (input == null || input.size() < 3)
        return null;

    try {

        Object objGeometry = input.get(0);
        Geometry geometry = _geometryParser.parseGeometry(objGeometry);
        String className = DataType.toString(input.get(1));
        String path = DataType.toString(input.get(2));

        AWSCredentials credentials = new BasicAWSCredentials(_accessKey, _secretKey);
        AmazonS3 conn = new AmazonS3Client(credentials);
        conn.setEndpoint("https://s3.amazonaws.com");

        /*File temp = File.createTempFile(className, ".wkt");
                
         // Delete temp file when program exits.
         temp.deleteOnExit();
                     
         BufferedWriter out = new BufferedWriter(new FileWriter(temp));
         out.write(new WKTWriter().write(geometry));
         out.close();*/

        /*
                
        File temp = File.createTempFile(className, ".wkt.snappy");
                   
        temp.deleteOnExit();*/

        String geom = new WKTWriter().write(geometry);

        ByteArrayOutputStream out = new ByteArrayOutputStream();

        OutputStream snappyOut = new SnappyOutputStream(out);
        snappyOut.write(geom.getBytes());
        snappyOut.close();

        /*PutObjectRequest putObjectRequest = new PutObjectRequest(_bucket, path + className + ".wkt.snappy", temp);
        putObjectRequest.withCannedAcl(CannedAccessControlList.PublicRead); // public for all*/

        PutObjectRequest putObjectRequest = new PutObjectRequest(_bucket, path + className + ".wkts",
                new ByteArrayInputStream(out.toByteArray()), new ObjectMetadata());
        putObjectRequest.withCannedAcl(CannedAccessControlList.PublicRead); // public for all

        TransferManager tx = new TransferManager(credentials);
        tx.upload(putObjectRequest);

        return true;

    } catch (Exception e) {
        throw new IOException("Caught exception processing input row ", e);
    }
}

From source file:ch.entwine.weblounge.maven.S3DeployMojo.java

License:Open Source License

/**
 * //from www . j av a2  s. c o m
 * {@inheritDoc}
 * 
 * @see org.apache.maven.plugin.Mojo#execute()
 */
public void execute() throws MojoExecutionException, MojoFailureException {

    // Setup AWS S3 client
    AWSCredentials credentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey);
    AmazonS3Client uploadClient = new AmazonS3Client(credentials);
    TransferManager transfers = new TransferManager(credentials);

    // Make sure key prefix does not start with a slash but has one at the
    // end
    if (keyPrefix.startsWith("/"))
        keyPrefix = keyPrefix.substring(1);
    if (!keyPrefix.endsWith("/"))
        keyPrefix = keyPrefix + "/";

    // Keep track of how much data has been transferred
    long totalBytesTransferred = 0L;
    int items = 0;
    Queue<Upload> uploads = new LinkedBlockingQueue<Upload>();

    try {
        // Check if S3 bucket exists
        getLog().debug("Checking whether bucket " + bucket + " exists");
        if (!uploadClient.doesBucketExist(bucket)) {
            getLog().error("Desired bucket '" + bucket + "' does not exist!");
            return;
        }

        getLog().debug("Collecting files to transfer from " + resources.getDirectory());
        List<File> res = getResources();
        for (File file : res) {
            // Make path of resource relative to resources directory
            String filename = file.getName();
            String extension = FilenameUtils.getExtension(filename);
            String path = file.getPath().substring(resources.getDirectory().length());
            String key = concat("/", keyPrefix, path).substring(1);

            // Delete old file version in bucket
            getLog().debug("Removing existing object at " + key);
            uploadClient.deleteObject(bucket, key);

            // Setup meta data
            ObjectMetadata meta = new ObjectMetadata();
            meta.setCacheControl("public, max-age=" + String.valueOf(valid * 3600));

            FileInputStream fis = null;
            GZIPOutputStream gzipos = null;
            final File fileToUpload;

            if (gzip && ("js".equals(extension) || "css".equals(extension))) {
                try {
                    fis = new FileInputStream(file);
                    File gzFile = File.createTempFile(file.getName(), null);
                    gzipos = new GZIPOutputStream(new FileOutputStream(gzFile));
                    IOUtils.copy(fis, gzipos);
                    fileToUpload = gzFile;
                    meta.setContentEncoding("gzip");
                    if ("js".equals(extension))
                        meta.setContentType("text/javascript");
                    if ("css".equals(extension))
                        meta.setContentType("text/css");
                } catch (FileNotFoundException e) {
                    getLog().error(e);
                    continue;
                } catch (IOException e) {
                    getLog().error(e);
                    continue;
                } finally {
                    IOUtils.closeQuietly(fis);
                    IOUtils.closeQuietly(gzipos);
                }
            } else {
                fileToUpload = file;
            }

            // Do a random check for existing errors before starting the next upload
            if (erroneousUpload != null)
                break;

            // Create put object request
            long bytesToTransfer = fileToUpload.length();
            totalBytesTransferred += bytesToTransfer;
            PutObjectRequest request = new PutObjectRequest(bucket, key, fileToUpload);
            request.setProgressListener(new UploadListener(credentials, bucket, key, bytesToTransfer));
            request.setMetadata(meta);

            // Schedule put object request
            getLog().info(
                    "Uploading " + key + " (" + FileUtils.byteCountToDisplaySize((int) bytesToTransfer) + ")");
            Upload upload = transfers.upload(request);
            uploads.add(upload);
            items++;
        }
    } catch (AmazonServiceException e) {
        getLog().error("Uploading resources failed: " + e.getMessage());
    } catch (AmazonClientException e) {
        getLog().error("Uploading resources failed: " + e.getMessage());
    }

    // Wait for uploads to be finished
    String currentUpload = null;
    try {
        Thread.sleep(1000);
        getLog().info("Waiting for " + uploads.size() + " uploads to finish...");
        while (!uploads.isEmpty()) {
            Upload upload = uploads.poll();
            currentUpload = upload.getDescription().substring("Uploading to ".length());
            if (TransferState.InProgress.equals(upload.getState()))
                getLog().debug("Waiting for upload " + currentUpload + " to finish");
            upload.waitForUploadResult();
        }
    } catch (AmazonServiceException e) {
        throw new MojoExecutionException("Error while uploading " + currentUpload);
    } catch (AmazonClientException e) {
        throw new MojoExecutionException("Error while uploading " + currentUpload);
    } catch (InterruptedException e) {
        getLog().debug("Interrupted while waiting for upload to finish");
    }

    // Check for errors that happened outside of the actual uploading
    if (erroneousUpload != null) {
        throw new MojoExecutionException("Error while uploading " + erroneousUpload);
    }

    getLog().info("Deployed " + items + " files ("
            + FileUtils.byteCountToDisplaySize((int) totalBytesTransferred) + ") to s3://" + bucket);
}

From source file:cloudExplorer.Put.java

License:Open Source License

public void run() {
    try {/*from  w  w  w .java 2 s. co  m*/
        AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
        AmazonS3 s3Client = new AmazonS3Client(credentials,
                new ClientConfiguration().withSignerOverride("S3SignerType"));
        s3Client.setEndpoint(endpoint);
        TransferManager tx = new TransferManager(s3Client);
        File file = new File(what);
        PutObjectRequest putRequest;
        if (!rrs) {
            putRequest = new PutObjectRequest(bucket, ObjectKey, file);
        } else {
            putRequest = new PutObjectRequest(bucket, ObjectKey, file)
                    .withStorageClass(StorageClass.ReducedRedundancy);
        }
        MimetypesFileTypeMap mimeTypesMap = new MimetypesFileTypeMap();
        String mimeType = mimeTypesMap.getContentType(file);
        mimeType = mimeTypesMap.getContentType(file);
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (encrypt) {
            objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
        }
        if ((ObjectKey.contains(".html")) || ObjectKey.contains(".txt")) {
            objectMetadata.setContentType("text/html");
        } else {
            objectMetadata.setContentType(mimeType);
        }
        long t1 = System.currentTimeMillis();
        putRequest.setMetadata(objectMetadata);
        Upload myUpload = tx.upload(putRequest);
        myUpload.waitForCompletion();
        tx.shutdownNow();
        long t2 = System.currentTimeMillis();
        long diff = t2 - t1;

        if (!mainFrame.perf) {
            if (terminal) {
                System.out.print("\nUploaded object: " + ObjectKey + " in " + diff / 1000 + " second(s).\n");
            } else {
                mainFrame.jTextArea1
                        .append("\nUploaded object: " + ObjectKey + " in " + diff / 1000 + " second(s).");
            }
        }
    } catch (AmazonServiceException ase) {
        if (NewJFrame.gui) {
            mainFrame.jTextArea1.append("\n\nError Message:    " + ase.getMessage());
            mainFrame.jTextArea1.append("\nHTTP Status Code: " + ase.getStatusCode());
            mainFrame.jTextArea1.append("\nAWS Error Code:   " + ase.getErrorCode());
            mainFrame.jTextArea1.append("\nError Type:       " + ase.getErrorType());
            mainFrame.jTextArea1.append("\nRequest ID:       " + ase.getRequestId());
            calibrate();
        } else {
            System.out.print("\n\nError Message:    " + ase.getMessage());
            System.out.print("\nHTTP Status Code: " + ase.getStatusCode());
            System.out.print("\nAWS Error Code:   " + ase.getErrorCode());
            System.out.print("\nError Type:       " + ase.getErrorType());
            System.out.print("\nRequest ID:       " + ase.getRequestId());
        }
    } catch (Exception put) {
    }

    calibrate();
}

From source file:com.cloudbees.plugins.binarydeployer.s3.S3Repository.java

License:Open Source License

@Override
protected void deploy(List<Binary> binaries, Run run) throws IOException {
    log.debug("Will deploy files to S3::{}" + bucketName);
    AWSCredentialsImpl credentials = CredentialsProvider.findCredentialById(credentialsId,
            AWSCredentialsImpl.class, run, Lists.<DomainRequirement>newArrayList());

    TransferManager transferManager = new TransferManager(credentials);
    for (Binary binary : binaries) {
        transferManager.upload(prepareUpload(binary.getFile(), binary.getName()));
    }//from ww w  . j a v a2s .  com
}

From source file:com.emc.ecs.sync.target.S3Target.java

License:Open Source License

protected void putObject(SyncObject obj, String targetKey) {
    ObjectMetadata om = AwsS3Util.s3MetaFromSyncMeta(obj.getMetadata());
    if (obj.isDirectory())
        om.setContentType(AwsS3Util.TYPE_DIRECTORY);

    PutObjectRequest req;//w ww  . j  av a2s .c om
    if (obj.isDirectory()) {
        req = new PutObjectRequest(bucketName, targetKey, new ByteArrayInputStream(new byte[0]), om);
    } else if (obj instanceof FileSyncObject) {
        req = new PutObjectRequest(bucketName, targetKey, ((FileSyncObject) obj).getRawSourceIdentifier());
    } else {
        req = new PutObjectRequest(bucketName, targetKey, obj.getInputStream(), om);
    }

    if (includeAcl)
        req.setAccessControlList(AwsS3Util.s3AclFromSyncAcl(obj.getMetadata().getAcl(), ignoreInvalidAcls));

    // xfer manager will figure out if MPU is needed (based on threshold), do the MPU if necessary,
    // and abort if it fails
    TransferManagerConfiguration xferConfig = new TransferManagerConfiguration();
    xferConfig.setMultipartUploadThreshold((long) mpuThresholdMB * 1024 * 1024);
    xferConfig.setMinimumUploadPartSize((long) mpuPartSizeMB * 1024 * 1024);
    TransferManager xferManager = new TransferManager(s3, Executors.newFixedThreadPool(mpuThreadCount));
    xferManager.setConfiguration(xferConfig);

    Upload upload = xferManager.upload(req);
    try {
        log.debug("Wrote {}, etag: {}", targetKey, upload.waitForUploadResult().getETag());
    } catch (InterruptedException e) {
        throw new RuntimeException("upload thread was interrupted", e);
    } finally {
        // make sure bytes read is accurate if we bypassed the counting stream
        if (obj instanceof FileSyncObject) {
            try {
                ((FileSyncObject) obj).setOverrideBytesRead(upload.getProgress().getBytesTransferred());
            } catch (Throwable t) {
                log.warn("could not get bytes transferred from upload", t);
            }
        }
    }
}

From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java

License:Open Source License

@Override
public boolean uploadObjectAndListenProgress(final String bucketName, final String fileName,
        final InputStream inputStream, final CannedAccessControlList cannedAcl)
        throws AmazonClientException, AmazonServiceException, IOException {
    LOGGER.info(//w ww  . jav a  2s.  c om
            "uploadObjectAndListenProgress invoked, bucketName: {} , fileName: {} and cannedAccessControlList: {}",
            bucketName, fileName, cannedAcl);
    File tempFile = null;
    PutObjectRequest putObjectRequest = null;
    Upload upload = null;
    try {
        // Create temporary file from stream to avoid 'out of memory' exception
        tempFile = AWSUtil.createTempFileFromStream(inputStream);
        putObjectRequest = new PutObjectRequest(bucketName, fileName, tempFile).withCannedAcl(cannedAcl);
        final TransferManager transferMgr = new TransferManager(s3client);
        upload = transferMgr.upload(putObjectRequest);
        // You can poll your transfer's status to check its progress
        if (upload.isDone()) {
            LOGGER.info("Start: {}  , State: {} and Progress (%): {}", upload.getDescription(),
                    upload.getState(), upload.getProgress().getPercentTransferred());
        }

        // Add progressListener to listen asynchronous notifications about your transfer's progress
        // Uncomment below code snippet during development
        /*upload.addProgressListener(new ProgressListener() {
           public void progressChanged(ProgressEvent event) {
              if (LOGGER.isDebugEnabled()) {
          LOGGER.debug("Transferred bytes: " + (long) event.getBytesTransferred());
              }
        }
        });*/

        try {
            //Block the current thread and wait for completion
            //If the transfer fails AmazonClientException will be thrown
            upload.waitForCompletion();
        } catch (AmazonClientException | InterruptedException excp) {
            LOGGER.error("Exception occured while waiting for transfer: ", excp);
        }
    } finally {
        AWSUtil.deleteTempFile(tempFile); // Delete the temporary file once uploaded
    }
    LOGGER.info("End: {} , State: {} , Progress (%): {}", upload.getDescription(), upload.getState(),
            upload.getProgress().getPercentTransferred());
    return upload.isDone();
}

From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java

License:Open Source License

@Override
public boolean uploadObjectAndListenProgress(final String bucketName, final String fileName,
        final InputStream inputStream, final boolean isPublicAccessible)
        throws AmazonClientException, AmazonServiceException, IOException {
    LOGGER.info(//from w ww .jav  a  2  s .c om
            "uploadObjectAndListenProgress invoked, bucketName: {} , fileName: {} and isPublicAccessible: {}",
            bucketName, fileName, isPublicAccessible);
    File tempFile = null;
    PutObjectRequest putObjectRequest = null;
    Upload upload = null;
    try {
        // Create temporary file from stream to avoid 'out of memory' exception
        tempFile = AWSUtil.createTempFileFromStream(inputStream);
        putObjectRequest = new PutObjectRequest(bucketName, fileName, tempFile);
        if (isPublicAccessible) {
            putObjectRequest.setCannedAcl(CannedAccessControlList.PublicRead);
        }
        final TransferManager transferMgr = new TransferManager(s3client);
        upload = transferMgr.upload(putObjectRequest);
        // You can poll your transfer's status to check its progress
        if (upload.isDone()) {
            LOGGER.info("Start: {}  , State: {} and Progress (%): {}", upload.getDescription(),
                    upload.getState(), upload.getProgress().getPercentTransferred());
        }

        // Add progressListener to listen asynchronous notifications about your transfer's progress
        // Uncomment below code snippet during development
        /*upload.addProgressListener(new ProgressListener() {
           public void progressChanged(ProgressEvent event) {
              if (LOGGER.isDebugEnabled()) {
          LOGGER.debug("Transferred bytes: " + (long) event.getBytesTransferred());
              }
        }
        });*/

        try {
            //Block the current thread and wait for completion
            //If the transfer fails AmazonClientException will be thrown
            upload.waitForCompletion();
        } catch (AmazonClientException | InterruptedException excp) {
            LOGGER.error("Exception occured while waiting for transfer: ", excp);
        }
    } finally {
        AWSUtil.deleteTempFile(tempFile); // Delete the temporary file once uploaded
    }
    LOGGER.info("End: {} , State: {} , Progress (%): {}", upload.getDescription(), upload.getState(),
            upload.getProgress().getPercentTransferred());
    return upload.isDone();
}

From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java

License:Open Source License

@Override
public Upload uploadFileAsync(final String bucketName, final String fileName, final File fileObj,
        final CannedAccessControlList cannedAcl)
        throws AmazonClientException, AmazonServiceException, IOException {
    LOGGER.info("uploadObjectAsync invoked, bucketName: {} , fileName: {} and cannedAccessControlList: {}",
            bucketName, fileName, cannedAcl);
    final PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileName, fileObj)
            .withCannedAcl(cannedAcl);//from  w  w  w  .j a v  a2s  .  c  o  m
    final TransferManager transferMgr = new TransferManager(s3client);
    return transferMgr.upload(putObjectRequest);
}

From source file:com.github.abhinavmishra14.aws.s3.service.impl.AwsS3IamServiceImpl.java

License:Open Source License

@Override
public Upload uploadFileAsync(final String bucketName, final String fileName, final File fileObj,
        final boolean isPublicAccessible) throws AmazonClientException, AmazonServiceException, IOException {
    LOGGER.info("uploadObjectAsync invoked, bucketName: {} , fileName: {} and isPublicAccessible: {}",
            bucketName, fileName, isPublicAccessible);
    final PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, fileName, fileObj);
    if (isPublicAccessible) {
        putObjectRequest.setCannedAcl(CannedAccessControlList.PublicRead);
    }/*from w  ww. ja va2  s  .c  o m*/
    final TransferManager transferMgr = new TransferManager(s3client);
    return transferMgr.upload(putObjectRequest);
}

From source file:com.github.rholder.esthree.command.Put.java

License:Apache License

@Override
public Integer call() throws Exception {
    TransferManager t = new TransferManager(amazonS3Client);

    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setUserMetadata(metadata);

    Upload u = t.upload(new PutObjectRequest(bucket, key, inputFile).withMetadata(objectMetadata));

    // TODO this listener spews out garbage >100% on a retry, add a test to verify
    if (progressListener != null) {
        progressListener.withTransferProgress(new TransferProgressWrapper(u.getProgress()));
        u.addProgressListener(progressListener);
    }/*from w  ww  . j  a v a2 s  .  c o  m*/
    try {
        u.waitForCompletion();
    } finally {
        t.shutdownNow();
    }
    return 0;
}