Example usage for com.amazonaws.services.s3.transfer TransferManager TransferManager

List of usage examples for com.amazonaws.services.s3.transfer TransferManager TransferManager

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.transfer.*
  5. TransferManager
  6. TransferManager

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.transfer TransferManager TransferManager.

Prototype

@Deprecated
public TransferManager(AmazonS3 s3, ExecutorService executorService)

Source Link

Document

Constructs a new TransferManager specifying the client and thread pool to use when making requests to Amazon S3.

Usage

From source file:com.emc.ecs.sync.target.S3Target.java

License:Open Source License

protected void putObject(SyncObject obj, String targetKey) {
    ObjectMetadata om = AwsS3Util.s3MetaFromSyncMeta(obj.getMetadata());
    if (obj.isDirectory())
        om.setContentType(AwsS3Util.TYPE_DIRECTORY);

    PutObjectRequest req;//from w  w w .  j  a v a  2s. co  m
    if (obj.isDirectory()) {
        req = new PutObjectRequest(bucketName, targetKey, new ByteArrayInputStream(new byte[0]), om);
    } else if (obj instanceof FileSyncObject) {
        req = new PutObjectRequest(bucketName, targetKey, ((FileSyncObject) obj).getRawSourceIdentifier());
    } else {
        req = new PutObjectRequest(bucketName, targetKey, obj.getInputStream(), om);
    }

    if (includeAcl)
        req.setAccessControlList(AwsS3Util.s3AclFromSyncAcl(obj.getMetadata().getAcl(), ignoreInvalidAcls));

    // xfer manager will figure out if MPU is needed (based on threshold), do the MPU if necessary,
    // and abort if it fails
    TransferManagerConfiguration xferConfig = new TransferManagerConfiguration();
    xferConfig.setMultipartUploadThreshold((long) mpuThresholdMB * 1024 * 1024);
    xferConfig.setMinimumUploadPartSize((long) mpuPartSizeMB * 1024 * 1024);
    TransferManager xferManager = new TransferManager(s3, Executors.newFixedThreadPool(mpuThreadCount));
    xferManager.setConfiguration(xferConfig);

    Upload upload = xferManager.upload(req);
    try {
        log.debug("Wrote {}, etag: {}", targetKey, upload.waitForUploadResult().getETag());
    } catch (InterruptedException e) {
        throw new RuntimeException("upload thread was interrupted", e);
    } finally {
        // make sure bytes read is accurate if we bypassed the counting stream
        if (obj instanceof FileSyncObject) {
            try {
                ((FileSyncObject) obj).setOverrideBytesRead(upload.getProgress().getBytesTransferred());
            } catch (Throwable t) {
                log.warn("could not get bytes transferred from upload", t);
            }
        }
    }
}

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

private void initTransferManager() {
    TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
    transferConfiguration.setMinimumUploadPartSize(partSize);
    transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
    transferConfiguration.setMultipartCopyPartSize(partSize);
    transferConfiguration.setMultipartCopyThreshold(multiPartThreshold);

    transfers = new TransferManager(mClient, unboundedThreadPool);
    transfers.setConfiguration(transferConfiguration);
}

From source file:org.anhonesteffort.p25.wav.TransferManagerFactory.java

License:Open Source License

public TransferManager create() {
    return new TransferManager(amazonS3(credentials()), executor);
}

From source file:org.apache.hadoop.fs.s3r.S3RFileSystem.java

License:Apache License

/** Called after a new FileSystem instance is constructed.
 * @param name a uri whose authority section names the host, port, etc.
 *   for this FileSystem//from  w w  w  .  ja  va  2  s . c  o  m
 * @param conf the configuration
 */
public void initialize(URI name, Configuration conf) throws IOException {
    super.initialize(name, conf);

    uri = URI.create(name.getScheme() + "://" + name.getAuthority());
    workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this.uri,
            this.getWorkingDirectory());

    // Try to get our credentials or just connect anonymously
    String accessKey = conf.get(ACCESS_KEY, null);
    String secretKey = conf.get(SECRET_KEY, null);

    String userInfo = name.getUserInfo();
    if (userInfo != null) {
        int index = userInfo.indexOf(':');
        if (index != -1) {
            accessKey = userInfo.substring(0, index);
            secretKey = userInfo.substring(index + 1);
        } else {
            accessKey = userInfo;
        }
    }

    AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain(
            new BasicAWSCredentialsProvider(accessKey, secretKey), new InstanceProfileCredentialsProvider(),
            new AnonymousAWSCredentialsProvider());

    bucket = name.getHost();

    ClientConfiguration awsConf = new ClientConfiguration();
    awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS));
    boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS);
    awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
    awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES));
    awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT));
    awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT));

    String proxyHost = conf.getTrimmed(PROXY_HOST, "");
    int proxyPort = conf.getInt(PROXY_PORT, -1);
    if (!proxyHost.isEmpty()) {
        awsConf.setProxyHost(proxyHost);
        if (proxyPort >= 0) {
            awsConf.setProxyPort(proxyPort);
        } else {
            if (secureConnections) {
                LOG.warn("Proxy host set without port. Using HTTPS default 443");
                awsConf.setProxyPort(443);
            } else {
                LOG.warn("Proxy host set without port. Using HTTP default 80");
                awsConf.setProxyPort(80);
            }
        }
        String proxyUsername = conf.getTrimmed(PROXY_USERNAME);
        String proxyPassword = conf.getTrimmed(PROXY_PASSWORD);
        if ((proxyUsername == null) != (proxyPassword == null)) {
            String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD + " set without the other.";
            LOG.error(msg);
            throw new IllegalArgumentException(msg);
        }
        awsConf.setProxyUsername(proxyUsername);
        awsConf.setProxyPassword(proxyPassword);
        awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
        awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
        if (LOG.isDebugEnabled()) {
            LOG.debug(
                    "Using proxy server {}:{} as user {} with password {} on " + "domain {} as workstation {}",
                    awsConf.getProxyHost(), awsConf.getProxyPort(), String.valueOf(awsConf.getProxyUsername()),
                    awsConf.getProxyPassword(), awsConf.getProxyDomain(), awsConf.getProxyWorkstation());
        }
    } else if (proxyPort >= 0) {
        String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
        LOG.error(msg);
        throw new IllegalArgumentException(msg);
    }

    s3 = new AmazonS3Client(credentials, awsConf);
    String endPoint = conf.getTrimmed(ENDPOINT, "");
    if (!endPoint.isEmpty()) {
        try {
            s3.setEndpoint(endPoint);
        } catch (IllegalArgumentException e) {
            String msg = "Incorrect endpoint: " + e.getMessage();
            LOG.error(msg);
            throw new IllegalArgumentException(msg, e);
        }
    }

    maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
    partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
    multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD, DEFAULT_MIN_MULTIPART_THRESHOLD);

    if (partSize < 5 * 1024 * 1024) {
        LOG.error(MULTIPART_SIZE + " must be at least 5 MB");
        partSize = 5 * 1024 * 1024;
    }

    if (multiPartThreshold < 5 * 1024 * 1024) {
        LOG.error(MIN_MULTIPART_THRESHOLD + " must be at least 5 MB");
        multiPartThreshold = 5 * 1024 * 1024;
    }

    int maxThreads = conf.getInt(MAX_THREADS, DEFAULT_MAX_THREADS);
    int coreThreads = conf.getInt(CORE_THREADS, DEFAULT_CORE_THREADS);
    if (maxThreads == 0) {
        maxThreads = Runtime.getRuntime().availableProcessors() * 8;
    }
    if (coreThreads == 0) {
        coreThreads = Runtime.getRuntime().availableProcessors() * 8;
    }
    long keepAliveTime = conf.getLong(KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME);
    LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(
            maxThreads * conf.getInt(MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS));
    threadPoolExecutor = new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS,
            workQueue, newDaemonThreadFactory("s3a-transfer-shared-"));
    threadPoolExecutor.allowCoreThreadTimeOut(true);

    TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
    transferConfiguration.setMinimumUploadPartSize(partSize);
    transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);

    transfers = new TransferManager(s3, threadPoolExecutor);
    transfers.setConfiguration(transferConfiguration);

    String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL);
    if (!cannedACLName.isEmpty()) {
        cannedACL = CannedAccessControlList.valueOf(cannedACLName);
    } else {
        cannedACL = null;
    }

    if (!s3.doesBucketExist(bucket)) {
        throw new IOException("Bucket " + bucket + " does not exist");
    }

    boolean purgeExistingMultipart = conf.getBoolean(PURGE_EXISTING_MULTIPART,
            DEFAULT_PURGE_EXISTING_MULTIPART);
    long purgeExistingMultipartAge = conf.getLong(PURGE_EXISTING_MULTIPART_AGE,
            DEFAULT_PURGE_EXISTING_MULTIPART_AGE);

    if (purgeExistingMultipart) {
        Date purgeBefore = new Date(new Date().getTime() - purgeExistingMultipartAge * 1000);

        transfers.abortMultipartUploads(bucket, purgeBefore);
    }

    serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM);

    setConf(conf);
}

From source file:org.apache.jackrabbit.aws.ext.ds.S3Backend.java

License:Apache License

public void init(CachingDataStore store, String homeDir, Properties prop) throws DataStoreException {

    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {/*  w  w  w. j a v  a2  s  .  com*/
        startTime = new Date();
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        LOG.debug("init");
        this.store = store;
        s3service = Utils.openService(prop);
        if (bucket == null || "".equals(bucket.trim())) {
            bucket = prop.getProperty(S3Constants.S3_BUCKET);
        }
        String region = prop.getProperty(S3Constants.S3_REGION);
        String endpoint = null;
        if (!s3service.doesBucketExist(bucket)) {

            if (DEFAULT_AWS_BUCKET_REGION.equals(region)) {
                s3service.createBucket(bucket, Region.US_Standard);
                endpoint = S3 + DOT + AWSDOTCOM;
            } else if (Region.EU_Ireland.toString().equals(region)) {
                s3service.createBucket(bucket, Region.EU_Ireland);
                endpoint = "s3-eu-west-1" + DOT + AWSDOTCOM;
            } else {
                s3service.createBucket(bucket, region);
                endpoint = S3 + DASH + region + DOT + AWSDOTCOM;
            }
            LOG.info("Created bucket [{}] in [{}] ", bucket, region);
        } else {
            LOG.info("Using bucket [{}]", bucket);
            if (DEFAULT_AWS_BUCKET_REGION.equals(region)) {
                endpoint = S3 + DOT + AWSDOTCOM;
            } else if (Region.EU_Ireland.toString().equals(region)) {
                endpoint = "s3-eu-west-1" + DOT + AWSDOTCOM;
            } else {
                endpoint = S3 + DASH + region + DOT + AWSDOTCOM;
            }
        }
        String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT);
        if (propEndPoint != null & !"".equals(propEndPoint)) {
            endpoint = propEndPoint;
        }
        /*
         * setting endpoint to remove latency of redirection. If endpoint is
         * not set, invocation first goes us standard region, which
         * redirects it to correct location.
         */
        s3service.setEndpoint(endpoint);
        LOG.info("S3 service endpoint [{}] ", endpoint);

        int writeThreads = 10;
        String writeThreadsStr = prop.getProperty(S3Constants.S3_WRITE_THREADS);
        if (writeThreadsStr != null) {
            writeThreads = Integer.parseInt(writeThreadsStr);
        }
        LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
        tmx = new TransferManager(s3service, (ThreadPoolExecutor) Executors.newFixedThreadPool(writeThreads,
                new NamedThreadFactory("s3-transfer-manager-worker")));

        int asyncWritePoolSize = 10;
        String maxConnsStr = prop.getProperty(S3Constants.S3_MAX_CONNS);
        if (maxConnsStr != null) {
            asyncWritePoolSize = Integer.parseInt(maxConnsStr) - writeThreads;
        }

        asyncWriteExecuter = (ThreadPoolExecutor) Executors.newFixedThreadPool(asyncWritePoolSize,
                new NamedThreadFactory("s3-write-worker"));
        String renameKeyProp = prop.getProperty(S3Constants.S3_RENAME_KEYS);
        boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) ? true
                : Boolean.parseBoolean(renameKeyProp);
        if (renameKeyBool) {
            renameKeys();
        }
        LOG.debug("S3 Backend initialized in [{}] ms", +(System.currentTimeMillis() - startTime.getTime()));
    } catch (Exception e) {
        LOG.debug("  error ", e);
        throw new DataStoreException("Could not initialize S3 from " + prop, e);
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}

From source file:org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend.java

License:Apache License

public void init(CachingDataStore store, String homeDir, Properties prop) throws DataStoreException {

    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {/* w  ww.  j  a v a2s . c  o m*/
        startTime = new Date();
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        LOG.debug("init");
        this.store = store;
        s3ReqDecorator = new S3RequestDecorator(prop);
        s3service = Utils.openService(prop);
        if (bucket == null || "".equals(bucket.trim())) {
            bucket = prop.getProperty(S3Constants.S3_BUCKET);
        }
        String region = prop.getProperty(S3Constants.S3_REGION);
        Region s3Region = null;
        if (StringUtils.isNullOrEmpty(region)) {
            com.amazonaws.regions.Region ec2Region = Regions.getCurrentRegion();
            if (ec2Region != null) {
                s3Region = Region.fromValue(ec2Region.getName());
            } else {
                throw new AmazonClientException("parameter [" + S3Constants.S3_REGION
                        + "] not configured and cannot be derived from environment");
            }
        } else {
            if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
                s3Region = Region.US_Standard;
            } else if (Region.EU_Ireland.toString().equals(region)) {
                s3Region = Region.EU_Ireland;
            } else {
                s3Region = Region.fromValue(region);
            }
        }

        if (!s3service.doesBucketExist(bucket)) {
            s3service.createBucket(bucket, s3Region);
            LOG.info("Created bucket [{}] in [{}] ", bucket, region);
        } else {
            LOG.info("Using bucket [{}] in [{}] ", bucket, region);
        }

        int writeThreads = 10;
        String writeThreadsStr = prop.getProperty(S3Constants.S3_WRITE_THREADS);
        if (writeThreadsStr != null) {
            writeThreads = Integer.parseInt(writeThreadsStr);
        }
        LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
        tmx = new TransferManager(s3service, (ThreadPoolExecutor) Executors.newFixedThreadPool(writeThreads,
                new NamedThreadFactory("s3-transfer-manager-worker")));

        int asyncWritePoolSize = 10;
        String maxConnsStr = prop.getProperty(S3Constants.S3_MAX_CONNS);
        if (maxConnsStr != null) {
            asyncWritePoolSize = Integer.parseInt(maxConnsStr) - writeThreads;
        }

        asyncWriteExecuter = (ThreadPoolExecutor) Executors.newFixedThreadPool(asyncWritePoolSize,
                new NamedThreadFactory("s3-write-worker"));
        String renameKeyProp = prop.getProperty(S3Constants.S3_RENAME_KEYS);
        boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) ? false
                : Boolean.parseBoolean(renameKeyProp);
        LOG.info("Rename keys [{}]", renameKeyBool);
        if (renameKeyBool) {
            renameKeys();
        }
        LOG.debug("S3 Backend initialized in [{}] ms", +(System.currentTimeMillis() - startTime.getTime()));
    } catch (Exception e) {
        LOG.debug("  error ", e);
        Map<String, String> filteredMap = Maps.newHashMap();
        if (prop != null) {
            filteredMap = Maps.filterKeys(Maps.fromProperties(prop), new Predicate<String>() {
                @Override
                public boolean apply(String input) {
                    return !input.equals(S3Constants.ACCESS_KEY) && !input.equals(S3Constants.SECRET_KEY);
                }
            });
        }
        throw new DataStoreException("Could not initialize S3 from " + filteredMap, e);
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}