Example usage for com.amazonaws.services.s3.transfer TransferManagerConfiguration setMultipartCopyThreshold

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.transfer TransferManagerConfiguration setMultipartCopyThreshold.

Prototype

public void setMultipartCopyThreshold(long multipartCopyThreshold) 

Source Link

Document

Sets the size threshold in bytes for when to use multi-part copy requests.

Usage

From source file:alluxio.underfs.s3a.S3AUnderFileSystem.java

License:Apache License

/**
 * Constructs a new instance of {@link S3AUnderFileSystem}.
 *
 * @param uri the {@link AlluxioURI} for this UFS
 *///from  w  w w.  ja v  a  2  s  .  c o  m
public S3AUnderFileSystem(AlluxioURI uri) {
    super(uri);
    mBucketName = uri.getHost();
    mBucketPrefix = PathUtils.normalizePath(Constants.HEADER_S3A + mBucketName, PATH_SEPARATOR);

    // Set the aws credential system properties based on Alluxio properties, if they are set
    if (Configuration.containsKey(PropertyKey.S3A_ACCESS_KEY)) {
        System.setProperty(SDKGlobalConfiguration.ACCESS_KEY_SYSTEM_PROPERTY,
                Configuration.get(PropertyKey.S3A_ACCESS_KEY));
    }
    if (Configuration.containsKey(PropertyKey.S3A_SECRET_KEY)) {
        System.setProperty(SDKGlobalConfiguration.SECRET_KEY_SYSTEM_PROPERTY,
                Configuration.get(PropertyKey.S3A_SECRET_KEY));
    }

    // Checks, in order, env variables, system properties, profile file, and instance profile
    AWSCredentialsProvider credentials = new AWSCredentialsProviderChain(
            new DefaultAWSCredentialsProviderChain());

    // Set the client configuration based on Alluxio configuration values
    ClientConfiguration clientConf = new ClientConfiguration();

    // Socket timeout
    clientConf.setSocketTimeout(Configuration.getInt(PropertyKey.UNDERFS_S3A_SOCKET_TIMEOUT_MS));

    // HTTP protocol
    if (Configuration.getBoolean(PropertyKey.UNDERFS_S3A_SECURE_HTTP_ENABLED)) {
        clientConf.setProtocol(Protocol.HTTPS);
    } else {
        clientConf.setProtocol(Protocol.HTTP);
    }

    // Proxy host
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_PROXY_HOST)) {
        clientConf.setProxyHost(Configuration.get(PropertyKey.UNDERFS_S3_PROXY_HOST));
    }

    // Proxy port
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_PROXY_PORT)) {
        clientConf.setProxyPort(Configuration.getInt(PropertyKey.UNDERFS_S3_PROXY_PORT));
    }

    mClient = new AmazonS3Client(credentials, clientConf);
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_ENDPOINT)) {
        mClient.setEndpoint(Configuration.get(PropertyKey.UNDERFS_S3_ENDPOINT));
    }
    mManager = new TransferManager(mClient);

    TransferManagerConfiguration transferConf = new TransferManagerConfiguration();
    transferConf.setMultipartCopyThreshold(MULTIPART_COPY_THRESHOLD);
    mManager.setConfiguration(transferConf);

    mAccountOwnerId = mClient.getS3AccountOwner().getId();
    // Gets the owner from user-defined static mapping from S3 canonical user id  to Alluxio
    // user name.
    String owner = CommonUtils.getValueFromStaticMapping(
            Configuration.get(PropertyKey.UNDERFS_S3_OWNER_ID_TO_USERNAME_MAPPING), mAccountOwnerId);
    // If there is no user-defined mapping, use the display name.
    if (owner == null) {
        owner = mClient.getS3AccountOwner().getDisplayName();
    }
    mAccountOwner = owner == null ? mAccountOwnerId : owner;

    AccessControlList acl = mClient.getBucketAcl(mBucketName);
    mBucketMode = S3AUtils.translateBucketAcl(acl, mAccountOwnerId);
}

---

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

private void initTransferManager() {
    TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
    transferConfiguration.setMinimumUploadPartSize(partSize);
    transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
    transferConfiguration.setMultipartCopyPartSize(partSize);
    transferConfiguration.setMultipartCopyThreshold(multiPartThreshold);

    transfers = new TransferManager(mClient, unboundedThreadPool);
    transfers.setConfiguration(transferConfiguration);
}

---