List of usage examples for com.amazonaws.services.s3.transfer TransferManagerConfiguration TransferManagerConfiguration
TransferManagerConfiguration
From source file:alluxio.underfs.s3a.S3AUnderFileSystem.java
License:Apache License
/** * Constructs a new instance of {@link S3AUnderFileSystem}. * * @param uri the {@link AlluxioURI} for this UFS *//*from w w w .j av a 2 s . c o m*/ public S3AUnderFileSystem(AlluxioURI uri) { super(uri); mBucketName = uri.getHost(); mBucketPrefix = PathUtils.normalizePath(Constants.HEADER_S3A + mBucketName, PATH_SEPARATOR); // Set the aws credential system properties based on Alluxio properties, if they are set if (Configuration.containsKey(PropertyKey.S3A_ACCESS_KEY)) { System.setProperty(SDKGlobalConfiguration.ACCESS_KEY_SYSTEM_PROPERTY, Configuration.get(PropertyKey.S3A_ACCESS_KEY)); } if (Configuration.containsKey(PropertyKey.S3A_SECRET_KEY)) { System.setProperty(SDKGlobalConfiguration.SECRET_KEY_SYSTEM_PROPERTY, Configuration.get(PropertyKey.S3A_SECRET_KEY)); } // Checks, in order, env variables, system properties, profile file, and instance profile AWSCredentialsProvider credentials = new AWSCredentialsProviderChain( new DefaultAWSCredentialsProviderChain()); // Set the client configuration based on Alluxio configuration values ClientConfiguration clientConf = new ClientConfiguration(); // Socket timeout clientConf.setSocketTimeout(Configuration.getInt(PropertyKey.UNDERFS_S3A_SOCKET_TIMEOUT_MS)); // HTTP protocol if (Configuration.getBoolean(PropertyKey.UNDERFS_S3A_SECURE_HTTP_ENABLED)) { clientConf.setProtocol(Protocol.HTTPS); } else { clientConf.setProtocol(Protocol.HTTP); } // Proxy host if (Configuration.containsKey(PropertyKey.UNDERFS_S3_PROXY_HOST)) { clientConf.setProxyHost(Configuration.get(PropertyKey.UNDERFS_S3_PROXY_HOST)); } // Proxy port if (Configuration.containsKey(PropertyKey.UNDERFS_S3_PROXY_PORT)) { clientConf.setProxyPort(Configuration.getInt(PropertyKey.UNDERFS_S3_PROXY_PORT)); } mClient = new AmazonS3Client(credentials, clientConf); if (Configuration.containsKey(PropertyKey.UNDERFS_S3_ENDPOINT)) { mClient.setEndpoint(Configuration.get(PropertyKey.UNDERFS_S3_ENDPOINT)); } mManager = new TransferManager(mClient); TransferManagerConfiguration transferConf = new TransferManagerConfiguration(); transferConf.setMultipartCopyThreshold(MULTIPART_COPY_THRESHOLD); mManager.setConfiguration(transferConf); mAccountOwnerId = mClient.getS3AccountOwner().getId(); // Gets the owner from user-defined static mapping from S3 canonical user id to Alluxio // user name. String owner = CommonUtils.getValueFromStaticMapping( Configuration.get(PropertyKey.UNDERFS_S3_OWNER_ID_TO_USERNAME_MAPPING), mAccountOwnerId); // If there is no user-defined mapping, use the display name. if (owner == null) { owner = mClient.getS3AccountOwner().getDisplayName(); } mAccountOwner = owner == null ? mAccountOwnerId : owner; AccessControlList acl = mClient.getBucketAcl(mBucketName); mBucketMode = S3AUtils.translateBucketAcl(acl, mAccountOwnerId); }
From source file:br.com.ingenieux.mojo.aws.util.BeanstalkerS3Client.java
License:Apache License
protected void init(Region region) { transferManager = new TransferManager(this); TransferManagerConfiguration configuration = new TransferManagerConfiguration(); configuration.setMultipartUploadThreshold(100 * Constants.KB); transferManager.setConfiguration(configuration); this.setRegion(region); }
From source file:com.emc.ecs.sync.target.S3Target.java
License:Open Source License
protected void putObject(SyncObject obj, String targetKey) { ObjectMetadata om = AwsS3Util.s3MetaFromSyncMeta(obj.getMetadata()); if (obj.isDirectory()) om.setContentType(AwsS3Util.TYPE_DIRECTORY); PutObjectRequest req;//from ww w . j a v a 2 s. c o m if (obj.isDirectory()) { req = new PutObjectRequest(bucketName, targetKey, new ByteArrayInputStream(new byte[0]), om); } else if (obj instanceof FileSyncObject) { req = new PutObjectRequest(bucketName, targetKey, ((FileSyncObject) obj).getRawSourceIdentifier()); } else { req = new PutObjectRequest(bucketName, targetKey, obj.getInputStream(), om); } if (includeAcl) req.setAccessControlList(AwsS3Util.s3AclFromSyncAcl(obj.getMetadata().getAcl(), ignoreInvalidAcls)); // xfer manager will figure out if MPU is needed (based on threshold), do the MPU if necessary, // and abort if it fails TransferManagerConfiguration xferConfig = new TransferManagerConfiguration(); xferConfig.setMultipartUploadThreshold((long) mpuThresholdMB * 1024 * 1024); xferConfig.setMinimumUploadPartSize((long) mpuPartSizeMB * 1024 * 1024); TransferManager xferManager = new TransferManager(s3, Executors.newFixedThreadPool(mpuThreadCount)); xferManager.setConfiguration(xferConfig); Upload upload = xferManager.upload(req); try { log.debug("Wrote {}, etag: {}", targetKey, upload.waitForUploadResult().getETag()); } catch (InterruptedException e) { throw new RuntimeException("upload thread was interrupted", e); } finally { // make sure bytes read is accurate if we bypassed the counting stream if (obj instanceof FileSyncObject) { try { ((FileSyncObject) obj).setOverrideBytesRead(upload.getProgress().getBytesTransferred()); } catch (Throwable t) { log.warn("could not get bytes transferred from upload", t); } } } }
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
private void initTransferManager() { TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); transferConfiguration.setMinimumUploadPartSize(partSize); transferConfiguration.setMultipartUploadThreshold(multiPartThreshold); transferConfiguration.setMultipartCopyPartSize(partSize); transferConfiguration.setMultipartCopyThreshold(multiPartThreshold); transfers = new TransferManager(mClient, unboundedThreadPool); transfers.setConfiguration(transferConfiguration); }
From source file:com.liferay.portal.store.s3.S3Store.java
License:Open Source License
protected TransferManager getTransferManager(AmazonS3 amazonS3) { ExecutorService executorService = new ThreadPoolExecutor(_s3StoreConfiguration.corePoolSize(), _s3StoreConfiguration.maxPoolSize()); TransferManager transferManager = new TransferManager(amazonS3, executorService, false); TransferManagerConfiguration transferManagerConfiguration = new TransferManagerConfiguration(); transferManagerConfiguration.setMinimumUploadPartSize(_s3StoreConfiguration.minimumUploadPartSize()); transferManagerConfiguration.setMultipartUploadThreshold(_s3StoreConfiguration.multipartUploadThreshold()); transferManager.setConfiguration(transferManagerConfiguration); return transferManager; }
From source file:fr.eurecom.hybris.kvs.drivers.AmazonKvs.java
License:Apache License
public AmazonKvs(String id, final String accessKey, final String secretKey, String container, boolean enabled, int cost) throws IOException { super(id, container, enabled, cost); BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); this.s3 = new AmazonS3Client(credentials); this.tm = new TransferManager(credentials); TransferManagerConfiguration tmc = new TransferManagerConfiguration(); tmc.setMultipartUploadThreshold(30000000L); // 30 MB tmc.setMinimumUploadPartSize(10000000); // 10 MB this.tm.setConfiguration(tmc); this.createContainer(); }
From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java
License:Apache License
/** * The src file is on the local disk. Add it to FS at * the given dst name.//from w w w . j a v a2 s .co m * * This version doesn't need to create a temporary file to calculate the md5. Sadly this doesn't seem to be * used by the shell cp :( * * delSrc indicates if the source should be removed * @param delSrc whether to delete the src * @param overwrite whether to overwrite an existing file * @param src path * @param dst path */ @Override public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException { String key = pathToKey(dst); if (!overwrite && exists(dst)) { throw new IOException(dst + " already exists"); } LOG.info("Copying local file from " + src + " to " + dst); // Since we have a local file, we don't need to stream into a temporary file LocalFileSystem local = getLocal(getConf()); File srcfile = local.pathToFile(src); TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); transferConfiguration.setMinimumUploadPartSize(partSize); transferConfiguration.setMultipartUploadThreshold(partSizeThreshold); TransferManager transfers = new TransferManager(s3); transfers.setConfiguration(transferConfiguration); final ObjectMetadata om = new ObjectMetadata(); if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { om.setServerSideEncryption(serverSideEncryptionAlgorithm); } PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile); putObjectRequest.setCannedAcl(cannedACL); putObjectRequest.setMetadata(om); ProgressListener progressListener = new ProgressListener() { public void progressChanged(ProgressEvent progressEvent) { switch (progressEvent.getEventCode()) { case ProgressEvent.PART_COMPLETED_EVENT_CODE: statistics.incrementWriteOps(1); break; } } }; Upload up = transfers.upload(putObjectRequest); up.addProgressListener(progressListener); try { up.waitForUploadResult(); statistics.incrementWriteOps(1); } catch (InterruptedException e) { throw new IOException("Got interrupted, cancelling"); } finally { transfers.shutdownNow(false); } // This will delete unnecessary fake parent directories finishedWrite(key); if (delSrc) { local.delete(src, false); } }
From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java
License:Apache License
private void copyFile(String srcKey, String dstKey) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("copyFile " + srcKey + " -> " + dstKey); }//from www.j a v a 2s . com TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); transferConfiguration.setMultipartCopyPartSize(partSize); TransferManager transfers = new TransferManager(s3); transfers.setConfiguration(transferConfiguration); ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey); final ObjectMetadata dstom = srcom.clone(); if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { dstom.setServerSideEncryption(serverSideEncryptionAlgorithm); } CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey); copyObjectRequest.setCannedAccessControlList(cannedACL); copyObjectRequest.setNewObjectMetadata(dstom); ProgressListener progressListener = new ProgressListener() { public void progressChanged(ProgressEvent progressEvent) { switch (progressEvent.getEventCode()) { case ProgressEvent.PART_COMPLETED_EVENT_CODE: statistics.incrementWriteOps(1); break; } } }; Copy copy = transfers.copy(copyObjectRequest); copy.addProgressListener(progressListener); try { copy.waitForCopyResult(); statistics.incrementWriteOps(1); } catch (InterruptedException e) { throw new IOException("Got interrupted, cancelling"); } finally { transfers.shutdownNow(false); } }
From source file:org.apache.hadoop.fs.s3a.S3AOutputStream.java
License:Apache License
@Override public synchronized void close() throws IOException { if (closed) { return;/*from w ww . java 2s .c om*/ } backupStream.close(); LOG.info("OutputStream for key '" + key + "' closed. Now beginning upload"); LOG.info("Minimum upload part size: " + partSize + " threshold " + partSizeThreshold); try { TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); transferConfiguration.setMinimumUploadPartSize(partSize); transferConfiguration.setMultipartUploadThreshold(partSizeThreshold); TransferManager transfers = new TransferManager(client); transfers.setConfiguration(transferConfiguration); final ObjectMetadata om = new ObjectMetadata(); if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { om.setServerSideEncryption(serverSideEncryptionAlgorithm); } PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile); putObjectRequest.setCannedAcl(cannedACL); putObjectRequest.setMetadata(om); Upload upload = transfers.upload(putObjectRequest); ProgressableProgressListener listener = new ProgressableProgressListener(upload, progress, statistics); upload.addProgressListener(listener); upload.waitForUploadResult(); long delta = upload.getProgress().getBytesTransferred() - listener.getLastBytesTransferred(); if (statistics != null && delta != 0) { if (LOG.isDebugEnabled()) { LOG.debug("S3A write delta changed after finished: " + delta + " bytes"); } statistics.incrementBytesWritten(delta); } // This will delete unnecessary fake parent directories fs.finishedWrite(key); } catch (InterruptedException e) { throw new IOException(e); } finally { if (!backupFile.delete()) { LOG.warn("Could not delete temporary s3a file: " + backupFile); } super.close(); closed = true; } LOG.info("OutputStream for key '" + key + "' upload complete"); }
From source file:org.apache.hadoop.fs.s3r.S3RFileSystem.java
License:Apache License
/** Called after a new FileSystem instance is constructed. * @param name a uri whose authority section names the host, port, etc. * for this FileSystem//from w w w. j a va 2s.c o m * @param conf the configuration */ public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); uri = URI.create(name.getScheme() + "://" + name.getAuthority()); workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this.uri, this.getWorkingDirectory()); // Try to get our credentials or just connect anonymously String accessKey = conf.get(ACCESS_KEY, null); String secretKey = conf.get(SECRET_KEY, null); String userInfo = name.getUserInfo(); if (userInfo != null) { int index = userInfo.indexOf(':'); if (index != -1) { accessKey = userInfo.substring(0, index); secretKey = userInfo.substring(index + 1); } else { accessKey = userInfo; } } AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain( new BasicAWSCredentialsProvider(accessKey, secretKey), new InstanceProfileCredentialsProvider(), new AnonymousAWSCredentialsProvider()); bucket = name.getHost(); ClientConfiguration awsConf = new ClientConfiguration(); awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS)); boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS); awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES)); awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT)); awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT)); String proxyHost = conf.getTrimmed(PROXY_HOST, ""); int proxyPort = conf.getInt(PROXY_PORT, -1); if (!proxyHost.isEmpty()) { awsConf.setProxyHost(proxyHost); if (proxyPort >= 0) { awsConf.setProxyPort(proxyPort); } else { if (secureConnections) { LOG.warn("Proxy host set without port. Using HTTPS default 443"); awsConf.setProxyPort(443); } else { LOG.warn("Proxy host set without port. Using HTTP default 80"); awsConf.setProxyPort(80); } } String proxyUsername = conf.getTrimmed(PROXY_USERNAME); String proxyPassword = conf.getTrimmed(PROXY_PASSWORD); if ((proxyUsername == null) != (proxyPassword == null)) { String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD + " set without the other."; LOG.error(msg); throw new IllegalArgumentException(msg); } awsConf.setProxyUsername(proxyUsername); awsConf.setProxyPassword(proxyPassword); awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN)); awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); if (LOG.isDebugEnabled()) { LOG.debug( "Using proxy server {}:{} as user {} with password {} on " + "domain {} as workstation {}", awsConf.getProxyHost(), awsConf.getProxyPort(), String.valueOf(awsConf.getProxyUsername()), awsConf.getProxyPassword(), awsConf.getProxyDomain(), awsConf.getProxyWorkstation()); } } else if (proxyPort >= 0) { String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST; LOG.error(msg); throw new IllegalArgumentException(msg); } s3 = new AmazonS3Client(credentials, awsConf); String endPoint = conf.getTrimmed(ENDPOINT, ""); if (!endPoint.isEmpty()) { try { s3.setEndpoint(endPoint); } catch (IllegalArgumentException e) { String msg = "Incorrect endpoint: " + e.getMessage(); LOG.error(msg); throw new IllegalArgumentException(msg, e); } } maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS); partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE); multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD, DEFAULT_MIN_MULTIPART_THRESHOLD); if (partSize < 5 * 1024 * 1024) { LOG.error(MULTIPART_SIZE + " must be at least 5 MB"); partSize = 5 * 1024 * 1024; } if (multiPartThreshold < 5 * 1024 * 1024) { LOG.error(MIN_MULTIPART_THRESHOLD + " must be at least 5 MB"); multiPartThreshold = 5 * 1024 * 1024; } int maxThreads = conf.getInt(MAX_THREADS, DEFAULT_MAX_THREADS); int coreThreads = conf.getInt(CORE_THREADS, DEFAULT_CORE_THREADS); if (maxThreads == 0) { maxThreads = Runtime.getRuntime().availableProcessors() * 8; } if (coreThreads == 0) { coreThreads = Runtime.getRuntime().availableProcessors() * 8; } long keepAliveTime = conf.getLong(KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME); LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>( maxThreads * conf.getInt(MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS)); threadPoolExecutor = new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, newDaemonThreadFactory("s3a-transfer-shared-")); threadPoolExecutor.allowCoreThreadTimeOut(true); TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); transferConfiguration.setMinimumUploadPartSize(partSize); transferConfiguration.setMultipartUploadThreshold(multiPartThreshold); transfers = new TransferManager(s3, threadPoolExecutor); transfers.setConfiguration(transferConfiguration); String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL); if (!cannedACLName.isEmpty()) { cannedACL = CannedAccessControlList.valueOf(cannedACLName); } else { cannedACL = null; } if (!s3.doesBucketExist(bucket)) { throw new IOException("Bucket " + bucket + " does not exist"); } boolean purgeExistingMultipart = conf.getBoolean(PURGE_EXISTING_MULTIPART, DEFAULT_PURGE_EXISTING_MULTIPART); long purgeExistingMultipartAge = conf.getLong(PURGE_EXISTING_MULTIPART_AGE, DEFAULT_PURGE_EXISTING_MULTIPART_AGE); if (purgeExistingMultipart) { Date purgeBefore = new Date(new Date().getTime() - purgeExistingMultipartAge * 1000); transfers.abortMultipartUploads(bucket, purgeBefore); } serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM); setConf(conf); }