Example usage for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client.

Prototype

@Deprecated
public AmazonS3Client(AWSCredentialsProvider credentialsProvider, ClientConfiguration clientConfiguration) 

Source Link

Document

Constructs a new Amazon S3 client using the specified AWS credentials and client configuration to access Amazon S3.

Usage

From source file:com.netflix.ice.common.AwsUtils.java

License:Apache License

/**
 * List all object summary with given prefix in the s3 bucket.
 * @param bucket//from  w w  w . jav a2 s  . c  o m
 * @param prefix
 * @return
 */
public static List<S3ObjectSummary> listAllObjects(String bucket, String prefix, String accountId,
        String assumeRole, String externalId) {
    AmazonS3Client s3Client = AwsUtils.s3Client;

    try {
        ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix);
        List<S3ObjectSummary> result = Lists.newLinkedList();

        if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) {
            Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId);
            s3Client = new AmazonS3Client(
                    new BasicSessionCredentials(assumedCredentials.getAccessKeyId(),
                            assumedCredentials.getSecretAccessKey(), assumedCredentials.getSessionToken()),
                    clientConfig);
        }

        ObjectListing page = null;
        do {
            if (page != null)
                request.setMarker(page.getNextMarker());
            page = s3Client.listObjects(request);
            result.addAll(page.getObjectSummaries());

        } while (page.isTruncated());

        return result;
    } finally {
        if (s3Client != AwsUtils.s3Client)
            s3Client.shutdown();
    }
}

---

From source file:com.netflix.ice.common.AwsUtils.java

License:Apache License

public static boolean downloadFileIfChangedSince(String bucketName, String bucketFilePrefix, File file,
        long milles, String accountId, String assumeRole, String externalId) {
    AmazonS3Client s3Client = AwsUtils.s3Client;

    try {//from w w  w  .j av  a 2  s . c  o  m
        if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) {
            Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId);
            s3Client = new AmazonS3Client(
                    new BasicSessionCredentials(assumedCredentials.getAccessKeyId(),
                            assumedCredentials.getSecretAccessKey(), assumedCredentials.getSessionToken()),
                    clientConfig);
        }

        ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName());
        boolean download = !file.exists() || metadata.getLastModified().getTime() > milles;

        if (download) {
            return download(s3Client, bucketName, bucketFilePrefix + file.getName(), file);
        } else
            return download;
    } finally {
        if (s3Client != AwsUtils.s3Client)
            s3Client.shutdown();
    }
}

---

From source file:com.nextdoor.bender.aws.S3MockClientFactory.java

License:Apache License

public S3MockClientFactory(int port, String username, String pass) {
    try {//from  w  ww  .ja v  a2 s.c  o m
        this.s3.start(port, username, pass);
    } catch (IOException | InterruptedException | InvalidExitValueException | TimeoutException e) {
        throw new RuntimeException(e);
    }

    BasicAWSCredentials awsCredentials = new BasicAWSCredentials(username, pass);
    this.client = new AmazonS3Client(awsCredentials, new ClientConfiguration());
    this.client.setEndpoint("http://127.0.0.1:" + port);
    this.client.createBucket(S3_BUCKET);
}

---

From source file:com.pinterest.secor.uploader.S3UploadManager.java

License:Apache License

public S3UploadManager(SecorConfig config) {
    super(config);

    final String accessKey = mConfig.getAwsAccessKey();
    final String secretKey = mConfig.getAwsSecretKey();
    final String endpoint = mConfig.getAwsEndpoint();
    final String region = mConfig.getAwsRegion();
    final String awsRole = mConfig.getAwsRole();

    s3Path = mConfig.getS3Path();

    AmazonS3 client;//from  w  ww .j a  va2  s . c o m
    AWSCredentialsProvider provider;

    ClientConfiguration clientConfiguration = new ClientConfiguration();
    boolean isHttpProxyEnabled = mConfig.getAwsProxyEnabled();

    //proxy settings
    if (isHttpProxyEnabled) {
        LOG.info("Http Proxy Enabled for S3UploadManager");
        String httpProxyHost = mConfig.getAwsProxyHttpHost();
        int httpProxyPort = mConfig.getAwsProxyHttpPort();
        clientConfiguration.setProxyHost(httpProxyHost);
        clientConfiguration.setProxyPort(httpProxyPort);
    }

    if (accessKey.isEmpty() || secretKey.isEmpty()) {
        provider = new DefaultAWSCredentialsProviderChain();
    } else {
        provider = new AWSCredentialsProvider() {
            public AWSCredentials getCredentials() {
                return new BasicAWSCredentials(accessKey, secretKey);
            }

            public void refresh() {
            }
        };
    }

    if (!awsRole.isEmpty()) {
        provider = new STSAssumeRoleSessionCredentialsProvider(provider, awsRole, "secor");
    }

    client = new AmazonS3Client(provider, clientConfiguration);

    if (!endpoint.isEmpty()) {
        client.setEndpoint(endpoint);
    } else if (!region.isEmpty()) {
        client.setRegion(Region.getRegion(Regions.fromName(region)));
    }

    mManager = new TransferManager(client);
}

---

From source file:com.streamsets.datacollector.bundles.SupportBundleManager.java

License:Apache License

/**
 * Instead of providing support bundle directly to user, upload it to StreamSets backend services.
 *///from  w  w w  .j  a  v a  2  s.  c o m
public void uploadNewBundleFromInstances(List<BundleContentGenerator> generators, BundleType bundleType)
        throws IOException {
    // Generate bundle
    SupportBundle bundle = generateNewBundleFromInstances(generators, bundleType);

    boolean enabled = configuration.get(Constants.UPLOAD_ENABLED, Constants.DEFAULT_UPLOAD_ENABLED);
    String accessKey = configuration.get(Constants.UPLOAD_ACCESS, Constants.DEFAULT_UPLOAD_ACCESS);
    String secretKey = configuration.get(Constants.UPLOAD_SECRET, Constants.DEFAULT_UPLOAD_SECRET);
    String bucket = configuration.get(Constants.UPLOAD_BUCKET, Constants.DEFAULT_UPLOAD_BUCKET);
    int bufferSize = configuration.get(Constants.UPLOAD_BUFFER_SIZE, Constants.DEFAULT_UPLOAD_BUFFER_SIZE);

    if (!enabled) {
        throw new IOException("Uploading support bundles was disabled by administrator.");
    }

    AWSCredentialsProvider credentialsProvider = new StaticCredentialsProvider(
            new BasicAWSCredentials(accessKey, secretKey));
    AmazonS3Client s3Client = new AmazonS3Client(credentialsProvider, new ClientConfiguration());
    s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
    s3Client.setRegion(Region.getRegion(Regions.US_WEST_2));

    // Object Metadata
    ObjectMetadata s3Metadata = new ObjectMetadata();
    for (Map.Entry<Object, Object> entry : getMetadata(bundleType).entrySet()) {
        s3Metadata.addUserMetadata((String) entry.getKey(), (String) entry.getValue());
    }

    List<PartETag> partETags;
    InitiateMultipartUploadResult initResponse = null;
    try {
        // Uploading part by part
        LOG.info("Initiating multi-part support bundle upload");
        partETags = new ArrayList<>();
        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket,
                bundle.getBundleKey());
        initRequest.setObjectMetadata(s3Metadata);
        initResponse = s3Client.initiateMultipartUpload(initRequest);
    } catch (AmazonClientException e) {
        LOG.error("Support bundle upload failed: ", e);
        throw new IOException("Support bundle upload failed", e);
    }

    try {
        byte[] buffer = new byte[bufferSize];
        int partId = 1;
        int size = -1;
        while ((size = readFully(bundle.getInputStream(), buffer)) != -1) {
            LOG.debug("Uploading part {} of size {}", partId, size);
            UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(bucket)
                    .withKey(bundle.getBundleKey()).withUploadId(initResponse.getUploadId())
                    .withPartNumber(partId++).withInputStream(new ByteArrayInputStream(buffer))
                    .withPartSize(size);

            partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
        }

        CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket,
                bundle.getBundleKey(), initResponse.getUploadId(), partETags);

        s3Client.completeMultipartUpload(compRequest);
        LOG.info("Support bundle upload finished");
    } catch (Exception e) {
        LOG.error("Support bundle upload failed", e);
        s3Client.abortMultipartUpload(
                new AbortMultipartUploadRequest(bucket, bundle.getBundleKey(), initResponse.getUploadId()));

        throw new IOException("Can't upload support bundle", e);
    } finally {
        // Close the client
        s3Client.shutdown();
    }
}

---

From source file:com.streamsets.pipeline.stage.origin.s3.S3Config.java

License:Apache License

private void validateConnection(Stage.Context context, List<Stage.ConfigIssue> issues) {
    //Access Key ID - username [unique in aws]
    //secret access key - password
    AWSCredentials credentials = new BasicAWSCredentials(accessKeyId, secretAccessKey);
    s3Client = new AmazonS3Client(credentials, new ClientConfiguration());
    s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
    if (endPoint != null && !endPoint.isEmpty()) {
        s3Client.setEndpoint(endPoint);//from   w  w  w  .  ja  v  a 2  s.c o m
    } else {
        s3Client.setRegion(Region.getRegion(region));
    }
    try {
        //check if the credentials are right by trying to list buckets
        s3Client.listBuckets();
    } catch (AmazonS3Exception e) {
        issues.add(context.createConfigIssue(Groups.S3.name(), "accessKeyId", Errors.S3_SPOOLDIR_20,
                e.toString()));
    }
}

---

From source file:com.tango.BucketSyncer.StorageClients.S3Client.java

License:Apache License

public void createClient(MirrorOptions options) {
    ClientConfiguration clientConfiguration = new ClientConfiguration().withProtocol(Protocol.HTTP)
            .withMaxConnections(options.getMaxConnections());
    if (options.getHasProxy()) {
        clientConfiguration = clientConfiguration.withProxyHost(options.getProxyHost())
                .withProxyPort(options.getProxyPort());
    }//ww w  .  ja v  a2  s  .c o m
    this.s3Client = new AmazonS3Client(options, clientConfiguration);
    if (options.hasEndpoint()) {
        s3Client.setEndpoint(options.getEndpoint());
    }
}

---

From source file:com.treasure_data.td_import.source.S3Source.java

License:Apache License

static AmazonS3Client createAmazonS3Client(SourceDesc desc) {
    String accessKey = desc.getUser();
    if (accessKey == null || accessKey.isEmpty()) {
        throw new IllegalArgumentException("S3 AccessKey is null or empty.");
    }//w  ww.ja va  2 s.  c  om
    String secretAccessKey = desc.getPassword();
    if (secretAccessKey == null || secretAccessKey.isEmpty()) {
        throw new IllegalArgumentException("S3 SecretAccessKey is null or empty.");
    }
    AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey);

    ClientConfiguration conf = new ClientConfiguration();
    conf.setProtocol(Configuration.BI_PREPARE_S3_PROTOCOL);
    conf.setMaxConnections(Configuration.BI_PREPARE_S3_MAX_CONNECTIONS);
    conf.setMaxErrorRetry(Configuration.BI_PREPARE_S3_MAX_ERRORRETRY);
    conf.setSocketTimeout(Configuration.BI_PREPARE_S3_SOCKET_TIMEOUT);

    return new AmazonS3Client(credentials, conf);
}

---

From source file:com.uiintl.backup.agent.AwsBackupAgent.java

License:Open Source License

void initS3Client() {

    ClientConfiguration configuration = new ClientConfiguration();
    configuration.setSocketTimeout(EXTENDED_SO_TIMEOUT);

    s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider(), configuration);
    Region usWest2 = Region.getRegion(Regions.AP_SOUTHEAST_2);
    s3.setRegion(usWest2);//from ww  w. ja v  a  2 s  .  com
}

---

From source file:com.yahoo.ycsb.db.S3Client.java

License:Open Source License

/**
* Initialize any state for the storage./*www  .ja va2  s .co  m*/
* Called once per S3 instance; If the client is not null it is re-used.
*/
@Override
public void init() throws DBException {
    final int count = INIT_COUNT.incrementAndGet();
    synchronized (S3Client.class) {
        Properties propsCL = getProperties();
        int recordcount = Integer.parseInt(propsCL.getProperty("recordcount"));
        int operationcount = Integer.parseInt(propsCL.getProperty("operationcount"));
        int numberOfOperations = 0;
        if (recordcount > 0) {
            if (recordcount > operationcount) {
                numberOfOperations = recordcount;
            } else {
                numberOfOperations = operationcount;
            }
        } else {
            numberOfOperations = operationcount;
        }
        if (count <= numberOfOperations) {
            String accessKeyId = null;
            String secretKey = null;
            String endPoint = null;
            String region = null;
            String maxErrorRetry = null;
            String maxConnections = null;
            String protocol = null;
            BasicAWSCredentials s3Credentials;
            ClientConfiguration clientConfig;
            if (s3Client != null) {
                System.out.println("Reusing the same client");
                return;
            }
            try {
                InputStream propFile = S3Client.class.getClassLoader().getResourceAsStream("s3.properties");
                Properties props = new Properties(System.getProperties());
                props.load(propFile);
                accessKeyId = props.getProperty("s3.accessKeyId");
                if (accessKeyId == null) {
                    accessKeyId = propsCL.getProperty("s3.accessKeyId");
                }
                System.out.println(accessKeyId);
                secretKey = props.getProperty("s3.secretKey");
                if (secretKey == null) {
                    secretKey = propsCL.getProperty("s3.secretKey");
                }
                System.out.println(secretKey);
                endPoint = props.getProperty("s3.endPoint");
                if (endPoint == null) {
                    endPoint = propsCL.getProperty("s3.endPoint", "s3.amazonaws.com");
                }
                System.out.println(endPoint);
                region = props.getProperty("s3.region");
                if (region == null) {
                    region = propsCL.getProperty("s3.region", "us-east-1");
                }
                System.out.println(region);
                maxErrorRetry = props.getProperty("s3.maxErrorRetry");
                if (maxErrorRetry == null) {
                    maxErrorRetry = propsCL.getProperty("s3.maxErrorRetry", "15");
                }
                maxConnections = props.getProperty("s3.maxConnections");
                if (maxConnections == null) {
                    maxConnections = propsCL.getProperty("s3.maxConnections");
                }
                protocol = props.getProperty("s3.protocol");
                if (protocol == null) {
                    protocol = propsCL.getProperty("s3.protocol", "HTTPS");
                }
                sse = props.getProperty("s3.sse");
                if (sse == null) {
                    sse = propsCL.getProperty("s3.sse", "false");
                }
                String ssec = props.getProperty("s3.ssec");
                if (ssec == null) {
                    ssec = propsCL.getProperty("s3.ssec", null);
                } else {
                    ssecKey = new SSECustomerKey(ssec);
                }
            } catch (Exception e) {
                System.err.println("The file properties doesn't exist " + e.toString());
                e.printStackTrace();
            }
            try {
                System.out.println("Inizializing the S3 connection");
                s3Credentials = new BasicAWSCredentials(accessKeyId, secretKey);
                clientConfig = new ClientConfiguration();
                clientConfig.setMaxErrorRetry(Integer.parseInt(maxErrorRetry));
                if (protocol.equals("HTTP")) {
                    clientConfig.setProtocol(Protocol.HTTP);
                } else {
                    clientConfig.setProtocol(Protocol.HTTPS);
                }
                if (maxConnections != null) {
                    clientConfig.setMaxConnections(Integer.parseInt(maxConnections));
                }
                s3Client = new AmazonS3Client(s3Credentials, clientConfig);
                s3Client.setRegion(Region.getRegion(Regions.fromName(region)));
                s3Client.setEndpoint(endPoint);
                System.out.println("Connection successfully initialized");
            } catch (Exception e) {
                System.err.println("Could not connect to S3 storage: " + e.toString());
                e.printStackTrace();
                throw new DBException(e);
            }
        } else {
            System.err.println("The number of threads must be less or equal than the operations");
            throw new DBException(new Error("The number of threads must be less or equal than the operations"));
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• »