Example usage for com.amazonaws.services.s3 AmazonS3Client setS3ClientOptions

List of usage examples for com.amazonaws.services.s3 AmazonS3Client setS3ClientOptions

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.*
  5. AmazonS3Client
  6. setS3ClientOptions

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 AmazonS3Client setS3ClientOptions.

Prototype

@Override
public synchronized void setS3ClientOptions(S3ClientOptions clientOptions)

Source Link

Document

Override the default S3 client options for this client.

Usage

From source file:UploadUrlGenerator.java

License:Open Source License

public static void main(String[] args) throws Exception {
    Options opts = new Options();

    opts.addOption(Option.builder().longOpt(ENDPOINT_OPTION).required().hasArg().argName("url")
            .desc("Sets the ECS S3 endpoint to use, e.g. https://ecs.company.com:9021").build());
    opts.addOption(Option.builder().longOpt(ACCESS_KEY_OPTION).required().hasArg().argName("access-key")
            .desc("Sets the Access Key (user) to sign the request").build());
    opts.addOption(Option.builder().longOpt(SECRET_KEY_OPTION).required().hasArg().argName("secret")
            .desc("Sets the secret key to sign the request").build());
    opts.addOption(Option.builder().longOpt(BUCKET_OPTION).required().hasArg().argName("bucket-name")
            .desc("The bucket containing the object").build());
    opts.addOption(Option.builder().longOpt(KEY_OPTION).required().hasArg().argName("object-key")
            .desc("The object name (key) to access with the URL").build());
    opts.addOption(Option.builder().longOpt(EXPIRES_OPTION).hasArg().argName("minutes")
            .desc("Minutes from local time to expire the request.  1 day = 1440, 1 week=10080, "
                    + "1 month (30 days)=43200, 1 year=525600.  Defaults to 1 hour (60).")
            .build());// w  ww  .j  ava2s.  co  m
    opts.addOption(Option.builder().longOpt(VERB_OPTION).hasArg().argName("http-verb").type(HttpMethod.class)
            .desc("The HTTP verb that will be used with the URL (PUT, GET, etc).  Defaults to GET.").build());
    opts.addOption(Option.builder().longOpt(CONTENT_TYPE_OPTION).hasArg().argName("mimetype")
            .desc("Sets the Content-Type header (e.g. image/jpeg) that will be used with the request.  "
                    + "Must match exactly.  Defaults to application/octet-stream for PUT/POST and "
                    + "null for all others")
            .build());

    DefaultParser dp = new DefaultParser();

    CommandLine cmd = null;
    try {
        cmd = dp.parse(opts, args);
    } catch (ParseException e) {
        System.err.println("Error: " + e.getMessage());
        HelpFormatter hf = new HelpFormatter();
        hf.printHelp("java -jar UploadUrlGenerator-xxx.jar", opts, true);
        System.exit(255);
    }

    URI endpoint = URI.create(cmd.getOptionValue(ENDPOINT_OPTION));
    String accessKey = cmd.getOptionValue(ACCESS_KEY_OPTION);
    String secretKey = cmd.getOptionValue(SECRET_KEY_OPTION);
    String bucket = cmd.getOptionValue(BUCKET_OPTION);
    String key = cmd.getOptionValue(KEY_OPTION);
    HttpMethod method = HttpMethod.GET;
    if (cmd.hasOption(VERB_OPTION)) {
        method = HttpMethod.valueOf(cmd.getOptionValue(VERB_OPTION).toUpperCase());
    }
    int expiresMinutes = 60;
    if (cmd.hasOption(EXPIRES_OPTION)) {
        expiresMinutes = Integer.parseInt(cmd.getOptionValue(EXPIRES_OPTION));
    }
    String contentType = null;
    if (method == HttpMethod.PUT || method == HttpMethod.POST) {
        contentType = "application/octet-stream";
    }

    if (cmd.hasOption(CONTENT_TYPE_OPTION)) {
        contentType = cmd.getOptionValue(CONTENT_TYPE_OPTION);
    }

    BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
    ClientConfiguration cc = new ClientConfiguration();
    // Force use of v2 Signer.  ECS does not support v4 signatures yet.
    cc.setSignerOverride("S3SignerType");
    AmazonS3Client s3 = new AmazonS3Client(credentials, cc);
    s3.setEndpoint(endpoint.toString());
    S3ClientOptions s3c = new S3ClientOptions();
    s3c.setPathStyleAccess(true);
    s3.setS3ClientOptions(s3c);

    // Sign the URL
    Calendar c = Calendar.getInstance();
    c.add(Calendar.MINUTE, expiresMinutes);
    GeneratePresignedUrlRequest req = new GeneratePresignedUrlRequest(bucket, key).withExpiration(c.getTime())
            .withMethod(method);
    if (contentType != null) {
        req = req.withContentType(contentType);
    }
    URL u = s3.generatePresignedUrl(req);
    System.out.printf("URL: %s\n", u.toURI().toASCIIString());
    System.out.printf("HTTP Verb: %s\n", method);
    System.out.printf("Expires: %s\n", c.getTime());
    System.out.println("To Upload with curl:");

    StringBuilder sb = new StringBuilder();
    sb.append("curl ");

    if (method != HttpMethod.GET) {
        sb.append("-X ");
        sb.append(method.toString());
        sb.append(" ");
    }

    if (contentType != null) {
        sb.append("-H \"Content-Type: ");
        sb.append(contentType);
        sb.append("\" ");
    }

    if (method == HttpMethod.POST || method == HttpMethod.PUT) {
        sb.append("-T <filename> ");
    }

    sb.append("\"");
    sb.append(u.toURI().toASCIIString());
    sb.append("\"");

    System.out.println(sb.toString());

    System.exit(0);
}

From source file:br.com.semanticwot.cd.conf.AmazonConfiguration.java

@Bean
@Profile("dev")/*ww  w . j  a va 2 s  .  c  o  m*/
public AmazonS3Client s3Ninja() {
    AWSCredentials credentials = new BasicAWSCredentials("AKIAIOSFODNN7EXAMPLE",
            "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
    AmazonS3Client newClient = new AmazonS3Client(credentials, new ClientConfiguration());
    newClient.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));

    newClient.setEndpoint("http://localhost:9444/s3");
    return newClient;

}

From source file:br.com.semanticwot.cd.conf.AmazonConfiguration.java

@Bean
@Profile("prod")//from w  ww .  j  a  v  a 2s . com
public AmazonS3Client s3Amazon() {
    AWSCredentials credentials = new BasicAWSCredentials("AKIAIOSFODNN7EXAMPLE",
            "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
    AmazonS3Client newClient = new AmazonS3Client(credentials, new ClientConfiguration());
    newClient.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));

    return newClient;

}

From source file:com.eucalyptus.objectstorage.client.GenericS3ClientFactory.java

License:Open Source License

public static AmazonS3Client getS3Client(AWSCredentialsProvider provider, boolean https)
        throws NoSuchElementException {
    ClientConfiguration config = getDefaultConfiguration(https);
    AmazonS3Client s3Client = new AmazonS3Client(provider, config);
    s3Client.setS3ClientOptions(getDefaultClientOptions());
    s3Client.setEndpoint(getRandomOSGUri().toString());
    return s3Client;
}

From source file:com.eucalyptus.objectstorage.client.GenericS3ClientFactory.java

License:Open Source License

/**
 * Returns a configured S3 client for the specified set of credentials.
 * @param credentials/*from ww w.ja v  a  2s .c om*/
 * @param https
 * @return
 * @throws NoSuchElementException if no ENABLED OSG found
 */
public static AmazonS3Client getS3Client(AWSCredentials credentials, boolean https)
        throws NoSuchElementException {
    ClientConfiguration config = getDefaultConfiguration(https);
    AmazonS3Client s3Client = new AmazonS3Client(credentials, config);
    s3Client.setS3ClientOptions(getDefaultClientOptions());
    s3Client.setEndpoint(getRandomOSGUri().toString());
    return s3Client;
}

From source file:com.facebook.presto.hive.s3.PrestoS3FileSystem.java

License:Apache License

private AmazonS3Client createAmazonS3Client(URI uri, Configuration hadoopConfig,
        ClientConfiguration clientConfig) {
    AWSCredentialsProvider credentials = getAwsCredentialsProvider(uri, hadoopConfig);
    Optional<EncryptionMaterialsProvider> emp = createEncryptionMaterialsProvider(hadoopConfig);
    AmazonS3Client client;
    String signerType = hadoopConfig.get(S3_SIGNER_TYPE);
    if (signerType != null) {
        clientConfig.withSignerOverride(signerType);
    }/*  w ww  . ja  v  a 2 s.  c om*/
    if (emp.isPresent()) {
        client = new AmazonS3EncryptionClient(credentials, emp.get(), clientConfig, new CryptoConfiguration(),
                METRIC_COLLECTOR);
    } else {
        client = new AmazonS3Client(credentials, clientConfig, METRIC_COLLECTOR);
    }

    if (isPathStyleAccess) {
        S3ClientOptions clientOptions = S3ClientOptions.builder().setPathStyleAccess(true).build();
        client.setS3ClientOptions(clientOptions);
    }

    // use local region when running inside of EC2
    if (pinS3ClientToCurrentRegion) {
        Region region = Regions.getCurrentRegion();
        if (region != null) {
            client.setRegion(region);
        }
    }

    String endpoint = hadoopConfig.get(S3_ENDPOINT);
    if (endpoint != null) {
        client.setEndpoint(endpoint);
    }

    return client;
}

From source file:com.ge.predix.sample.blobstore.connector.spring.BlobstoreServiceConnectorCreator.java

License:Apache License

/**
 * Creates the BlobStore context using S3Client
 *
 * @param serviceInfo Object Store Service Info Object
 * @param serviceConnectorConfig Cloud Foundry Service Connector Configuration
 *
 * @return BlobstoreService Instance of the ObjectStore Service
 *///  w ww .j av  a 2  s  . c  o  m
@Override
public BlobstoreService create(BlobstoreServiceInfo serviceInfo,
        ServiceConnectorConfig serviceConnectorConfig) {
    log.info("create() invoked with serviceInfo? = " + (serviceInfo == null));
    ClientConfiguration config = new ClientConfiguration();
    config.setProtocol(Protocol.HTTPS);

    S3ClientOptions options = new S3ClientOptions();
    config.setSignerOverride("S3SignerType");

    BasicAWSCredentials creds = new BasicAWSCredentials(serviceInfo.getObjectStoreAccessKey(),
            serviceInfo.getObjectStoreSecretKey());
    AmazonS3Client s3Client = new AmazonS3Client(creds, config);
    s3Client.setEndpoint(serviceInfo.getUrl());
    s3Client.setS3ClientOptions(options);

    try {
        // Remove the Credentials from the Object Store URL
        URL url = new URL(serviceInfo.getUrl());
        String urlWithoutCredentials = url.getProtocol() + "://" + url.getHost();

        // Return BlobstoreService
        return new BlobstoreService(s3Client, serviceInfo.getBucket(), urlWithoutCredentials);
    } catch (MalformedURLException e) {
        log.error("create(): Couldnt parse the URL provided by VCAP_SERVICES. Exception = " + e.getMessage());
        throw new RuntimeException("Blobstore URL is Invalid", e);
    }
}

From source file:com.mesosphere.dcos.cassandra.executor.backup.S3StorageDriver.java

License:Apache License

private AmazonS3Client getAmazonS3Client(BackupRestoreContext ctx) throws URISyntaxException {
    final String accessKey = ctx.getAccountId();
    final String secretKey = ctx.getSecretKey();
    String endpoint = getEndpoint(ctx);
    LOGGER.info("endpoint: {}", endpoint);

    final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretKey);
    final AmazonS3Client amazonS3Client = new AmazonS3Client(basicAWSCredentials);
    amazonS3Client.setEndpoint(endpoint);

    if (ctx.usesEmc()) {
        final S3ClientOptions options = new S3ClientOptions();
        options.setPathStyleAccess(true);
        amazonS3Client.setS3ClientOptions(options);
    }/*from w  ww  . j ava2 s .  com*/

    return amazonS3Client;
}

From source file:com.streamsets.datacollector.bundles.SupportBundleManager.java

License:Apache License

/**
 * Instead of providing support bundle directly to user, upload it to StreamSets backend services.
 *//*  w ww.ja v  a 2 s  .  c  o  m*/
public void uploadNewBundleFromInstances(List<BundleContentGenerator> generators, BundleType bundleType)
        throws IOException {
    // Generate bundle
    SupportBundle bundle = generateNewBundleFromInstances(generators, bundleType);

    boolean enabled = configuration.get(Constants.UPLOAD_ENABLED, Constants.DEFAULT_UPLOAD_ENABLED);
    String accessKey = configuration.get(Constants.UPLOAD_ACCESS, Constants.DEFAULT_UPLOAD_ACCESS);
    String secretKey = configuration.get(Constants.UPLOAD_SECRET, Constants.DEFAULT_UPLOAD_SECRET);
    String bucket = configuration.get(Constants.UPLOAD_BUCKET, Constants.DEFAULT_UPLOAD_BUCKET);
    int bufferSize = configuration.get(Constants.UPLOAD_BUFFER_SIZE, Constants.DEFAULT_UPLOAD_BUFFER_SIZE);

    if (!enabled) {
        throw new IOException("Uploading support bundles was disabled by administrator.");
    }

    AWSCredentialsProvider credentialsProvider = new StaticCredentialsProvider(
            new BasicAWSCredentials(accessKey, secretKey));
    AmazonS3Client s3Client = new AmazonS3Client(credentialsProvider, new ClientConfiguration());
    s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
    s3Client.setRegion(Region.getRegion(Regions.US_WEST_2));

    // Object Metadata
    ObjectMetadata s3Metadata = new ObjectMetadata();
    for (Map.Entry<Object, Object> entry : getMetadata(bundleType).entrySet()) {
        s3Metadata.addUserMetadata((String) entry.getKey(), (String) entry.getValue());
    }

    List<PartETag> partETags;
    InitiateMultipartUploadResult initResponse = null;
    try {
        // Uploading part by part
        LOG.info("Initiating multi-part support bundle upload");
        partETags = new ArrayList<>();
        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket,
                bundle.getBundleKey());
        initRequest.setObjectMetadata(s3Metadata);
        initResponse = s3Client.initiateMultipartUpload(initRequest);
    } catch (AmazonClientException e) {
        LOG.error("Support bundle upload failed: ", e);
        throw new IOException("Support bundle upload failed", e);
    }

    try {
        byte[] buffer = new byte[bufferSize];
        int partId = 1;
        int size = -1;
        while ((size = readFully(bundle.getInputStream(), buffer)) != -1) {
            LOG.debug("Uploading part {} of size {}", partId, size);
            UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(bucket)
                    .withKey(bundle.getBundleKey()).withUploadId(initResponse.getUploadId())
                    .withPartNumber(partId++).withInputStream(new ByteArrayInputStream(buffer))
                    .withPartSize(size);

            partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
        }

        CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket,
                bundle.getBundleKey(), initResponse.getUploadId(), partETags);

        s3Client.completeMultipartUpload(compRequest);
        LOG.info("Support bundle upload finished");
    } catch (Exception e) {
        LOG.error("Support bundle upload failed", e);
        s3Client.abortMultipartUpload(
                new AbortMultipartUploadRequest(bucket, bundle.getBundleKey(), initResponse.getUploadId()));

        throw new IOException("Can't upload support bundle", e);
    } finally {
        // Close the client
        s3Client.shutdown();
    }
}

From source file:hu.mta.sztaki.lpds.cloud.entice.imageoptimizer.iaashandler.amazontarget.Storage.java

License:Apache License

/**
 * @param endpoint S3 endpoint URL//  w  w w .j  a  v  a  2  s.c om
 * @param accessKey Access key
 * @param secretKey Secret key
 * @param bucket Bucket name 
 * @param path Key name of the object to download (path + file name)
 * @param file Local file to download to 
 * @throws Exception On any error
 */
public static void download(String endpoint, String accessKey, String secretKey, String bucket, String path,
        File file) throws Exception {
    AmazonS3Client amazonS3Client = null;
    InputStream in = null;
    OutputStream out = null;
    try {
        AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey);
        ClientConfiguration clientConfiguration = new ClientConfiguration();
        clientConfiguration.setMaxConnections(MAX_CONNECTIONS);
        clientConfiguration.setMaxErrorRetry(PredefinedRetryPolicies.DEFAULT_MAX_ERROR_RETRY);
        clientConfiguration.setConnectionTimeout(ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT);
        amazonS3Client = new AmazonS3Client(awsCredentials, clientConfiguration);
        S3ClientOptions clientOptions = new S3ClientOptions().withPathStyleAccess(true);
        amazonS3Client.setS3ClientOptions(clientOptions);
        amazonS3Client.setEndpoint(endpoint);
        S3Object object = amazonS3Client.getObject(new GetObjectRequest(bucket, path));
        in = object.getObjectContent();
        byte[] buf = new byte[BUFFER_SIZE];
        out = new FileOutputStream(file);
        int count;
        while ((count = in.read(buf)) != -1)
            out.write(buf, 0, count);
        out.close();
        in.close();
    } catch (AmazonServiceException x) {
        Shrinker.myLogger.info("download error: " + x.getMessage());
        throw new Exception("download exception", x);
    } catch (AmazonClientException x) {
        Shrinker.myLogger.info("download error: " + x.getMessage());
        throw new Exception("download exception", x);
    } catch (IOException x) {
        Shrinker.myLogger.info("download error: " + x.getMessage());
        throw new Exception("download exception", x);
    } finally {
        if (in != null) {
            try {
                in.close();
            } catch (Exception e) {
            }
        }
        if (out != null) {
            try {
                out.close();
            } catch (Exception e) {
            }
        }
        if (amazonS3Client != null) {
            try {
                amazonS3Client.shutdown();
            } catch (Exception e) {
            }
        }
    }
}