Example usage for com.amazonaws.services.s3 S3ClientOptions builder

List of usage examples for com.amazonaws.services.s3 S3ClientOptions builder

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.*
  5. S3ClientOptions
  6. builder

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 S3ClientOptions builder.

Prototype

public static Builder builder()

Source Link

Usage

From source file:S3ClientSideEncryptionWithSymmetricMasterKey.java

License:Apache License

public static void main(String[] args) throws Exception {
    SecretKey mySymmetricKey = loadSymmetricAESKey(masterKeyDir, "AES");

    EncryptionMaterials encryptionMaterials = new EncryptionMaterials(mySymmetricKey);

    AWSCredentials credentials = new BasicAWSCredentials("Q3AM3UQ867SPQQA43P2F",
            "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG");
    AmazonS3EncryptionClient encryptionClient = new AmazonS3EncryptionClient(credentials,
            new StaticEncryptionMaterialsProvider(encryptionMaterials));
    Region usEast1 = Region.getRegion(Regions.US_EAST_1);
    encryptionClient.setRegion(usEast1);
    encryptionClient.setEndpoint("https://play.minio.io:9000");

    final S3ClientOptions clientOptions = S3ClientOptions.builder().setPathStyleAccess(true).build();
    encryptionClient.setS3ClientOptions(clientOptions);

    // Create the bucket
    encryptionClient.createBucket(bucketName);

    // Upload object using the encryption client.
    byte[] plaintext = "Hello World, S3 Client-side Encryption Using Asymmetric Master Key!".getBytes();
    System.out.println("plaintext's length: " + plaintext.length);
    encryptionClient.putObject(new PutObjectRequest(bucketName, objectKey, new ByteArrayInputStream(plaintext),
            new ObjectMetadata()));

    // Download the object.
    S3Object downloadedObject = encryptionClient.getObject(bucketName, objectKey);
    byte[] decrypted = IOUtils.toByteArray(downloadedObject.getObjectContent());

    // Verify same data.
    Assert.assertTrue(Arrays.equals(plaintext, decrypted));
    //deleteBucketAndAllContents(encryptionClient);
}

From source file:S3ClientSideEncryptionAsymmetricMasterKey.java

License:Apache License

public static void main(String[] args) throws Exception {

    // 1. Load keys from files
    byte[] bytes = FileUtils.readFileToByteArray(new File(keyDir + "/private.key"));
    KeyFactory kf = KeyFactory.getInstance("RSA");
    PKCS8EncodedKeySpec ks = new PKCS8EncodedKeySpec(bytes);
    PrivateKey pk = kf.generatePrivate(ks);

    bytes = FileUtils.readFileToByteArray(new File(keyDir + "/public.key"));
    PublicKey publicKey = KeyFactory.getInstance("RSA").generatePublic(new X509EncodedKeySpec(bytes));

    KeyPair loadedKeyPair = new KeyPair(publicKey, pk);

    // 2. Construct an instance of AmazonS3EncryptionClient.
    EncryptionMaterials encryptionMaterials = new EncryptionMaterials(loadedKeyPair);
    AWSCredentials credentials = new BasicAWSCredentials("Q3AM3UQ867SPQQA43P2F",
            "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG");
    AmazonS3EncryptionClient encryptionClient = new AmazonS3EncryptionClient(credentials,
            new StaticEncryptionMaterialsProvider(encryptionMaterials));
    Region usEast1 = Region.getRegion(Regions.US_EAST_1);
    encryptionClient.setRegion(usEast1);
    encryptionClient.setEndpoint("https://play.minio.io:9000");

    final S3ClientOptions clientOptions = S3ClientOptions.builder().setPathStyleAccess(true).build();
    encryptionClient.setS3ClientOptions(clientOptions);

    // Create the bucket
    encryptionClient.createBucket(bucketName);
    // 3. Upload the object.
    byte[] plaintext = "Hello World, S3 Client-side Encryption Using Asymmetric Master Key!".getBytes();
    System.out.println("plaintext's length: " + plaintext.length);
    encryptionClient.putObject(new PutObjectRequest(bucketName, objectKey, new ByteArrayInputStream(plaintext),
            new ObjectMetadata()));

    // 4. Download the object.
    S3Object downloadedObject = encryptionClient.getObject(bucketName, objectKey);
    byte[] decrypted = IOUtils.toByteArray(downloadedObject.getObjectContent());
    Assert.assertTrue(Arrays.equals(plaintext, decrypted));
    System.out.println("decrypted length: " + decrypted.length);
    //deleteBucketAndAllContents(encryptionClient);
}

From source file:com.facebook.presto.hive.s3.PrestoS3FileSystem.java

License:Apache License

private AmazonS3Client createAmazonS3Client(URI uri, Configuration hadoopConfig,
        ClientConfiguration clientConfig) {
    AWSCredentialsProvider credentials = getAwsCredentialsProvider(uri, hadoopConfig);
    Optional<EncryptionMaterialsProvider> emp = createEncryptionMaterialsProvider(hadoopConfig);
    AmazonS3Client client;/*from  w  w w .  ja  v  a2s . co  m*/
    String signerType = hadoopConfig.get(S3_SIGNER_TYPE);
    if (signerType != null) {
        clientConfig.withSignerOverride(signerType);
    }
    if (emp.isPresent()) {
        client = new AmazonS3EncryptionClient(credentials, emp.get(), clientConfig, new CryptoConfiguration(),
                METRIC_COLLECTOR);
    } else {
        client = new AmazonS3Client(credentials, clientConfig, METRIC_COLLECTOR);
    }

    if (isPathStyleAccess) {
        S3ClientOptions clientOptions = S3ClientOptions.builder().setPathStyleAccess(true).build();
        client.setS3ClientOptions(clientOptions);
    }

    // use local region when running inside of EC2
    if (pinS3ClientToCurrentRegion) {
        Region region = Regions.getCurrentRegion();
        if (region != null) {
            client.setRegion(region);
        }
    }

    String endpoint = hadoopConfig.get(S3_ENDPOINT);
    if (endpoint != null) {
        client.setEndpoint(endpoint);
    }

    return client;
}

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

@Override
public void initiate(String scheme) throws IOException, ConfigurationParseException {
    mCachedSparkOriginated = new HashMap<String, Boolean>();
    mCachedSparkJobsStatus = new HashMap<String, Boolean>();
    schemaProvided = scheme;/*from   w w w . j a  v  a 2  s.  c  om*/
    Properties props = ConfigurationHandler.initialize(filesystemURI, conf, scheme);
    // Set bucket name property
    int cacheSize = conf.getInt(CACHE_SIZE, GUAVA_CACHE_SIZE_DEFAULT);
    memoryCache = MemoryCache.getInstance(cacheSize);
    mBucket = props.getProperty(COS_BUCKET_PROPERTY);
    workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(filesystemURI,
            getWorkingDirectory());

    fModeAutomaticDelete = "true".equals(props.getProperty(FMODE_AUTOMATIC_DELETE_COS_PROPERTY, "false"));
    mIsV2Signer = "true".equals(props.getProperty(V2_SIGNER_TYPE_COS_PROPERTY, "false"));
    // Define COS client
    String accessKey = props.getProperty(ACCESS_KEY_COS_PROPERTY);
    String secretKey = props.getProperty(SECRET_KEY_COS_PROPERTY);

    if (accessKey == null) {
        throw new ConfigurationParseException("Access KEY is empty. Please provide valid access key");
    }
    if (secretKey == null) {
        throw new ConfigurationParseException("Secret KEY is empty. Please provide valid secret key");
    }

    BasicAWSCredentials creds = new BasicAWSCredentials(accessKey, secretKey);
    ClientConfiguration clientConf = new ClientConfiguration();

    int maxThreads = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, MAX_THREADS, DEFAULT_MAX_THREADS);
    if (maxThreads < 2) {
        LOG.warn(MAX_THREADS + " must be at least 2: forcing to 2.");
        maxThreads = 2;
    }
    int totalTasks = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS);
    long keepAliveTime = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME);
    threadPoolExecutor = BlockingThreadPoolExecutorService.newInstance(maxThreads, maxThreads + totalTasks,
            keepAliveTime, TimeUnit.SECONDS, "s3a-transfer-shared");

    unboundedThreadPool = new ThreadPoolExecutor(maxThreads, Integer.MAX_VALUE, keepAliveTime, TimeUnit.SECONDS,
            new LinkedBlockingQueue<Runnable>(),
            BlockingThreadPoolExecutorService.newDaemonThreadFactory("s3a-transfer-unbounded"));

    boolean secureConnections = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, SECURE_CONNECTIONS,
            DEFAULT_SECURE_CONNECTIONS);
    clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);

    String proxyHost = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_HOST, "");
    int proxyPort = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, PROXY_PORT, -1);
    if (!proxyHost.isEmpty()) {
        clientConf.setProxyHost(proxyHost);
        if (proxyPort >= 0) {
            clientConf.setProxyPort(proxyPort);
        } else {
            if (secureConnections) {
                LOG.warn("Proxy host set without port. Using HTTPS default 443");
                clientConf.setProxyPort(443);
            } else {
                LOG.warn("Proxy host set without port. Using HTTP default 80");
                clientConf.setProxyPort(80);
            }
        }
        String proxyUsername = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_USERNAME);
        String proxyPassword = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_PASSWORD);
        if ((proxyUsername == null) != (proxyPassword == null)) {
            String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD + " set without the other.";
            LOG.error(msg);
            throw new IllegalArgumentException(msg);
        }
        clientConf.setProxyUsername(proxyUsername);
        clientConf.setProxyPassword(proxyPassword);
        clientConf.setProxyDomain(Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_DOMAIN));
        clientConf.setProxyWorkstation(Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_WORKSTATION));
        if (LOG.isDebugEnabled()) {
            LOG.debug(
                    "Using proxy server {}:{} as user {} with password {} on " + "domain {} as workstation {}",
                    clientConf.getProxyHost(), clientConf.getProxyPort(),
                    String.valueOf(clientConf.getProxyUsername()), clientConf.getProxyPassword(),
                    clientConf.getProxyDomain(), clientConf.getProxyWorkstation());
        }
    } else if (proxyPort >= 0) {
        String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
        LOG.error(msg);
        throw new IllegalArgumentException(msg);
    }

    initConnectionSettings(conf, clientConf);
    if (mIsV2Signer) {
        clientConf.withSignerOverride("S3SignerType");
    }
    mClient = new AmazonS3Client(creds, clientConf);

    final String serviceUrl = props.getProperty(ENDPOINT_URL_COS_PROPERTY);
    if (serviceUrl != null && !serviceUrl.equals(amazonDefaultEndpoint)) {
        mClient.setEndpoint(serviceUrl);
    }
    mClient.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());

    // Set block size property
    String mBlockSizeString = props.getProperty(BLOCK_SIZE_COS_PROPERTY, "128");
    mBlockSize = Long.valueOf(mBlockSizeString).longValue() * 1024 * 1024L;

    boolean autoCreateBucket = "true"
            .equalsIgnoreCase((props.getProperty(AUTO_BUCKET_CREATE_COS_PROPERTY, "false")));

    partSize = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
    multiPartThreshold = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, MIN_MULTIPART_THRESHOLD,
            DEFAULT_MIN_MULTIPART_THRESHOLD);
    readAhead = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, READAHEAD_RANGE, DEFAULT_READAHEAD_RANGE);
    LOG.debug(READAHEAD_RANGE + ":" + readAhead);
    inputPolicy = COSInputPolicy
            .getPolicy(Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, INPUT_FADVISE, INPUT_FADV_NORMAL));

    initTransferManager();
    maxKeys = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
    flatListingFlag = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, FLAT_LISTING, DEFAULT_FLAT_LISTING);

    if (autoCreateBucket) {
        try {
            boolean bucketExist = mClient.doesBucketExist(mBucket);
            if (bucketExist) {
                LOG.trace("Bucket {} exists", mBucket);
            } else {
                LOG.trace("Bucket {} doesn`t exists and autocreate", mBucket);
                String mRegion = props.getProperty(REGION_COS_PROPERTY);
                if (mRegion == null) {
                    mClient.createBucket(mBucket);
                } else {
                    LOG.trace("Creating bucket {} in region {}", mBucket, mRegion);
                    mClient.createBucket(mBucket, mRegion);
                }
            }
        } catch (AmazonServiceException ase) {
            /*
            *  we ignore the BucketAlreadyExists exception since multiple processes or threads
            *  might try to create the bucket in parrallel, therefore it is expected that
            *  some will fail to create the bucket
            */
            if (!ase.getErrorCode().equals("BucketAlreadyExists")) {
                LOG.error(ase.getMessage());
                throw (ase);
            }
        } catch (Exception e) {
            LOG.error(e.getMessage());
            throw (e);
        }
    }

    initMultipartUploads(conf);
    enableMultiObjectsDelete = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, ENABLE_MULTI_DELETE, true);

    blockUploadEnabled = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, FAST_UPLOAD, DEFAULT_FAST_UPLOAD);

    if (blockUploadEnabled) {
        blockOutputBuffer = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, FAST_UPLOAD_BUFFER,
                DEFAULT_FAST_UPLOAD_BUFFER);
        partSize = COSUtils.ensureOutputParameterInRange(MULTIPART_SIZE, partSize);
        blockFactory = COSDataBlocks.createFactory(this, blockOutputBuffer);
        blockOutputActiveBlocks = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, FAST_UPLOAD_ACTIVE_BLOCKS,
                DEFAULT_FAST_UPLOAD_ACTIVE_BLOCKS);
        LOG.debug("Using COSBlockOutputStream with buffer = {}; block={};" + " queue limit={}",
                blockOutputBuffer, partSize, blockOutputActiveBlocks);
    } else {
        LOG.debug("Using COSOutputStream");
    }
}

From source file:com.upplication.s3fs.S3FileSystemProvider.java

License:Open Source License

protected S3FileSystem createFileSystem0(URI uri, Object accessKey, Object secretKey, Object sessionToken) {
    AmazonS3Client client;/*from w  w w  .j a  v a  2s.  com*/
    ClientConfiguration config = createClientConfig(props);

    if (accessKey == null && secretKey == null) {
        client = new AmazonS3Client(new com.amazonaws.services.s3.AmazonS3Client(config));
    } else {

        AWSCredentials credentials = (sessionToken == null
                ? new BasicAWSCredentials(accessKey.toString(), secretKey.toString())
                : new BasicSessionCredentials(accessKey.toString(), secretKey.toString(),
                        sessionToken.toString()));
        client = new AmazonS3Client(new com.amazonaws.services.s3.AmazonS3Client(credentials, config));
    }

    // note: path style access is going to be deprecated
    // https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/
    boolean usePathStyle = "true".equals(props.getProperty("s_3_path_style_access"))
            || "true".equals(props.getProperty("s3_path_style_access"));
    if (usePathStyle) {
        S3ClientOptions options = S3ClientOptions.builder().setPathStyleAccess(usePathStyle).build();
        client.client.setS3ClientOptions(options);
    }

    if (uri.getHost() != null) {
        client.setEndpoint(uri.getHost());
    } else if (props.getProperty("endpoint") != null) {
        client.setEndpoint(props.getProperty("endpoint"));
    } else if (props.getProperty("region") != null) {
        client.setRegion(props.getProperty("region"));
    }

    S3FileSystem result = new S3FileSystem(this, client, uri.getHost());
    return result;
}

From source file:org.geoserver.taskmanager.external.impl.S3FileServiceImpl.java

License:Open Source License

private AmazonS3 getS3Client() {
    if (endpoint == null) {
        throw new IllegalArgumentException("The endpoint is required, add a property: alias.s3.endpoint");
    }/*ww w .  java 2 s  . co m*/
    if (user == null) {
        throw new IllegalArgumentException("The user is required, add a property: alias.s3.user");
    }
    if (password == null) {
        throw new IllegalArgumentException("The password is required, add a property: alias.s3.password");
    }
    if (rootFolder == null) {
        throw new IllegalStateException("The rootfolder is required, add a property: alias.s3.rootfolder");
    }

    AmazonS3 s3;
    // custom endpoint

    s3 = new AmazonS3Client(new BasicAWSCredentials(user, password));

    final S3ClientOptions clientOptions = S3ClientOptions.builder().setPathStyleAccess(true).build();
    s3.setS3ClientOptions(clientOptions);
    String endpoint = this.endpoint;
    if (!endpoint.endsWith("/")) {
        endpoint = endpoint + "/";
    }
    s3.setEndpoint(endpoint);

    return s3;
}

From source file:org.whispersystems.textsecuregcm.s3.UrlSigner.java

License:Open Source License

public URL getPreSignedUrl(long attachmentId, HttpMethod method) {
    AmazonS3 client = new AmazonS3Client(credentials);
    GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket, String.valueOf(attachmentId),
            method);/*from  www . ja v  a 2  s .  c  o  m*/

    request.setExpiration(new Date(System.currentTimeMillis() + DURATION));
    request.setContentType("application/octet-stream");

    client.setS3ClientOptions(S3ClientOptions.builder().setAccelerateModeEnabled(true).build());

    return client.generatePresignedUrl(request);
}