Example usage for com.amazonaws.services.kinesisfirehose.model S3DestinationConfiguration setPrefix

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesisfirehose.model S3DestinationConfiguration setPrefix.

Prototype

public void setPrefix(String prefix) 

Source Link

Document

The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered Amazon S3 files.

Usage

From source file:AmazonKinesisFirehoseToRedshiftSample.java

License:Open Source License

/**
 * Method to create delivery stream with Redshift destination configuration.
 *
 * @throws Exception/*ww  w. ja  v  a  2s.  c o  m*/
 */
private static void createDeliveryStream() throws Exception {

    boolean deliveryStreamExists = false;

    LOG.info("Checking if " + deliveryStreamName + " already exits");
    List<String> deliveryStreamNames = listDeliveryStreams();
    if (deliveryStreamNames != null && deliveryStreamNames.contains(deliveryStreamName)) {
        deliveryStreamExists = true;
        LOG.info("DeliveryStream " + deliveryStreamName
                + " already exists. Not creating the new delivery stream");
    } else {
        LOG.info("DeliveryStream " + deliveryStreamName + " does not exist");
    }

    if (!deliveryStreamExists) {
        // Create DeliveryStream
        CreateDeliveryStreamRequest createDeliveryStreamRequest = new CreateDeliveryStreamRequest();
        createDeliveryStreamRequest.setDeliveryStreamName(deliveryStreamName);

        S3DestinationConfiguration redshiftS3Configuration = new S3DestinationConfiguration();
        redshiftS3Configuration.setBucketARN(s3BucketARN);
        redshiftS3Configuration.setPrefix(s3ObjectPrefix);

        BufferingHints bufferingHints = null;
        if (s3DestinationSizeInMBs != null || s3DestinationIntervalInSeconds != null) {
            bufferingHints = new BufferingHints();
            bufferingHints.setSizeInMBs(s3DestinationSizeInMBs);
            bufferingHints.setIntervalInSeconds(s3DestinationIntervalInSeconds);
        }
        redshiftS3Configuration.setBufferingHints(bufferingHints);

        // Create and set IAM role so that firehose service has access to the S3Buckets to put data. 
        // Please check the trustPolicyDocument.json and permissionsPolicyDocument.json files 
        // for the trust and permissions policies set for the role.
        String iamRoleArn = createIamRole(s3ObjectPrefix);
        redshiftS3Configuration.setRoleARN(iamRoleArn);

        CopyCommand copyCommand = new CopyCommand();
        copyCommand.withCopyOptions(copyOptions).withDataTableName(dataTableName);

        RedshiftDestinationConfiguration redshiftDestinationConfiguration = new RedshiftDestinationConfiguration();
        redshiftDestinationConfiguration.withClusterJDBCURL(clusterJDBCUrl).withRoleARN(iamRoleArn)
                .withUsername(username).withPassword(password).withCopyCommand(copyCommand)
                .withS3Configuration(redshiftS3Configuration);

        createDeliveryStreamRequest.setRedshiftDestinationConfiguration(redshiftDestinationConfiguration);

        firehoseClient.createDeliveryStream(createDeliveryStreamRequest);

        // The Delivery Stream is now being created.
        LOG.info("Creating DeliveryStream : " + deliveryStreamName);
        waitForDeliveryStreamToBecomeAvailable(deliveryStreamName);
    }
}

---

From source file:AmazonKinesisFirehoseToS3Sample.java

License:Open Source License

/**
 * Method to create delivery stream for S3 destination configuration.
 *
 * @throws Exception//from w  ww .j  a va2  s .  c  o m
 */
private static void createDeliveryStream() throws Exception {

    boolean deliveryStreamExists = false;

    LOG.info("Checking if " + deliveryStreamName + " already exits");
    List<String> deliveryStreamNames = listDeliveryStreams();
    if (deliveryStreamNames != null && deliveryStreamNames.contains(deliveryStreamName)) {
        deliveryStreamExists = true;
        LOG.info("DeliveryStream " + deliveryStreamName
                + " already exists. Not creating the new delivery stream");
    } else {
        LOG.info("DeliveryStream " + deliveryStreamName + " does not exist");
    }

    if (!deliveryStreamExists) {
        // Create deliveryStream
        CreateDeliveryStreamRequest createDeliveryStreamRequest = new CreateDeliveryStreamRequest();
        createDeliveryStreamRequest.setDeliveryStreamName(deliveryStreamName);

        S3DestinationConfiguration s3DestinationConfiguration = new S3DestinationConfiguration();
        s3DestinationConfiguration.setBucketARN(s3BucketARN);
        s3DestinationConfiguration.setPrefix(s3ObjectPrefix);
        // Could also specify GZIP or ZIP
        s3DestinationConfiguration.setCompressionFormat(CompressionFormat.UNCOMPRESSED);

        // Encryption configuration is optional
        EncryptionConfiguration encryptionConfiguration = new EncryptionConfiguration();
        if (!StringUtils.isNullOrEmpty(s3DestinationAWSKMSKeyId)) {
            encryptionConfiguration.setKMSEncryptionConfig(
                    new KMSEncryptionConfig().withAWSKMSKeyARN(s3DestinationAWSKMSKeyId));
        } else {
            encryptionConfiguration.setNoEncryptionConfig(NoEncryptionConfig.NoEncryption);
        }
        s3DestinationConfiguration.setEncryptionConfiguration(encryptionConfiguration);

        BufferingHints bufferingHints = null;
        if (s3DestinationSizeInMBs != null || s3DestinationIntervalInSeconds != null) {
            bufferingHints = new BufferingHints();
            bufferingHints.setSizeInMBs(s3DestinationSizeInMBs);
            bufferingHints.setIntervalInSeconds(s3DestinationIntervalInSeconds);
        }
        s3DestinationConfiguration.setBufferingHints(bufferingHints);

        // Create and set IAM role so that firehose service has access to the S3Buckets to put data 
        // and KMS keys (if provided) to encrypt data. Please check the trustPolicyDocument.json and 
        // permissionsPolicyDocument.json files for the trust and permissions policies set for the role.
        String iamRoleArn = createIamRole(s3ObjectPrefix);
        s3DestinationConfiguration.setRoleARN(iamRoleArn);

        createDeliveryStreamRequest.setS3DestinationConfiguration(s3DestinationConfiguration);

        firehoseClient.createDeliveryStream(createDeliveryStreamRequest);

        // The Delivery Stream is now being created.
        LOG.info("Creating DeliveryStream : " + deliveryStreamName);
        waitForDeliveryStreamToBecomeAvailable(deliveryStreamName);
    }
}

---