Example usage for com.amazonaws.services.kinesisfirehose.model BufferingHints setSizeInMBs

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesisfirehose.model BufferingHints setSizeInMBs.

Prototype


public void setSizeInMBs(Integer sizeInMBs)

Source Link

Document

Buffer incoming data to the specified size, in MiBs, before delivering it to the destination.

Usage

From source file:AmazonKinesisFirehoseToRedshiftSample.java

License:Open Source License

/**
 * Method to create delivery stream with Redshift destination configuration.
 *
 * @throws Exception//w  w  w . j  av  a  2s .  c  o m
 */
private static void createDeliveryStream() throws Exception {

    boolean deliveryStreamExists = false;

    LOG.info("Checking if " + deliveryStreamName + " already exits");
    List<String> deliveryStreamNames = listDeliveryStreams();
    if (deliveryStreamNames != null && deliveryStreamNames.contains(deliveryStreamName)) {
        deliveryStreamExists = true;
        LOG.info("DeliveryStream " + deliveryStreamName
                + " already exists. Not creating the new delivery stream");
    } else {
        LOG.info("DeliveryStream " + deliveryStreamName + " does not exist");
    }

    if (!deliveryStreamExists) {
        // Create DeliveryStream
        CreateDeliveryStreamRequest createDeliveryStreamRequest = new CreateDeliveryStreamRequest();
        createDeliveryStreamRequest.setDeliveryStreamName(deliveryStreamName);

        S3DestinationConfiguration redshiftS3Configuration = new S3DestinationConfiguration();
        redshiftS3Configuration.setBucketARN(s3BucketARN);
        redshiftS3Configuration.setPrefix(s3ObjectPrefix);

        BufferingHints bufferingHints = null;
        if (s3DestinationSizeInMBs != null || s3DestinationIntervalInSeconds != null) {
            bufferingHints = new BufferingHints();
            bufferingHints.setSizeInMBs(s3DestinationSizeInMBs);
            bufferingHints.setIntervalInSeconds(s3DestinationIntervalInSeconds);
        }
        redshiftS3Configuration.setBufferingHints(bufferingHints);

        // Create and set IAM role so that firehose service has access to the S3Buckets to put data. 
        // Please check the trustPolicyDocument.json and permissionsPolicyDocument.json files 
        // for the trust and permissions policies set for the role.
        String iamRoleArn = createIamRole(s3ObjectPrefix);
        redshiftS3Configuration.setRoleARN(iamRoleArn);

        CopyCommand copyCommand = new CopyCommand();
        copyCommand.withCopyOptions(copyOptions).withDataTableName(dataTableName);

        RedshiftDestinationConfiguration redshiftDestinationConfiguration = new RedshiftDestinationConfiguration();
        redshiftDestinationConfiguration.withClusterJDBCURL(clusterJDBCUrl).withRoleARN(iamRoleArn)
                .withUsername(username).withPassword(password).withCopyCommand(copyCommand)
                .withS3Configuration(redshiftS3Configuration);

        createDeliveryStreamRequest.setRedshiftDestinationConfiguration(redshiftDestinationConfiguration);

        firehoseClient.createDeliveryStream(createDeliveryStreamRequest);

        // The Delivery Stream is now being created.
        LOG.info("Creating DeliveryStream : " + deliveryStreamName);
        waitForDeliveryStreamToBecomeAvailable(deliveryStreamName);
    }
}

From source file:AmazonKinesisFirehoseToS3Sample.java

License:Open Source License

/**
 * Method to create delivery stream for S3 destination configuration.
 *
 * @throws Exception//  ww w.j  a v  a2 s. co m
 */
private static void createDeliveryStream() throws Exception {

    boolean deliveryStreamExists = false;

    LOG.info("Checking if " + deliveryStreamName + " already exits");
    List<String> deliveryStreamNames = listDeliveryStreams();
    if (deliveryStreamNames != null && deliveryStreamNames.contains(deliveryStreamName)) {
        deliveryStreamExists = true;
        LOG.info("DeliveryStream " + deliveryStreamName
                + " already exists. Not creating the new delivery stream");
    } else {
        LOG.info("DeliveryStream " + deliveryStreamName + " does not exist");
    }

    if (!deliveryStreamExists) {
        // Create deliveryStream
        CreateDeliveryStreamRequest createDeliveryStreamRequest = new CreateDeliveryStreamRequest();
        createDeliveryStreamRequest.setDeliveryStreamName(deliveryStreamName);

        S3DestinationConfiguration s3DestinationConfiguration = new S3DestinationConfiguration();
        s3DestinationConfiguration.setBucketARN(s3BucketARN);
        s3DestinationConfiguration.setPrefix(s3ObjectPrefix);
        // Could also specify GZIP or ZIP
        s3DestinationConfiguration.setCompressionFormat(CompressionFormat.UNCOMPRESSED);

        // Encryption configuration is optional
        EncryptionConfiguration encryptionConfiguration = new EncryptionConfiguration();
        if (!StringUtils.isNullOrEmpty(s3DestinationAWSKMSKeyId)) {
            encryptionConfiguration.setKMSEncryptionConfig(
                    new KMSEncryptionConfig().withAWSKMSKeyARN(s3DestinationAWSKMSKeyId));
        } else {
            encryptionConfiguration.setNoEncryptionConfig(NoEncryptionConfig.NoEncryption);
        }
        s3DestinationConfiguration.setEncryptionConfiguration(encryptionConfiguration);

        BufferingHints bufferingHints = null;
        if (s3DestinationSizeInMBs != null || s3DestinationIntervalInSeconds != null) {
            bufferingHints = new BufferingHints();
            bufferingHints.setSizeInMBs(s3DestinationSizeInMBs);
            bufferingHints.setIntervalInSeconds(s3DestinationIntervalInSeconds);
        }
        s3DestinationConfiguration.setBufferingHints(bufferingHints);

        // Create and set IAM role so that firehose service has access to the S3Buckets to put data 
        // and KMS keys (if provided) to encrypt data. Please check the trustPolicyDocument.json and 
        // permissionsPolicyDocument.json files for the trust and permissions policies set for the role.
        String iamRoleArn = createIamRole(s3ObjectPrefix);
        s3DestinationConfiguration.setRoleARN(iamRoleArn);

        createDeliveryStreamRequest.setS3DestinationConfiguration(s3DestinationConfiguration);

        firehoseClient.createDeliveryStream(createDeliveryStreamRequest);

        // The Delivery Stream is now being created.
        LOG.info("Creating DeliveryStream : " + deliveryStreamName);
        waitForDeliveryStreamToBecomeAvailable(deliveryStreamName);
    }
}

From source file:AmazonKinesisFirehoseToS3Sample.java

License:Open Source License

/**
 * Method to update s3 destination with updated s3Prefix, and buffering hints values.
 *
 * @throws Exception/*w  ww .j  a v  a2  s .c o m*/
 */
private static void updateDeliveryStream() throws Exception {
    DeliveryStreamDescription deliveryStreamDescription = describeDeliveryStream(deliveryStreamName);

    LOG.info("Updating DeliveryStream Destination: " + deliveryStreamName + " with new configuration options");
    // get(0) -> DeliveryStream currently supports only one destination per DeliveryStream
    UpdateDestinationRequest updateDestinationRequest = new UpdateDestinationRequest()
            .withDeliveryStreamName(deliveryStreamName)
            .withCurrentDeliveryStreamVersionId(deliveryStreamDescription.getVersionId())
            .withDestinationId(deliveryStreamDescription.getDestinations().get(0).getDestinationId());

    S3DestinationUpdate s3DestinationUpdate = new S3DestinationUpdate();
    s3DestinationUpdate.withPrefix(updateS3ObjectPrefix);

    BufferingHints bufferingHints = null;
    if (updateSizeInMBs != null || updateIntervalInSeconds != null) {
        bufferingHints = new BufferingHints();
        bufferingHints.setSizeInMBs(updateSizeInMBs);
        bufferingHints.setIntervalInSeconds(updateIntervalInSeconds);
    }
    s3DestinationUpdate.setBufferingHints(bufferingHints);

    // Update the role policy with new s3Prefix configuration
    putRolePolicy(updateS3ObjectPrefix);

    updateDestinationRequest.setS3DestinationUpdate(s3DestinationUpdate);

    // Update deliveryStream destination with new configuration options such as s3Prefix and Buffering Hints.
    // Can also update Compression format, KMS key values and IAM Role.
    firehoseClient.updateDestination(updateDestinationRequest);
}