Example usage for com.amazonaws.services.kinesisfirehose.model CopyCommand CopyCommand

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesisfirehose.model CopyCommand CopyCommand.

Prototype

CopyCommand

Source Link

Usage

From source file:AmazonKinesisFirehoseToRedshiftSample.java

License:Open Source License

/**
 * Method to create delivery stream with Redshift destination configuration.
 *
 * @throws Exception/*from  www .  ja  v a 2 s  .  com*/
 */
private static void createDeliveryStream() throws Exception {

    boolean deliveryStreamExists = false;

    LOG.info("Checking if " + deliveryStreamName + " already exits");
    List<String> deliveryStreamNames = listDeliveryStreams();
    if (deliveryStreamNames != null && deliveryStreamNames.contains(deliveryStreamName)) {
        deliveryStreamExists = true;
        LOG.info("DeliveryStream " + deliveryStreamName
                + " already exists. Not creating the new delivery stream");
    } else {
        LOG.info("DeliveryStream " + deliveryStreamName + " does not exist");
    }

    if (!deliveryStreamExists) {
        // Create DeliveryStream
        CreateDeliveryStreamRequest createDeliveryStreamRequest = new CreateDeliveryStreamRequest();
        createDeliveryStreamRequest.setDeliveryStreamName(deliveryStreamName);

        S3DestinationConfiguration redshiftS3Configuration = new S3DestinationConfiguration();
        redshiftS3Configuration.setBucketARN(s3BucketARN);
        redshiftS3Configuration.setPrefix(s3ObjectPrefix);

        BufferingHints bufferingHints = null;
        if (s3DestinationSizeInMBs != null || s3DestinationIntervalInSeconds != null) {
            bufferingHints = new BufferingHints();
            bufferingHints.setSizeInMBs(s3DestinationSizeInMBs);
            bufferingHints.setIntervalInSeconds(s3DestinationIntervalInSeconds);
        }
        redshiftS3Configuration.setBufferingHints(bufferingHints);

        // Create and set IAM role so that firehose service has access to the S3Buckets to put data. 
        // Please check the trustPolicyDocument.json and permissionsPolicyDocument.json files 
        // for the trust and permissions policies set for the role.
        String iamRoleArn = createIamRole(s3ObjectPrefix);
        redshiftS3Configuration.setRoleARN(iamRoleArn);

        CopyCommand copyCommand = new CopyCommand();
        copyCommand.withCopyOptions(copyOptions).withDataTableName(dataTableName);

        RedshiftDestinationConfiguration redshiftDestinationConfiguration = new RedshiftDestinationConfiguration();
        redshiftDestinationConfiguration.withClusterJDBCURL(clusterJDBCUrl).withRoleARN(iamRoleArn)
                .withUsername(username).withPassword(password).withCopyCommand(copyCommand)
                .withS3Configuration(redshiftS3Configuration);

        createDeliveryStreamRequest.setRedshiftDestinationConfiguration(redshiftDestinationConfiguration);

        firehoseClient.createDeliveryStream(createDeliveryStreamRequest);

        // The Delivery Stream is now being created.
        LOG.info("Creating DeliveryStream : " + deliveryStreamName);
        waitForDeliveryStreamToBecomeAvailable(deliveryStreamName);
    }
}

---

From source file:AmazonKinesisFirehoseToRedshiftSample.java

License:Open Source License

/**
 * Method to update redshift destination with updated copy options.
 *//*  w  w  w .  j  ava 2 s.  co m*/
private static void updateDeliveryStream() {
    DeliveryStreamDescription deliveryStreamDescription = describeDeliveryStream(deliveryStreamName);

    LOG.info("Updating DeliveryStream Destination: " + deliveryStreamName + " with new configuration options");
    // get(0) -> DeliveryStream currently supports only one destination per DeliveryStream
    UpdateDestinationRequest updateDestinationRequest = new UpdateDestinationRequest()
            .withDeliveryStreamName(deliveryStreamName)
            .withCurrentDeliveryStreamVersionId(deliveryStreamDescription.getVersionId())
            .withDestinationId(deliveryStreamDescription.getDestinations().get(0).getDestinationId());

    CopyCommand updatedCopyCommand = new CopyCommand().withDataTableName(dataTableName)
            .withCopyOptions(updatedCopyOptions);
    RedshiftDestinationUpdate redshiftDestinationUpdate = new RedshiftDestinationUpdate()
            .withCopyCommand(updatedCopyCommand);

    updateDestinationRequest.setRedshiftDestinationUpdate(redshiftDestinationUpdate);

    // Update DeliveryStream destination with new configuration options such as s3Prefix and Buffering Hints.
    // Can also update Compression format, KMS key values and IAM Role.
    firehoseClient.updateDestination(updateDestinationRequest);
}

---