List of usage examples for com.amazonaws.services.kinesisfirehose.model BufferingHints setIntervalInSeconds
public void setIntervalInSeconds(Integer intervalInSeconds)
Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination.
From source file:AmazonKinesisFirehoseToRedshiftSample.java
License:Open Source License
/** * Method to create delivery stream with Redshift destination configuration. * * @throws Exception// w w w . ja va 2 s . c om */ private static void createDeliveryStream() throws Exception { boolean deliveryStreamExists = false; LOG.info("Checking if " + deliveryStreamName + " already exits"); List<String> deliveryStreamNames = listDeliveryStreams(); if (deliveryStreamNames != null && deliveryStreamNames.contains(deliveryStreamName)) { deliveryStreamExists = true; LOG.info("DeliveryStream " + deliveryStreamName + " already exists. Not creating the new delivery stream"); } else { LOG.info("DeliveryStream " + deliveryStreamName + " does not exist"); } if (!deliveryStreamExists) { // Create DeliveryStream CreateDeliveryStreamRequest createDeliveryStreamRequest = new CreateDeliveryStreamRequest(); createDeliveryStreamRequest.setDeliveryStreamName(deliveryStreamName); S3DestinationConfiguration redshiftS3Configuration = new S3DestinationConfiguration(); redshiftS3Configuration.setBucketARN(s3BucketARN); redshiftS3Configuration.setPrefix(s3ObjectPrefix); BufferingHints bufferingHints = null; if (s3DestinationSizeInMBs != null || s3DestinationIntervalInSeconds != null) { bufferingHints = new BufferingHints(); bufferingHints.setSizeInMBs(s3DestinationSizeInMBs); bufferingHints.setIntervalInSeconds(s3DestinationIntervalInSeconds); } redshiftS3Configuration.setBufferingHints(bufferingHints); // Create and set IAM role so that firehose service has access to the S3Buckets to put data. // Please check the trustPolicyDocument.json and permissionsPolicyDocument.json files // for the trust and permissions policies set for the role. String iamRoleArn = createIamRole(s3ObjectPrefix); redshiftS3Configuration.setRoleARN(iamRoleArn); CopyCommand copyCommand = new CopyCommand(); copyCommand.withCopyOptions(copyOptions).withDataTableName(dataTableName); RedshiftDestinationConfiguration redshiftDestinationConfiguration = new RedshiftDestinationConfiguration(); redshiftDestinationConfiguration.withClusterJDBCURL(clusterJDBCUrl).withRoleARN(iamRoleArn) .withUsername(username).withPassword(password).withCopyCommand(copyCommand) .withS3Configuration(redshiftS3Configuration); createDeliveryStreamRequest.setRedshiftDestinationConfiguration(redshiftDestinationConfiguration); firehoseClient.createDeliveryStream(createDeliveryStreamRequest); // The Delivery Stream is now being created. LOG.info("Creating DeliveryStream : " + deliveryStreamName); waitForDeliveryStreamToBecomeAvailable(deliveryStreamName); } }
From source file:AmazonKinesisFirehoseToS3Sample.java
License:Open Source License
/** * Method to create delivery stream for S3 destination configuration. * * @throws Exception// w w w .j ava 2 s . c o m */ private static void createDeliveryStream() throws Exception { boolean deliveryStreamExists = false; LOG.info("Checking if " + deliveryStreamName + " already exits"); List<String> deliveryStreamNames = listDeliveryStreams(); if (deliveryStreamNames != null && deliveryStreamNames.contains(deliveryStreamName)) { deliveryStreamExists = true; LOG.info("DeliveryStream " + deliveryStreamName + " already exists. Not creating the new delivery stream"); } else { LOG.info("DeliveryStream " + deliveryStreamName + " does not exist"); } if (!deliveryStreamExists) { // Create deliveryStream CreateDeliveryStreamRequest createDeliveryStreamRequest = new CreateDeliveryStreamRequest(); createDeliveryStreamRequest.setDeliveryStreamName(deliveryStreamName); S3DestinationConfiguration s3DestinationConfiguration = new S3DestinationConfiguration(); s3DestinationConfiguration.setBucketARN(s3BucketARN); s3DestinationConfiguration.setPrefix(s3ObjectPrefix); // Could also specify GZIP or ZIP s3DestinationConfiguration.setCompressionFormat(CompressionFormat.UNCOMPRESSED); // Encryption configuration is optional EncryptionConfiguration encryptionConfiguration = new EncryptionConfiguration(); if (!StringUtils.isNullOrEmpty(s3DestinationAWSKMSKeyId)) { encryptionConfiguration.setKMSEncryptionConfig( new KMSEncryptionConfig().withAWSKMSKeyARN(s3DestinationAWSKMSKeyId)); } else { encryptionConfiguration.setNoEncryptionConfig(NoEncryptionConfig.NoEncryption); } s3DestinationConfiguration.setEncryptionConfiguration(encryptionConfiguration); BufferingHints bufferingHints = null; if (s3DestinationSizeInMBs != null || s3DestinationIntervalInSeconds != null) { bufferingHints = new BufferingHints(); bufferingHints.setSizeInMBs(s3DestinationSizeInMBs); bufferingHints.setIntervalInSeconds(s3DestinationIntervalInSeconds); } s3DestinationConfiguration.setBufferingHints(bufferingHints); // Create and set IAM role so that firehose service has access to the S3Buckets to put data // and KMS keys (if provided) to encrypt data. Please check the trustPolicyDocument.json and // permissionsPolicyDocument.json files for the trust and permissions policies set for the role. String iamRoleArn = createIamRole(s3ObjectPrefix); s3DestinationConfiguration.setRoleARN(iamRoleArn); createDeliveryStreamRequest.setS3DestinationConfiguration(s3DestinationConfiguration); firehoseClient.createDeliveryStream(createDeliveryStreamRequest); // The Delivery Stream is now being created. LOG.info("Creating DeliveryStream : " + deliveryStreamName); waitForDeliveryStreamToBecomeAvailable(deliveryStreamName); } }
From source file:AmazonKinesisFirehoseToS3Sample.java
License:Open Source License
/** * Method to update s3 destination with updated s3Prefix, and buffering hints values. * * @throws Exception/*from ww w . jav a 2 s .c o m*/ */ private static void updateDeliveryStream() throws Exception { DeliveryStreamDescription deliveryStreamDescription = describeDeliveryStream(deliveryStreamName); LOG.info("Updating DeliveryStream Destination: " + deliveryStreamName + " with new configuration options"); // get(0) -> DeliveryStream currently supports only one destination per DeliveryStream UpdateDestinationRequest updateDestinationRequest = new UpdateDestinationRequest() .withDeliveryStreamName(deliveryStreamName) .withCurrentDeliveryStreamVersionId(deliveryStreamDescription.getVersionId()) .withDestinationId(deliveryStreamDescription.getDestinations().get(0).getDestinationId()); S3DestinationUpdate s3DestinationUpdate = new S3DestinationUpdate(); s3DestinationUpdate.withPrefix(updateS3ObjectPrefix); BufferingHints bufferingHints = null; if (updateSizeInMBs != null || updateIntervalInSeconds != null) { bufferingHints = new BufferingHints(); bufferingHints.setSizeInMBs(updateSizeInMBs); bufferingHints.setIntervalInSeconds(updateIntervalInSeconds); } s3DestinationUpdate.setBufferingHints(bufferingHints); // Update the role policy with new s3Prefix configuration putRolePolicy(updateS3ObjectPrefix); updateDestinationRequest.setS3DestinationUpdate(s3DestinationUpdate); // Update deliveryStream destination with new configuration options such as s3Prefix and Buffering Hints. // Can also update Compression format, KMS key values and IAM Role. firehoseClient.updateDestination(updateDestinationRequest); }