Example usage for com.amazonaws.services.kinesis AmazonKinesisClient setEndpoint

List of usage examples for com.amazonaws.services.kinesis AmazonKinesisClient setEndpoint

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.kinesis.*
  5. AmazonKinesisClient
  6. setEndpoint

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis AmazonKinesisClient setEndpoint.

Prototype

@Deprecated
public void setEndpoint(String endpoint) throws IllegalArgumentException

Source Link

Document

Overrides the default endpoint for this client.

Usage

From source file:com.amazon.services.awsrum.utils.KinesisUtils.java

License:Open Source License

/**
 * Creates the Kinesis stream specified by config.KINESIS_INPUT_STREAM
 * //from w  w  w.  j  a v  a 2 s.  co  m
 * @param config
 *            The configuration with the specified input stream name and
 *            {@link AWSCredentialsProvider}
 * @param shardCount
 *            The shard count to create the stream with
 */
public static void createInputStream(KinesisConnectorConfiguration config, int shardCount) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    createAndWaitForStreamToBecomeAvailable(kinesisClient, config.KINESIS_INPUT_STREAM, shardCount);
}

From source file:com.amazon.services.awsrum.utils.KinesisUtils.java

License:Open Source License

/**
 * Creates the Kinesis stream specified by config.KINESIS_OUTPUT_STREAM.
 * //from www .j  a v  a  2 s.c o  m
 * @param config
 *            The configuration with the specified output stream name and
 *            {@link AWSCredentialsProvider}
 * @param shardCount
 *            The shard count to create the stream with
 */
public static void createOutputStream(KinesisConnectorConfiguration config, int shardCount) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    createAndWaitForStreamToBecomeAvailable(kinesisClient, config.KINESIS_OUTPUT_STREAM, shardCount);
}

From source file:com.amazon.services.awsrum.utils.KinesisUtils.java

License:Open Source License

/**
 * Deletes the input stream specified by config.KINESIS_INPUT_STREAM
 * // w w  w .j  a v a  2s  .com
 * @param config
 *            The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteInputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    deleteStream(kinesisClient, config.KINESIS_INPUT_STREAM);
}

From source file:com.amazon.services.awsrum.utils.KinesisUtils.java

License:Open Source License

/**
 * Deletes the output stream specified by config.KINESIS_OUTPUT_STREAM
 * //from   w  ww .ja va2s . c  o  m
 * @param config
 *            The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteOutputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    deleteStream(kinesisClient, config.KINESIS_OUTPUT_STREAM);
}

From source file:com.example.JavaKinesisWordCountASL.java

License:Apache License

public static void main(String[] args) {

    // Populate the appropriate variables from the given args
    String kinesisAppName = "testApp";
    String streamName = "test";
    String endpointUrl = "kinesis.us-east-1.amazonaws.com";

    // Create a Kinesis client in order to determine the number of shards for the given stream
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(
            CredentialsProvider.getAwsSessionCredentialsProvider());
    kinesisClient.setEndpoint(endpointUrl);
    int numShards = kinesisClient.describeStream(streamName).getStreamDescription().getShards().size();

    // In this com, we're going to create 1 Kinesis Receiver/input DStream for each shard.
    // This is not a necessity; if there are less receivers/DStreams than the number of shards,
    // then the shards will be automatically distributed among the receivers and each receiver
    // will receive data from multiple shards.
    int numStreams = numShards;

    // Spark Streaming batch interval
    Duration batchInterval = new Duration(2000);

    // Kinesis checkpoint interval.  Same as batchInterval for this com.
    Duration kinesisCheckpointInterval = batchInterval;

    // Get the region name from the endpoint URL to save Kinesis Client Library metadata in
    // DynamoDB of the same region as the Kinesis stream
    String regionName = RegionUtils.getRegionByEndpoint(endpointUrl).getName();

    // Setup the Spark config and StreamingContext
    SparkConf sparkConfig = new SparkConf().setAppName("JavaKinesisWordCountASL").setMaster("local[2]");
    JavaStreamingContext jssc = new JavaStreamingContext(sparkConfig, batchInterval);

    // Create the Kinesis DStreams
    List<JavaDStream<byte[]>> streamsList = new ArrayList<JavaDStream<byte[]>>(numStreams);
    for (int i = 0; i < numStreams; i++) {
        streamsList.add(KinesisUtils.createStream(jssc, kinesisAppName, streamName, endpointUrl, regionName,
                InitialPositionInStream.TRIM_HORIZON, kinesisCheckpointInterval,
                StorageLevel.MEMORY_AND_DISK_2()));
    }//from  w  w w . j av a 2  s . c  o  m

    // Union all the streams if there is more than 1 stream
    JavaDStream<byte[]> unionStreams;
    if (streamsList.size() > 1) {
        unionStreams = jssc.union(streamsList.get(0), streamsList.subList(1, streamsList.size()));
    } else {
        // Otherwise, just use the 1 stream
        unionStreams = streamsList.get(0);
    }

    // Convert each line of Array[Byte] to String, and split into words
    JavaDStream<String> words = unionStreams.flatMap(new FlatMapFunction<byte[], String>() {
        public Iterable<String> call(byte[] line) {
            return Lists.newArrayList(WORD_SEPARATOR.split(new String(line)));
        }
    });

    // Map each word to a (word, 1) tuple so we can reduce by key to count the words
    JavaPairDStream<String, Integer> wordCounts = words.mapToPair(new PairFunction<String, String, Integer>() {
        public Tuple2<String, Integer> call(String s) {
            return new Tuple2<String, Integer>(s, 1);
        }
    }).reduceByKey(new Function2<Integer, Integer, Integer>() {
        public Integer call(Integer i1, Integer i2) {
            return i1 + i2;
        }
    });

    // Print the first 10 wordCounts
    wordCounts.print();

    // Start the streaming context and await termination
    jssc.start();
    jssc.awaitTermination();
}

From source file:com.example.utils.KinesisUtils.java

License:Open Source License

/**
 * Creates the Amazon Kinesis stream specified by config.KINESIS_INPUT_STREAM
 * /*w w w  .j a  v  a  2s.c  om*/
 * @param config
 *        The configuration with the specified input stream name and {@link AWSCredentialsProvider}
 * @param shardCount
 *        The shard count to create the stream with
 */
public static void createInputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    createAndWaitForStreamToBecomeAvailable(kinesisClient, config.KINESIS_INPUT_STREAM,
            config.KINESIS_INPUT_STREAM_SHARD_COUNT);
}

From source file:com.example.utils.KinesisUtils.java

License:Open Source License

/**
 * Creates the Amazon Kinesis stream specified by config.KINESIS_OUTPUT_STREAM.
 * /*ww w  .ja  v  a2  s .  c o  m*/
 * @param config
 *        The configuration with the specified output stream name and {@link AWSCredentialsProvider}
 * @param shardCount
 *        The shard count to create the stream with
 */
public static void createOutputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    createAndWaitForStreamToBecomeAvailable(kinesisClient, config.KINESIS_OUTPUT_STREAM,
            config.KINESIS_OUTPUT_STREAM_SHARD_COUNT);
}

From source file:com.example.utils.KinesisUtils.java

License:Open Source License

/**
 * Deletes the input stream specified by config.KINESIS_INPUT_STREAM
 * //from  w  w  w . j a  v a2s.  co  m
 * @param config
 *        The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteInputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    deleteStream(kinesisClient, config.KINESIS_INPUT_STREAM);
}

From source file:com.example.utils.KinesisUtils.java

License:Open Source License

/**
 * Deletes the output stream specified by config.KINESIS_OUTPUT_STREAM
 * /*from w  ww . java 2 s  . c  o m*/
 * @param config
 *        The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteOutputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    deleteStream(kinesisClient, config.KINESIS_OUTPUT_STREAM);
}

From source file:com.mh2c.LogProcessor.java

License:Apache License

/**
 * Processes a Kinesis stream./*from   ww  w  .jav a  2s. co  m*/
 *
 * @param streamName Kinesis stream name
 * @param region AWS region housing Kinesis stream
 * @param batchInterval streaming batch interval, in milliseconds
 * @param hadoopDir directory prefix in Hadoop where files are written
 * @throws InterruptedException if processing is interrupted
 */
public void process(String streamName, String region, int batchInterval, String hadoopDir)
        throws InterruptedException {

    String kinesisEndpoint = String.format("https://kinesis.%s.amazonaws.com/", region);

    AmazonKinesisClient client = new AmazonKinesisClient();
    client.setEndpoint(kinesisEndpoint);

    int numShards = client.describeStream(streamName).getStreamDescription().getShards().size();
    SparkConf conf = new SparkConf().setAppName(APP_NAME);
    JavaStreamingContext ctx = new JavaStreamingContext(conf, new Duration(batchInterval));

    JavaDStream<byte[]> kinesisStream = KinesisUtils.createStream(ctx, APP_NAME, streamName, kinesisEndpoint,
            region, InitialPositionInStream.LATEST, new Duration(batchInterval),
            StorageLevel.MEMORY_AND_DISK_2());

    // Make more DStreams
    JavaDStream<ApacheLogRecord> processedRecords = kinesisStream
            .map(line -> new ApacheLogRecord(new String(line, StandardCharsets.UTF_8)))
            .map(record -> record.withIpAddress(anonymizeIpAddress(record.getIpAddress())))
            .map(record -> record.withUserAgent(categorizeUserAgent(record.getUserAgent())));

    // Only pair streams can be written as Hadoop files
    JavaPairDStream<String, ApacheLogRecord> markedRecords = processedRecords.transformToPair(
            recordRdd -> recordRdd.mapToPair(record -> new Tuple2<>(UUID.randomUUID().toString(), record)));

    // Write out to Hadoop
    markedRecords.print();
    markedRecords.saveAsHadoopFiles(hadoopDir, "txt", Text.class, Text.class, TextOutputFormat.class);

    ctx.start();
    try {
        ctx.awaitTermination();
    } catch (InterruptedException e) {
        System.out.println("Streaming stopped");
        return;
    }
}