Example usage for com.amazonaws.services.kinesis AmazonKinesisClient describeStream

List of usage examples for com.amazonaws.services.kinesis AmazonKinesisClient describeStream

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.kinesis.*
  5. AmazonKinesisClient
  6. describeStream

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis AmazonKinesisClient describeStream.

Prototype

@Override
    public DescribeStreamResult describeStream(String streamName)

Source Link

Usage

From source file:whgHelper.java

License:Open Source License

public static void setStream(AmazonKinesisClient kinesis, String streamName, int shardCount) {

    try {/* w w w . j  a v  a2 s .  c o m*/
        // Describe the stream and check if it exists
        DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(streamName);
        StreamDescription streamDescription = kinesis.describeStream(describeStreamRequest)
                .getStreamDescription();
        System.out.printf("Stream %s has a status of %s.\n", streamName, streamDescription.getStreamStatus());

        if ("DELETING".equals(streamDescription.getStreamStatus())) {
            System.out.println("Stream is being deleted. This sample will now exit.");
            System.exit(0);
        }

        // Wait for the stream to become active if it is not yet ACTIVE.
        if (!"ACTIVE".equals(streamDescription.getStreamStatus())) {
            try {
                waitForStreamToBecomeAvailable(kinesis, streamName);
            } catch (InterruptedException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    } catch (ResourceNotFoundException ex) {

        System.out.printf("Stream %s does not exist. Creating it now.\n", streamName);
        // Create a stream. The number of shards determines the provisioned throughput.
        CreateStreamRequest createStreamRequest = new CreateStreamRequest();
        createStreamRequest.setStreamName(streamName);
        createStreamRequest.setShardCount(shardCount);
        kinesis.createStream(createStreamRequest);
        // The stream is now being created. Wait for it to become active.
        try {
            waitForStreamToBecomeAvailable(kinesis, streamName);
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
}

From source file:whgHelper.java

License:Open Source License

private static void waitForStreamToBecomeAvailable(AmazonKinesisClient kinesis, String myStreamName)
        throws InterruptedException {

    System.out.printf("Waiting for %s to become ACTIVE...\n", myStreamName);

    long startTime = System.currentTimeMillis();
    long endTime = startTime + TimeUnit.MINUTES.toMillis(10);
    while (System.currentTimeMillis() < endTime) {
        Thread.sleep(TimeUnit.SECONDS.toMillis(20));

        try {//from   ww  w .j a v a2  s  . c  om
            DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
            describeStreamRequest.setStreamName(myStreamName);
            // ask for no more than 10 shards at a time -- this is an optional parameter
            describeStreamRequest.setLimit(10);
            DescribeStreamResult describeStreamResponse = kinesis.describeStream(describeStreamRequest);

            String streamStatus = describeStreamResponse.getStreamDescription().getStreamStatus();
            System.out.printf("\t- current state: %s\n", streamStatus);
            if ("ACTIVE".equals(streamStatus)) {
                return;
            }
        } catch (ResourceNotFoundException ex) {
            // ResourceNotFound means the stream doesn't exist yet,
            // so ignore this error and just keep polling.
        } catch (AmazonServiceException ase) {
            throw ase;
        }
    }

    throw new RuntimeException(String.format("Stream %s never became active", myStreamName));
}

From source file:com.amazon.services.awsrum.utils.KinesisUtils.java

License:Open Source License

/**
 * Helper method to determine if a Kinesis stream exists.
 * /*  w  ww  .  ja v a  2 s  .  c o m*/
 * @param kinesisClient
 *            The {@link AmazonKinesisClient} with Kinesis read privileges
 * @param streamName
 *            The Kinesis stream to check for
 * @return true if the Kinesis stream exists, otherwise return false
 */
private static boolean streamExists(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        kinesisClient.describeStream(describeStreamRequest);
        return true;
    } catch (ResourceNotFoundException e) {
        return false;
    }
}

From source file:com.amazon.services.awsrum.utils.KinesisUtils.java

License:Open Source License

/**
 * Return the state of a Kinesis stream.
 * //ww  w  .j  a  v  a2 s . co  m
 * @param kinesisClient
 *            The {@link AmazonKinesisClient} with Kinesis read privileges
 * @param streamName
 *            The Kinesis stream to get the state of
 * @return String representation of the Stream state
 */
private static String streamState(AmazonKinesisClient kinesisClient, String streamName) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        return kinesisClient.describeStream(describeStreamRequest).getStreamDescription().getStreamStatus();
    } catch (AmazonServiceException e) {
        return null;
    }
}

From source file:com.example.JavaKinesisWordCountASL.java

License:Apache License

public static void main(String[] args) {

    // Populate the appropriate variables from the given args
    String kinesisAppName = "testApp";
    String streamName = "test";
    String endpointUrl = "kinesis.us-east-1.amazonaws.com";

    // Create a Kinesis client in order to determine the number of shards for the given stream
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(
            CredentialsProvider.getAwsSessionCredentialsProvider());
    kinesisClient.setEndpoint(endpointUrl);
    int numShards = kinesisClient.describeStream(streamName).getStreamDescription().getShards().size();

    // In this com, we're going to create 1 Kinesis Receiver/input DStream for each shard.
    // This is not a necessity; if there are less receivers/DStreams than the number of shards,
    // then the shards will be automatically distributed among the receivers and each receiver
    // will receive data from multiple shards.
    int numStreams = numShards;

    // Spark Streaming batch interval
    Duration batchInterval = new Duration(2000);

    // Kinesis checkpoint interval.  Same as batchInterval for this com.
    Duration kinesisCheckpointInterval = batchInterval;

    // Get the region name from the endpoint URL to save Kinesis Client Library metadata in
    // DynamoDB of the same region as the Kinesis stream
    String regionName = RegionUtils.getRegionByEndpoint(endpointUrl).getName();

    // Setup the Spark config and StreamingContext
    SparkConf sparkConfig = new SparkConf().setAppName("JavaKinesisWordCountASL").setMaster("local[2]");
    JavaStreamingContext jssc = new JavaStreamingContext(sparkConfig, batchInterval);

    // Create the Kinesis DStreams
    List<JavaDStream<byte[]>> streamsList = new ArrayList<JavaDStream<byte[]>>(numStreams);
    for (int i = 0; i < numStreams; i++) {
        streamsList.add(KinesisUtils.createStream(jssc, kinesisAppName, streamName, endpointUrl, regionName,
                InitialPositionInStream.TRIM_HORIZON, kinesisCheckpointInterval,
                StorageLevel.MEMORY_AND_DISK_2()));
    }/*from  w w  w.j  av  a 2s .co  m*/

    // Union all the streams if there is more than 1 stream
    JavaDStream<byte[]> unionStreams;
    if (streamsList.size() > 1) {
        unionStreams = jssc.union(streamsList.get(0), streamsList.subList(1, streamsList.size()));
    } else {
        // Otherwise, just use the 1 stream
        unionStreams = streamsList.get(0);
    }

    // Convert each line of Array[Byte] to String, and split into words
    JavaDStream<String> words = unionStreams.flatMap(new FlatMapFunction<byte[], String>() {
        public Iterable<String> call(byte[] line) {
            return Lists.newArrayList(WORD_SEPARATOR.split(new String(line)));
        }
    });

    // Map each word to a (word, 1) tuple so we can reduce by key to count the words
    JavaPairDStream<String, Integer> wordCounts = words.mapToPair(new PairFunction<String, String, Integer>() {
        public Tuple2<String, Integer> call(String s) {
            return new Tuple2<String, Integer>(s, 1);
        }
    }).reduceByKey(new Function2<Integer, Integer, Integer>() {
        public Integer call(Integer i1, Integer i2) {
            return i1 + i2;
        }
    });

    // Print the first 10 wordCounts
    wordCounts.print();

    // Start the streaming context and await termination
    jssc.start();
    jssc.awaitTermination();
}

From source file:com.kinesisboard.amazonaws.utils.KinesisUtils.java

License:Open Source License

/**
 * Helper method to determine if an Amazon Kinesis stream exists.
 * /*  ww w. jav  a2s  .  c o  m*/
 * @param kinesisClient
 *        The {@link AmazonKinesisClient} with Amazon Kinesis read privileges
 * @param streamName
 *        The Amazon Kinesis stream to check for
 * @return true if the Amazon Kinesis stream exists, otherwise return false
 */
private static boolean streamExists(AmazonKinesisClient kinesisClient, String streamName) {
    System.out.println("KinesisUtils : Stream Exists .. ");
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    try {
        kinesisClient.describeStream(describeStreamRequest);
        return true;
    } catch (ResourceNotFoundException e) {
        return false;
    }
}

From source file:com.mh2c.LogProcessor.java

License:Apache License

/**
 * Processes a Kinesis stream.//from   ww w. java2s  . co m
 *
 * @param streamName Kinesis stream name
 * @param region AWS region housing Kinesis stream
 * @param batchInterval streaming batch interval, in milliseconds
 * @param hadoopDir directory prefix in Hadoop where files are written
 * @throws InterruptedException if processing is interrupted
 */
public void process(String streamName, String region, int batchInterval, String hadoopDir)
        throws InterruptedException {

    String kinesisEndpoint = String.format("https://kinesis.%s.amazonaws.com/", region);

    AmazonKinesisClient client = new AmazonKinesisClient();
    client.setEndpoint(kinesisEndpoint);

    int numShards = client.describeStream(streamName).getStreamDescription().getShards().size();
    SparkConf conf = new SparkConf().setAppName(APP_NAME);
    JavaStreamingContext ctx = new JavaStreamingContext(conf, new Duration(batchInterval));

    JavaDStream<byte[]> kinesisStream = KinesisUtils.createStream(ctx, APP_NAME, streamName, kinesisEndpoint,
            region, InitialPositionInStream.LATEST, new Duration(batchInterval),
            StorageLevel.MEMORY_AND_DISK_2());

    // Make more DStreams
    JavaDStream<ApacheLogRecord> processedRecords = kinesisStream
            .map(line -> new ApacheLogRecord(new String(line, StandardCharsets.UTF_8)))
            .map(record -> record.withIpAddress(anonymizeIpAddress(record.getIpAddress())))
            .map(record -> record.withUserAgent(categorizeUserAgent(record.getUserAgent())));

    // Only pair streams can be written as Hadoop files
    JavaPairDStream<String, ApacheLogRecord> markedRecords = processedRecords.transformToPair(
            recordRdd -> recordRdd.mapToPair(record -> new Tuple2<>(UUID.randomUUID().toString(), record)));

    // Write out to Hadoop
    markedRecords.print();
    markedRecords.saveAsHadoopFiles(hadoopDir, "txt", Text.class, Text.class, TextOutputFormat.class);

    ctx.start();
    try {
        ctx.awaitTermination();
    } catch (InterruptedException e) {
        System.out.println("Streaming stopped");
        return;
    }
}

From source file:com.streamsets.pipeline.stage.destination.kinesis.KinesisTarget.java

License:Apache License

private void checkStreamExists(List<ConfigIssue> issues) {
    ClientConfiguration kinesisConfiguration = new ClientConfiguration();
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(kinesisConfiguration);
    kinesisClient.setRegion(Region.getRegion(region));

    try {/*from   w w  w.  ja  va 2s .co  m*/
        DescribeStreamResult result = kinesisClient.describeStream(streamName);
        LOG.info("Connected successfully to stream: {} with description: {}", streamName,
                result.getStreamDescription().toString());
    } catch (Exception e) {
        issues.add(getContext().createConfigIssue(
                com.streamsets.pipeline.stage.origin.kinesis.Groups.KINESIS.name(), "streamName",
                Errors.KINESIS_01, e.toString()));
    } finally {
        kinesisClient.shutdown();
    }
}

From source file:com.streamsets.pipeline.stage.lib.kinesis.KinesisUtil.java

License:Apache License

public static long getShardCount(Regions region, AWSConfig awsConfig, String streamName)
        throws AmazonClientException {
    ClientConfiguration kinesisConfiguration = new ClientConfiguration();
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(AWSUtil.getCredentialsProvider(awsConfig),
            kinesisConfiguration);//from  ww  w.  j  ava2 s  .  co  m
    kinesisClient.setRegion(Region.getRegion(region));

    try {
        long numShards = 0;
        String lastShardId = null;
        StreamDescription description;
        do {
            if (lastShardId == null) {
                description = kinesisClient.describeStream(streamName).getStreamDescription();
            } else {
                description = kinesisClient.describeStream(streamName, lastShardId).getStreamDescription();
            }

            for (Shard shard : description.getShards()) {
                if (shard.getSequenceNumberRange().getEndingSequenceNumber() == null) {
                    // Then this shard is open, so we should count it. Shards with an ending sequence number
                    // are closed and cannot be written to, so we skip counting them.
                    ++numShards;
                }
            }

            int pageSize = description.getShards().size();
            lastShardId = description.getShards().get(pageSize - 1).getShardId();

        } while (description.getHasMoreShards());

        LOG.debug("Connected successfully to stream: '{}' with '{}' shards.", streamName, numShards);

        return numShards;
    } finally {
        kinesisClient.shutdown();
    }
}

From source file:com.streamsets.pipeline.stage.origin.kinesis.KinesisSource.java

License:Apache License

private void checkStreamExists(List<ConfigIssue> issues) {
    ClientConfiguration kinesisConfiguration = new ClientConfiguration();
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(kinesisConfiguration);
    kinesisClient.setRegion(Region.getRegion(region));

    try {/*  w w w  . j a  va 2 s  . c  o  m*/
        DescribeStreamResult result = kinesisClient.describeStream(streamName);
        LOG.info("Connected successfully to stream: {} with description: {}", streamName,
                result.getStreamDescription().toString());
    } catch (Exception e) {
        issues.add(getContext().createConfigIssue(Groups.KINESIS.name(), "streamName", Errors.KINESIS_01,
                e.toString()));
    } finally {
        kinesisClient.shutdown();
    }
}