Example usage for com.amazonaws.services.kinesis.model DescribeStreamResult getStreamDescription

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model DescribeStreamResult getStreamDescription.

Prototype

public StreamDescription getStreamDescription() 

Source Link

Document

The current status of the stream, the stream Amazon Resource Name (ARN), an array of shard objects that comprise the stream, and whether there are more shards available.

Usage

From source file:whgHelper.java

License:Open Source License

private static void waitForStreamToBecomeAvailable(AmazonKinesisClient kinesis, String myStreamName)
        throws InterruptedException {

    System.out.printf("Waiting for %s to become ACTIVE...\n", myStreamName);

    long startTime = System.currentTimeMillis();
    long endTime = startTime + TimeUnit.MINUTES.toMillis(10);
    while (System.currentTimeMillis() < endTime) {
        Thread.sleep(TimeUnit.SECONDS.toMillis(20));

        try {//from w ww . j av  a 2s  .  com
            DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
            describeStreamRequest.setStreamName(myStreamName);
            // ask for no more than 10 shards at a time -- this is an optional parameter
            describeStreamRequest.setLimit(10);
            DescribeStreamResult describeStreamResponse = kinesis.describeStream(describeStreamRequest);

            String streamStatus = describeStreamResponse.getStreamDescription().getStreamStatus();
            System.out.printf("\t- current state: %s\n", streamStatus);
            if ("ACTIVE".equals(streamStatus)) {
                return;
            }
        } catch (ResourceNotFoundException ex) {
            // ResourceNotFound means the stream doesn't exist yet,
            // so ignore this error and just keep polling.
        } catch (AmazonServiceException ase) {
            throw ase;
        }
    }

    throw new RuntimeException(String.format("Stream %s never became active", myStreamName));
}

---

From source file:AmazonKinesisSample.java

License:Open Source License

private static void waitForStreamToBecomeAvailable(String myStreamName) {

    System.out.println("Waiting for " + myStreamName + " to become ACTIVE...");

    long startTime = System.currentTimeMillis();
    long endTime = startTime + (10 * 60 * 1000);
    while (System.currentTimeMillis() < endTime) {
        try {//from w  w  w  .j a  v  a 2s.  c  om
            Thread.sleep(1000 * 20);
        } catch (InterruptedException e) {
            // Ignore interruption (doesn't impact stream creation)
        }
        try {
            DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
            describeStreamRequest.setStreamName(myStreamName);
            // ask for no more than 10 shards at a time -- this is an optional parameter
            describeStreamRequest.setLimit(10);
            DescribeStreamResult describeStreamResponse = kinesisClient.describeStream(describeStreamRequest);

            String streamStatus = describeStreamResponse.getStreamDescription().getStreamStatus();
            System.out.println("  - current state: " + streamStatus);
            if (streamStatus.equals("ACTIVE")) {
                return;
            }
        } catch (AmazonServiceException ase) {
            if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) {
                throw ase;
            }
            throw new RuntimeException("Stream " + myStreamName + " never went active");
        }
    }
}

---

From source file:kinesisAlertAnalysis.java

License:Open Source License

private static void waitForStreamToBecomeAvailable(String myStreamName) throws InterruptedException {
    System.out.printf("Waiting for %s to become ACTIVE...\n", myStreamName);

    long startTime = System.currentTimeMillis();
    long endTime = startTime + TimeUnit.MINUTES.toMillis(10);
    while (System.currentTimeMillis() < endTime) {
        Thread.sleep(TimeUnit.SECONDS.toMillis(20));

        try {//from ww  w  .j av a2s. co  m
            DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
            describeStreamRequest.setStreamName(myStreamName);
            // ask for no more than 10 shards at a time -- this is an optional parameter
            describeStreamRequest.setLimit(10);
            DescribeStreamResult describeStreamResponse = kinesis.describeStream(describeStreamRequest);

            String streamStatus = describeStreamResponse.getStreamDescription().getStreamStatus();
            System.out.printf("\t- current state: %s\n", streamStatus);
            if ("ACTIVE".equals(streamStatus)) {
                return;
            }
        } catch (ResourceNotFoundException ex) {
            // ResourceNotFound means the stream doesn't exist yet,
            // so ignore this error and just keep polling.
        } catch (AmazonServiceException ase) {
            throw ase;
        }
    }

    throw new RuntimeException(String.format("Stream %s never became active", myStreamName));
}

---

From source file:AmazonKinesisGet.java

License:Open Source License

public static void main(String[] args) throws Exception {
    init();//ww  w.  ja  v a  2 s  . c o m

    final String myStreamName = "philsteststream";
    final Integer myStreamSize = 1;

    // list all of my streams
    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    listStreamsRequest.setLimit(10);
    ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
    List<String> streamNames = listStreamsResult.getStreamNames();
    while (listStreamsResult.isHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }

        listStreamsResult = kinesisClient.listStreams(listStreamsRequest);

        streamNames.addAll(listStreamsResult.getStreamNames());

    }
    LOG.info("Printing my list of streams : ");

    // print all of my streams.
    if (!streamNames.isEmpty()) {
        System.out.println("List of my streams: ");
    }
    for (int i = 0; i < streamNames.size(); i++) {
        System.out.println(streamNames.get(i));
    }

    //System.out.println(streamNames.get(0));
    String myownstream = streamNames.get(0);

    // Retrieve the Shards from a Stream
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(myownstream);
    DescribeStreamResult describeStreamResult;
    List<Shard> shards = new ArrayList<>();
    String lastShardId = null;

    do {
        describeStreamRequest.setExclusiveStartShardId(lastShardId);
        describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
        shards.addAll(describeStreamResult.getStreamDescription().getShards());
        if (shards.size() > 0) {
            lastShardId = shards.get(shards.size() - 1).getShardId();
        }
    } while (describeStreamResult.getStreamDescription().getHasMoreShards());

    // Get Data from the Shards in a Stream
    // Hard-coded to use only 1 shard
    String shardIterator;
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(myownstream);
    //get(0) shows hardcoded to 1 stream
    getShardIteratorRequest.setShardId(shards.get(0).getShardId());
    // using TRIM_HORIZON but could use alternatives
    getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON");

    GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
    shardIterator = getShardIteratorResult.getShardIterator();

    // Continuously read data records from shard.
    List<Record> records;

    while (true) {
        // Create new GetRecordsRequest with existing shardIterator.
        // Set maximum records to return to 1000.

        GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
        getRecordsRequest.setShardIterator(shardIterator);
        getRecordsRequest.setLimit(1000);

        GetRecordsResult result = kinesisClient.getRecords(getRecordsRequest);

        // Put result into record list. Result may be empty.
        records = result.getRecords();

        // Print records
        for (Record record : records) {
            ByteBuffer byteBuffer = record.getData();
            System.out.println(String.format("Seq No: %s - %s", record.getSequenceNumber(),
                    new String(byteBuffer.array())));
        }

        try {
            Thread.sleep(1000);
        } catch (InterruptedException exception) {
            throw new RuntimeException(exception);
        }

        shardIterator = result.getNextShardIterator();
    }

}

---

From source file:StreamUtils.java

License:Open Source License

/**
 * Does the result of a describe stream request indicate the stream is ACTIVE?
 *
 * @param r The describe stream result to check for ACTIVE status.
 *//*from www  . j  a v  a 2  s  .c  o  m*/
private boolean isActive(DescribeStreamResult r) {
    return "ACTIVE".equals(r.getStreamDescription().getStreamStatus());
}

---

From source file:StreamUtils.java

License:Open Source License

/**
 * Split a shard by dividing the hash key space in half.
 *
 * @param streamName Name of the stream that contains the shard to split.
 * @param shardId The id of the shard to split.
 *
 * @throws IllegalArgumentException When either streamName or shardId are null or empty.
 * @throws LimitExceededException Shard limit for the account has been reached.
 * @throws ResourceNotFoundException The stream or shard cannot be found.
 * @throws InvalidArgumentException If the shard is closed and no eligible for splitting.
 * @throws AmazonClientException Error communicating with Amazon Kinesis.
 *
 *///from  ww  w  .  j a  va2  s.c  o  m
public void splitShardEvenly(String streamName, String shardId) throws LimitExceededException,
        ResourceNotFoundException, AmazonClientException, InvalidArgumentException, IllegalArgumentException {
    if (streamName == null || streamName.isEmpty()) {
        throw new IllegalArgumentException("stream name is required");
    }
    if (shardId == null || shardId.isEmpty()) {
        throw new IllegalArgumentException("shard id is required");
    }

    DescribeStreamResult result = kinesis.describeStream(streamName);
    StreamDescription description = result.getStreamDescription();

    // Find the shard we want to split
    Shard shardToSplit = null;
    for (Shard shard : description.getShards()) {
        if (shardId.equals(shard.getShardId())) {
            shardToSplit = shard;
            break;
        }
    }

    if (shardToSplit == null) {
        throw new ResourceNotFoundException(
                "Could not find shard with id '" + shardId + "' in stream '" + streamName + "'");
    }

    // Check if the shard is still open. Open shards do not have an ending sequence number.
    if (shardToSplit.getSequenceNumberRange().getEndingSequenceNumber() != null) {
        throw new InvalidArgumentException("Shard is CLOSED and is not eligible for splitting");
    }

    // Calculate the median hash key to use as the new starting hash key for the shard.
    BigInteger startingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getStartingHashKey());
    BigInteger endingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getEndingHashKey());
    BigInteger[] medianHashKey = startingHashKey.add(endingHashKey).divideAndRemainder(new BigInteger("2"));
    BigInteger newStartingHashKey = medianHashKey[0];
    if (!BigInteger.ZERO.equals(medianHashKey[1])) {
        // In order to more evenly distributed the new hash key ranges across the new shards we will "round up" to
        // the next integer when our current hash key range is not evenly divisible by 2.
        newStartingHashKey = newStartingHashKey.add(BigInteger.ONE);
    }

    // Submit the split shard request
    kinesis.splitShard(streamName, shardId, newStartingHashKey.toString());
}

---

From source file:com.boundary.aws.kinesis.Sample.java

License:Open Source License

private static void waitForStreamToBecomeAvailable(String myStreamName) {

    System.out.println("Waiting for " + myStreamName + " to become ACTIVE...");

    long startTime = System.currentTimeMillis();
    long endTime = startTime + (10 * 60 * 1000);
    while (System.currentTimeMillis() < endTime) {
        try {//from w  w  w .  j ava2  s .c  om
            Thread.sleep(1000 * 20);
        } catch (InterruptedException e) {
            // Ignore interruption (doesn't impact stream creation)
        }
        try {
            DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
            describeStreamRequest.setStreamName(myStreamName);
            // ask for no more than 10 shards at a time -- this is an
            // optional parameter
            describeStreamRequest.setLimit(10);
            DescribeStreamResult describeStreamResponse = kinesisClient.describeStream(describeStreamRequest);

            String streamStatus = describeStreamResponse.getStreamDescription().getStreamStatus();
            System.out.println("  - current state: " + streamStatus);
            if (streamStatus.equals("ACTIVE")) {
                return;
            }
        } catch (AmazonServiceException ase) {
            if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) {
                throw ase;
            }
            throw new RuntimeException("Stream " + myStreamName + " never went active");
        }
    }
}

---

From source file:com.datatorrent.contrib.kinesis.KinesisTestConsumer.java

License:Open Source License

@Override
public void run() {
    DescribeStreamRequest describeRequest = new DescribeStreamRequest();
    describeRequest.setStreamName(streamName);

    DescribeStreamResult describeResponse = client.describeStream(describeRequest);
    final List<Shard> shards = describeResponse.getStreamDescription().getShards();
    logger.debug("Inside consumer::run receiveCount= {}", receiveCount);
    while (isAlive) {
        Shard shId = shards.get(0);//from   w w  w . ja  v a  2  s . c  o  m
        GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
        iteratorRequest.setStreamName(streamName);
        iteratorRequest.setShardId(shId.getShardId());

        iteratorRequest.setShardIteratorType("TRIM_HORIZON");
        GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest);
        String iterator = iteratorResponse.getShardIterator();

        GetRecordsRequest getRequest = new GetRecordsRequest();
        getRequest.setLimit(1000);
        getRequest.setShardIterator(iterator);
        //call "get" operation and get everything in this shard range
        GetRecordsResult getResponse = client.getRecords(getRequest);
        //get reference to next iterator for this shard
        //retrieve records
        List<Record> records = getResponse.getRecords();
        if (records == null || records.isEmpty()) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        } else {
            String seqNo = "";
            for (Record rc : records) {
                if (latch != null) {
                    latch.countDown();
                }
                seqNo = rc.getSequenceNumber();
                if (getData(rc).equals(KinesisOperatorTestBase.END_TUPLE))
                    break;
                holdingBuffer.add(rc);
                receiveCount++;
                logger.debug("Consuming {}, receiveCount= {}", getData(rc), receiveCount);
            }
        }
    }
    logger.debug("DONE consuming");
}

---

From source file:com.datatorrent.contrib.kinesis.KinesisUtil.java

License:Open Source License

/**
 * Get the available shards from the kinesis
 * @param streamName Name of the stream from where the shards to be accessed
 * @return the list of shards from the given stream
 *///ww w.jav  a 2 s  . c  o m
public List<Shard> getShardList(String streamName) {
    assert client != null : "Illegal client";
    DescribeStreamRequest describeRequest = new DescribeStreamRequest();
    describeRequest.setStreamName(streamName);

    DescribeStreamResult describeResponse = client.describeStream(describeRequest);
    return describeResponse.getStreamDescription().getShards();
}

---

From source file:com.facebook.presto.kinesis.KinesisSplitManager.java

License:Apache License

@Override
public ConnectorPartitionResult getPartitions(ConnectorTableHandle tableHandle,
        TupleDomain<ColumnHandle> tupleDomain) {
    KinesisTableHandle kinesisTableHandle = handleResolver.convertTableHandle(tableHandle);

    DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest();
    describeStreamRequest.setStreamName(kinesisTableHandle.getStreamName());

    String exclusiveStartShardId = null;
    describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
    DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest);

    String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
    while ((streamStatus.equals("ACTIVE") == false) && (streamStatus.equals("UPDATING") == false)) {
        throw new ResourceNotFoundException("Stream not Active");
    }//from w ww. jav  a  2  s  .  c  o m

    List<Shard> shards = new ArrayList<>();
    ImmutableList.Builder<ConnectorPartition> builder = ImmutableList.builder();
    do {
        shards.addAll(describeStreamResult.getStreamDescription().getShards());

        for (Shard shard : shards) {
            builder.add(new KinesisShard(kinesisTableHandle.getStreamName(), shard));
        }

        if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) {
            exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
        } else {
            exclusiveStartShardId = null;
        }

    } while (exclusiveStartShardId != null);

    return new ConnectorPartitionResult(builder.build(), tupleDomain);
}

---

• «
• 1
• 2
• 3
• 4
• »