Example usage for com.amazonaws.services.kinesis.model GetShardIteratorResult getShardIterator

List of usage examples for com.amazonaws.services.kinesis.model GetShardIteratorResult getShardIterator

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.kinesis.model.*
  5. GetShardIteratorResult
  6. getShardIterator

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model GetShardIteratorResult getShardIterator.

Prototype


public String getShardIterator()

Source Link

Document

The position in the shard from which to start reading data records sequentially.

Usage

From source file:AmazonKinesisGet.java

License:Open Source License

public static void main(String[] args) throws Exception {
    init();/*w  w w.  j  a v a  2 s  . c o m*/

    final String myStreamName = "philsteststream";
    final Integer myStreamSize = 1;

    // list all of my streams
    ListStreamsRequest listStreamsRequest = new ListStreamsRequest();
    listStreamsRequest.setLimit(10);
    ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest);
    List<String> streamNames = listStreamsResult.getStreamNames();
    while (listStreamsResult.isHasMoreStreams()) {
        if (streamNames.size() > 0) {
            listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1));
        }

        listStreamsResult = kinesisClient.listStreams(listStreamsRequest);

        streamNames.addAll(listStreamsResult.getStreamNames());

    }
    LOG.info("Printing my list of streams : ");

    // print all of my streams.
    if (!streamNames.isEmpty()) {
        System.out.println("List of my streams: ");
    }
    for (int i = 0; i < streamNames.size(); i++) {
        System.out.println(streamNames.get(i));
    }

    //System.out.println(streamNames.get(0));
    String myownstream = streamNames.get(0);

    // Retrieve the Shards from a Stream
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(myownstream);
    DescribeStreamResult describeStreamResult;
    List<Shard> shards = new ArrayList<>();
    String lastShardId = null;

    do {
        describeStreamRequest.setExclusiveStartShardId(lastShardId);
        describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
        shards.addAll(describeStreamResult.getStreamDescription().getShards());
        if (shards.size() > 0) {
            lastShardId = shards.get(shards.size() - 1).getShardId();
        }
    } while (describeStreamResult.getStreamDescription().getHasMoreShards());

    // Get Data from the Shards in a Stream
    // Hard-coded to use only 1 shard
    String shardIterator;
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(myownstream);
    //get(0) shows hardcoded to 1 stream
    getShardIteratorRequest.setShardId(shards.get(0).getShardId());
    // using TRIM_HORIZON but could use alternatives
    getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON");

    GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
    shardIterator = getShardIteratorResult.getShardIterator();

    // Continuously read data records from shard.
    List<Record> records;

    while (true) {
        // Create new GetRecordsRequest with existing shardIterator.
        // Set maximum records to return to 1000.

        GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
        getRecordsRequest.setShardIterator(shardIterator);
        getRecordsRequest.setLimit(1000);

        GetRecordsResult result = kinesisClient.getRecords(getRecordsRequest);

        // Put result into record list. Result may be empty.
        records = result.getRecords();

        // Print records
        for (Record record : records) {
            ByteBuffer byteBuffer = record.getData();
            System.out.println(String.format("Seq No: %s - %s", record.getSequenceNumber(),
                    new String(byteBuffer.array())));
        }

        try {
            Thread.sleep(1000);
        } catch (InterruptedException exception) {
            throw new RuntimeException(exception);
        }

        shardIterator = result.getNextShardIterator();
    }

}

From source file:com.datatorrent.contrib.kinesis.KinesisTestConsumer.java

License:Open Source License

@Override
public void run() {
    DescribeStreamRequest describeRequest = new DescribeStreamRequest();
    describeRequest.setStreamName(streamName);

    DescribeStreamResult describeResponse = client.describeStream(describeRequest);
    final List<Shard> shards = describeResponse.getStreamDescription().getShards();
    logger.debug("Inside consumer::run receiveCount= {}", receiveCount);
    while (isAlive) {
        Shard shId = shards.get(0);//w w w  . j a  va  2 s .com
        GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
        iteratorRequest.setStreamName(streamName);
        iteratorRequest.setShardId(shId.getShardId());

        iteratorRequest.setShardIteratorType("TRIM_HORIZON");
        GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest);
        String iterator = iteratorResponse.getShardIterator();

        GetRecordsRequest getRequest = new GetRecordsRequest();
        getRequest.setLimit(1000);
        getRequest.setShardIterator(iterator);
        //call "get" operation and get everything in this shard range
        GetRecordsResult getResponse = client.getRecords(getRequest);
        //get reference to next iterator for this shard
        //retrieve records
        List<Record> records = getResponse.getRecords();
        if (records == null || records.isEmpty()) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        } else {
            String seqNo = "";
            for (Record rc : records) {
                if (latch != null) {
                    latch.countDown();
                }
                seqNo = rc.getSequenceNumber();
                if (getData(rc).equals(KinesisOperatorTestBase.END_TUPLE))
                    break;
                holdingBuffer.add(rc);
                receiveCount++;
                logger.debug("Consuming {}, receiveCount= {}", getData(rc), receiveCount);
            }
        }
    }
    logger.debug("DONE consuming");
}

From source file:com.datatorrent.contrib.kinesis.KinesisUtil.java

License:Open Source License

/**
 * Get the records from the particular shard
 * @param streamName Name of the stream from where the records to be accessed
 * @param recordsLimit Number of records to return from shard
 * @param shId Shard Id of the shard/*from   www  .  j ava  2s.co  m*/
 * @param iteratorType Shard iterator type
 * @param seqNo Record sequence number
 * @return the list of records from the given shard
 * @throws AmazonClientException
 */
public List<Record> getRecords(String streamName, Integer recordsLimit, Shard shId,
        ShardIteratorType iteratorType, String seqNo) throws AmazonClientException {
    assert client != null : "Illegal client";
    try {
        // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType to it
        GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
        iteratorRequest.setStreamName(streamName);
        iteratorRequest.setShardId(shId.getShardId());
        iteratorRequest.setShardIteratorType(iteratorType);

        // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest
        if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType))
            iteratorRequest.setStartingSequenceNumber(seqNo);

        // Get the Response from the getShardIterator service method & get the shardIterator from that response
        GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest);
        // getShardIterator() specifies the position in the shard
        String iterator = iteratorResponse.getShardIterator();

        // Create the GetRecordsRequest instance and set the recordsLimit and iterator
        GetRecordsRequest getRequest = new GetRecordsRequest();
        getRequest.setLimit(recordsLimit);
        getRequest.setShardIterator(iterator);

        // Get the Response from the getRecords service method and get the data records from that response.
        GetRecordsResult getResponse = client.getRecords(getRequest);
        return getResponse.getRecords();
    } catch (AmazonClientException e) {
        throw new RuntimeException(e);
    }
}

From source file:com.netflix.spectator.tdigest.KinesisTDigestReader.java

License:Apache License

private void init() {
    if (recRequest == null) {
        GetShardIteratorResult result = client.getShardIterator(iterRequest);
        recRequest = new GetRecordsRequest().withLimit(10).withShardIterator(result.getShardIterator());
    }//  w w  w .jav  a 2  s  . c o  m
}

From source file:com.srotya.flume.kinesis.source.KinesisSource.java

License:Apache License

protected void getShardIterator() {
    System.out.println("Listing shards:");
    shards.forEach(shard -> System.out.println("Shard id:" + shard.getShardId()));
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(streamName);
    getShardIteratorRequest.setShardId(shards.get(shardId).getShardId());
    getShardIteratorRequest.setShardIteratorType(shardIteratorType);

    GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest);
    shardIterator = getShardIteratorResult.getShardIterator();
}

From source file:com.trulia.stail.Stail.java

License:Apache License

private static String getShardIterator(AmazonKinesis client, String stream, Shard shard, String start) {
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(stream);
    getShardIteratorRequest.setShardId(shard.getShardId());

    if (!Strings.isNullOrEmpty(start)) {
        getShardIteratorRequest.setShardIteratorType(ShardIteratorType.AT_TIMESTAMP);
        getShardIteratorRequest/*from   ww w.ja  v a2 s  .  co m*/
                .setTimestamp(new Date(System.currentTimeMillis() - Duration.parse(start).toMillis()));
    } else {
        getShardIteratorRequest.setShardIteratorType(ShardIteratorType.LATEST);
    }

    GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest);
    return getShardIteratorResult.getShardIterator();
}

From source file:com.trulia.stail.Stail.java

License:Apache License

private static String getShardIteratorAtSequenceNumber(AmazonKinesis client, String stream, Shard shard,
        String sequenceNumber) {//from  w  w  w . ja v a 2s  . c  o  m
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(stream);
    getShardIteratorRequest.setShardId(shard.getShardId());

    getShardIteratorRequest.setShardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER);
    getShardIteratorRequest.setStartingSequenceNumber(sequenceNumber);

    GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest);
    return getShardIteratorResult.getShardIterator();
}

From source file:com.trulia.stail.Stail.java

License:Apache License

private static String getOldestShardIterator(AmazonKinesis client, String stream, Shard shard) {
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(stream);
    getShardIteratorRequest.setShardId(shard.getShardId());
    getShardIteratorRequest.setShardIteratorType(ShardIteratorType.TRIM_HORIZON);

    GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest);
    return getShardIteratorResult.getShardIterator();
}

From source file:org.apache.apex.malhar.contrib.kinesis.KinesisTestConsumer.java

License:Apache License

public String prepareIterator() {
    DescribeStreamRequest describeRequest = new DescribeStreamRequest();
    describeRequest.setStreamName(streamName);

    List<Shard> shards = null;
    for (int i = 0; i < MAX_TRY_TIMES; ++i) {
        try {/*from w w  w.j ava  2s  .  c o m*/
            DescribeStreamResult describeResponse = client.describeStream(describeRequest);
            shards = describeResponse.getStreamDescription().getShards();
            if (shards.isEmpty()) {
                logger.warn("shards is empty");
            } else {
                break;
            }
        } catch (Exception e) {
            logger.error("get Stream description exception: ", e);
            throw new RuntimeException(e);
        }
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
            //
        }
    }

    Shard shId = shards.get(0);
    GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
    iteratorRequest.setStreamName(streamName);
    iteratorRequest.setShardId(shId.getShardId());

    iteratorRequest.setShardIteratorType("TRIM_HORIZON");
    GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest);

    return iteratorResponse.getShardIterator();
}

From source file:org.apache.apex.malhar.contrib.kinesis.KinesisUtil.java

License:Apache License

/**
 * Get the records from the particular shard
 * @param streamName Name of the stream from where the records to be accessed
 * @param recordsLimit Number of records to return from shard
 * @param shId Shard Id of the shard//from ww  w . ja va 2s .com
 * @param iteratorType Shard iterator type
 * @param seqNo Record sequence number
 * @return the list of records from the given shard
 * @throws AmazonClientException
 */
public List<Record> getRecords(String streamName, Integer recordsLimit, String shId,
        ShardIteratorType iteratorType, String seqNo) throws AmazonClientException {
    assert client != null : "Illegal client";
    try {
        // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType to it
        GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
        iteratorRequest.setStreamName(streamName);
        iteratorRequest.setShardId(shId);
        iteratorRequest.setShardIteratorType(iteratorType);

        // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest
        if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType)
                || ShardIteratorType.AT_SEQUENCE_NUMBER.equals(iteratorType)) {
            iteratorRequest.setStartingSequenceNumber(seqNo);
        }
        // Get the Response from the getShardIterator service method & get the shardIterator from that response
        GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest);
        // getShardIterator() specifies the position in the shard
        String iterator = iteratorResponse.getShardIterator();

        // Create the GetRecordsRequest instance and set the recordsLimit and iterator
        GetRecordsRequest getRequest = new GetRecordsRequest();
        getRequest.setLimit(recordsLimit);
        getRequest.setShardIterator(iterator);

        // Get the Response from the getRecords service method and get the data records from that response.
        GetRecordsResult getResponse = client.getRecords(getRequest);
        return getResponse.getRecords();
    } catch (AmazonClientException e) {
        throw new RuntimeException(e);
    }
}