Example usage for com.amazonaws.services.kinesis.model GetRecordsResult getMillisBehindLatest

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model GetRecordsResult getMillisBehindLatest.

Prototype

public Long getMillisBehindLatest() 

Source Link

Document

The number of milliseconds the GetRecords response is from the tip of the stream, indicating how far behind current time the consumer is.

Usage

From source file:org.apache.beam.sdk.io.kinesis.SimplifiedKinesisClient.java

License:Apache License

/**
 * Gets records from Kinesis and deaggregates them if needed.
 *
 * @return list of deaggregated records//  w  w  w  .  j a v a2s.c  o m
 * @throws TransientKinesisException - in case of recoverable situation
 */
public GetKinesisRecordsResult getRecords(final String shardIterator, final String streamName,
        final String shardId, final Integer limit) throws TransientKinesisException {
    return wrapExceptions(() -> {
        GetRecordsResult response = kinesis
                .getRecords(new GetRecordsRequest().withShardIterator(shardIterator).withLimit(limit));
        return new GetKinesisRecordsResult(UserRecord.deaggregate(response.getRecords()),
                response.getNextShardIterator(), response.getMillisBehindLatest(), streamName, shardId);
    });
}

---

From source file:org.apache.streams.amazon.kinesis.KinesisPersistReaderTask.java

License:Apache License

@Override
public void run() {

    GetShardIteratorRequest shardIteratorRequest = new GetShardIteratorRequest().withStreamName(this.streamName)
            .withShardId(shardId).withShardIteratorType("TRIM_HORIZON");

    GetShardIteratorResult shardIteratorResult = reader.client.getShardIterator(shardIteratorRequest);

    shardIteratorId = shardIteratorResult.getShardIterator();

    Map<String, Object> metadata = new HashMap<>();
    metadata.put("streamName", streamName);
    metadata.put("shardId", shardId);

    while (true) {

        GetRecordsRequest recordsRequest = new GetRecordsRequest().withShardIterator(shardIteratorId);

        GetRecordsResult recordsResult = reader.client.getRecords(recordsRequest);

        LOGGER.info("{} records {} millis behind {}:{}:{} ", recordsResult.getRecords().size(),
                recordsResult.getMillisBehindLatest(), streamName, shardId, shardIteratorId);

        shardIteratorId = recordsResult.getNextShardIterator();

        List<Record> recordList = recordsResult.getRecords();

        for (Record record : recordList) {
            try {
                byte[] byteArray = record.getData().array();
                //byte[] decoded = Base64.decode(byteArray);
                String message = new String(byteArray, Charset.forName("UTF-8"));
                reader.persistQueue.add(new StreamsDatum(message, record.getPartitionKey(), new DateTime(),
                        new BigInteger(record.getSequenceNumber()), metadata));
            } catch (Exception ex) {
                LOGGER.warn("Exception processing record {}: {}", record, ex);
            }//from   w w w .  j  a v a2s . c om
        }
        try {
            Thread.sleep(reader.pollInterval);
        } catch (InterruptedException ex) {
            LOGGER.trace("InterruptedException", ex);
        }
    }

}

---