Example usage for com.amazonaws.services.kinesis AmazonKinesis getRecords

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis AmazonKinesis getRecords.

Prototype

GetRecordsResult getRecords(GetRecordsRequest getRecordsRequest);

Source Link

Document

Gets data records from a Kinesis data stream's shard.

Usage

From source file:com.trulia.stail.Stail.java

License:Apache License

public static void main(String[] args) {
    final Stail stail = new Stail();

    JCommander jct = new JCommander(stail);
    jct.setProgramName("stail");
    try {/*w ww.java  2  s .  c  o m*/
        jct.parse(args);

        AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
        if (stail.profile != null) {
            credentialsProvider = new ProfileCredentialsProvider(stail.profile);
        }

        if (stail.role != null) {
            credentialsProvider = new STSAssumeRoleSessionCredentialsProvider.Builder(stail.role, "stail")
                    .withStsClient(AWSSecurityTokenServiceClientBuilder.standard()
                            .withCredentials(credentialsProvider).build())
                    .build();
        }

        AmazonKinesis client = AmazonKinesisClientBuilder.standard().withRegion(stail.region)
                .withCredentials(credentialsProvider).build();

        // prepare the initial shard iterators at the LATEST position
        Map<Shard, String> shardIterators = getShardIterators(client, stail.stream, stail.start);

        IRecordProcessor processor = stail.json ? new JSONRecordProcessor() : new RawRecordProcessor();

        Map<Shard, RateLimiter> rateLimiters = new HashMap<>();
        shardIterators.keySet()
                .forEach(shard -> rateLimiters.put(shard, RateLimiter.create(MAX_SHARD_THROUGHPUT)));

        long end = Strings.isNullOrEmpty(stail.duration) ? Long.MAX_VALUE
                : System.currentTimeMillis() + Duration.parse(stail.duration).toMillis();

        Set<String> reshardedShards = new HashSet<>();

        Map<Shard, String> sequenceNumbers = new HashMap<>();

        while (System.currentTimeMillis() < end) {
            if (!reshardedShards.isEmpty()) {
                // get the new list of shards
                List<Shard> shards = getShards(client, stail.stream);
                for (Shard shard : shards) {
                    if (!Strings.isNullOrEmpty(shard.getParentShardId())
                            && reshardedShards.contains(shard.getParentShardId())) {
                        // the old shard was split, so we need to consume this new shard from the beginning
                        shardIterators.put(shard, getOldestShardIterator(client, stail.stream, shard));
                    } else if (!Strings.isNullOrEmpty(shard.getAdjacentParentShardId())
                            && reshardedShards.contains(shard.getAdjacentParentShardId())) {
                        // the old shards were merged into a new shard
                        shardIterators.put(shard, getOldestShardIterator(client, stail.stream, shard));
                    }
                }

                reshardedShards.clear();
            }

            for (Shard shard : Lists.newArrayList(shardIterators.keySet())) {
                String shardIterator = shardIterators.remove(shard);

                GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
                getRecordsRequest.setShardIterator(shardIterator);
                getRecordsRequest.setLimit(BATCH_SIZE);

                try {
                    GetRecordsResult getRecordsResult = client.getRecords(getRecordsRequest);
                    List<Record> records = getRecordsResult.getRecords();
                    processor.processRecords(records, null);

                    shardIterator = getRecordsResult.getNextShardIterator();

                    if (records.size() <= 0) {
                        // nothing on the stream yet, so lets wait a bit to see if something appears
                        TimeUnit.SECONDS.sleep(1);
                    } else {
                        int bytesRead = records.stream().map(record -> record.getData().position())
                                .reduce((_1, _2) -> _1 + _2).get();

                        sequenceNumbers.put(shard, records.get(records.size() - 1).getSequenceNumber());

                        // optionally sleep if we have hit the limit for this shard
                        rateLimiters.get(shard).acquire(bytesRead);
                    }

                    if (!Strings.isNullOrEmpty(shardIterator)) {
                        shardIterators.put(shard, shardIterator);
                    } else {
                        reshardedShards.add(shard.getShardId());
                    }
                } catch (ProvisionedThroughputExceededException e) {
                    logger.warn("tripped the max throughput.  Backing off: {}", e.getMessage());
                    TimeUnit.SECONDS.sleep(6); // we tripped the max throughput.  Back off

                    // add the original iterator back into the map so we can try it again
                    shardIterators.put(shard, shardIterator);
                } catch (ExpiredIteratorException e) {
                    logger.debug("Iterator expired", e);

                    String sequenceNumber = sequenceNumbers.get(shard);
                    if (sequenceNumber == null) {
                        logger.warn("No previously known sequence number for {}.  Moving to LATEST",
                                shard.getShardId());
                        shardIterators.put(shard, getShardIterator(client, stail.stream, shard, null));
                    } else {
                        shardIterators.put(shard,
                                getShardIteratorAtSequenceNumber(client, stail.stream, shard, sequenceNumber));
                    }
                }
            }
        }
    } catch (ParameterException e) {
        jct.usage();
        System.exit(1);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        System.exit(2);
    }
}

---