Example usage for com.amazonaws.services.kinesis.model Shard getAdjacentParentShardId

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model Shard getAdjacentParentShardId.

Prototype

public String getAdjacentParentShardId() 

Source Link

Document

The shard ID of the shard adjacent to the shard's parent.

Usage

From source file:com.trulia.stail.Stail.java

License:Apache License

public static void main(String[] args) {
    final Stail stail = new Stail();

    JCommander jct = new JCommander(stail);
    jct.setProgramName("stail");
    try {/*from w w  w  .ja  v a2  s  . c  om*/
        jct.parse(args);

        AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
        if (stail.profile != null) {
            credentialsProvider = new ProfileCredentialsProvider(stail.profile);
        }

        if (stail.role != null) {
            credentialsProvider = new STSAssumeRoleSessionCredentialsProvider.Builder(stail.role, "stail")
                    .withStsClient(AWSSecurityTokenServiceClientBuilder.standard()
                            .withCredentials(credentialsProvider).build())
                    .build();
        }

        AmazonKinesis client = AmazonKinesisClientBuilder.standard().withRegion(stail.region)
                .withCredentials(credentialsProvider).build();

        // prepare the initial shard iterators at the LATEST position
        Map<Shard, String> shardIterators = getShardIterators(client, stail.stream, stail.start);

        IRecordProcessor processor = stail.json ? new JSONRecordProcessor() : new RawRecordProcessor();

        Map<Shard, RateLimiter> rateLimiters = new HashMap<>();
        shardIterators.keySet()
                .forEach(shard -> rateLimiters.put(shard, RateLimiter.create(MAX_SHARD_THROUGHPUT)));

        long end = Strings.isNullOrEmpty(stail.duration) ? Long.MAX_VALUE
                : System.currentTimeMillis() + Duration.parse(stail.duration).toMillis();

        Set<String> reshardedShards = new HashSet<>();

        Map<Shard, String> sequenceNumbers = new HashMap<>();

        while (System.currentTimeMillis() < end) {
            if (!reshardedShards.isEmpty()) {
                // get the new list of shards
                List<Shard> shards = getShards(client, stail.stream);
                for (Shard shard : shards) {
                    if (!Strings.isNullOrEmpty(shard.getParentShardId())
                            && reshardedShards.contains(shard.getParentShardId())) {
                        // the old shard was split, so we need to consume this new shard from the beginning
                        shardIterators.put(shard, getOldestShardIterator(client, stail.stream, shard));
                    } else if (!Strings.isNullOrEmpty(shard.getAdjacentParentShardId())
                            && reshardedShards.contains(shard.getAdjacentParentShardId())) {
                        // the old shards were merged into a new shard
                        shardIterators.put(shard, getOldestShardIterator(client, stail.stream, shard));
                    }
                }

                reshardedShards.clear();
            }

            for (Shard shard : Lists.newArrayList(shardIterators.keySet())) {
                String shardIterator = shardIterators.remove(shard);

                GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
                getRecordsRequest.setShardIterator(shardIterator);
                getRecordsRequest.setLimit(BATCH_SIZE);

                try {
                    GetRecordsResult getRecordsResult = client.getRecords(getRecordsRequest);
                    List<Record> records = getRecordsResult.getRecords();
                    processor.processRecords(records, null);

                    shardIterator = getRecordsResult.getNextShardIterator();

                    if (records.size() <= 0) {
                        // nothing on the stream yet, so lets wait a bit to see if something appears
                        TimeUnit.SECONDS.sleep(1);
                    } else {
                        int bytesRead = records.stream().map(record -> record.getData().position())
                                .reduce((_1, _2) -> _1 + _2).get();

                        sequenceNumbers.put(shard, records.get(records.size() - 1).getSequenceNumber());

                        // optionally sleep if we have hit the limit for this shard
                        rateLimiters.get(shard).acquire(bytesRead);
                    }

                    if (!Strings.isNullOrEmpty(shardIterator)) {
                        shardIterators.put(shard, shardIterator);
                    } else {
                        reshardedShards.add(shard.getShardId());
                    }
                } catch (ProvisionedThroughputExceededException e) {
                    logger.warn("tripped the max throughput.  Backing off: {}", e.getMessage());
                    TimeUnit.SECONDS.sleep(6); // we tripped the max throughput.  Back off

                    // add the original iterator back into the map so we can try it again
                    shardIterators.put(shard, shardIterator);
                } catch (ExpiredIteratorException e) {
                    logger.debug("Iterator expired", e);

                    String sequenceNumber = sequenceNumbers.get(shard);
                    if (sequenceNumber == null) {
                        logger.warn("No previously known sequence number for {}.  Moving to LATEST",
                                shard.getShardId());
                        shardIterators.put(shard, getShardIterator(client, stail.stream, shard, null));
                    } else {
                        shardIterators.put(shard,
                                getShardIteratorAtSequenceNumber(client, stail.stream, shard, sequenceNumber));
                    }
                }
            }
        }
    } catch (ParameterException e) {
        jct.usage();
        System.exit(1);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        System.exit(2);
    }
}

---

From source file:org.apache.beam.sdk.io.kinesis.StartingPointShardsFinder.java

License:Apache License

private Set<Shard> findNextShards(List<Shard> allShards, Set<Shard> expiredShards) {
    Set<Shard> nextShards = new HashSet<>();
    for (Shard expiredShard : expiredShards) {
        boolean successorFound = false;
        for (Shard shard : allShards) {
            if (Objects.equals(expiredShard.getShardId(), shard.getParentShardId())) {
                nextShards.add(shard);/*from w w  w  .jav  a2 s  . c  o  m*/
                successorFound = true;
            } else if (Objects.equals(expiredShard.getShardId(), shard.getAdjacentParentShardId())) {
                successorFound = true;
            }
        }
        if (!successorFound) {
            // This can potentially happen during split/merge operation. Newly created shards might be
            // not listed in the allShards list and their predecessor is already considered expired.
            // Retrying should solve the issue.
            throw new IllegalStateException("No successors were found for shard: " + expiredShard);
        }
    }
    return nextShards;
}

---