Example usage for com.amazonaws.services.kinesis.model Shard getParentShardId

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model Shard getParentShardId.

Prototype

public String getParentShardId() 

Source Link

Document

The shard ID of the shard's parent.

Usage

From source file:com.trulia.stail.Stail.java

License:Apache License

public static void main(String[] args) {
    final Stail stail = new Stail();

    JCommander jct = new JCommander(stail);
    jct.setProgramName("stail");
    try {//from  ww  w  . j a v  a 2 s. c o m
        jct.parse(args);

        AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
        if (stail.profile != null) {
            credentialsProvider = new ProfileCredentialsProvider(stail.profile);
        }

        if (stail.role != null) {
            credentialsProvider = new STSAssumeRoleSessionCredentialsProvider.Builder(stail.role, "stail")
                    .withStsClient(AWSSecurityTokenServiceClientBuilder.standard()
                            .withCredentials(credentialsProvider).build())
                    .build();
        }

        AmazonKinesis client = AmazonKinesisClientBuilder.standard().withRegion(stail.region)
                .withCredentials(credentialsProvider).build();

        // prepare the initial shard iterators at the LATEST position
        Map<Shard, String> shardIterators = getShardIterators(client, stail.stream, stail.start);

        IRecordProcessor processor = stail.json ? new JSONRecordProcessor() : new RawRecordProcessor();

        Map<Shard, RateLimiter> rateLimiters = new HashMap<>();
        shardIterators.keySet()
                .forEach(shard -> rateLimiters.put(shard, RateLimiter.create(MAX_SHARD_THROUGHPUT)));

        long end = Strings.isNullOrEmpty(stail.duration) ? Long.MAX_VALUE
                : System.currentTimeMillis() + Duration.parse(stail.duration).toMillis();

        Set<String> reshardedShards = new HashSet<>();

        Map<Shard, String> sequenceNumbers = new HashMap<>();

        while (System.currentTimeMillis() < end) {
            if (!reshardedShards.isEmpty()) {
                // get the new list of shards
                List<Shard> shards = getShards(client, stail.stream);
                for (Shard shard : shards) {
                    if (!Strings.isNullOrEmpty(shard.getParentShardId())
                            && reshardedShards.contains(shard.getParentShardId())) {
                        // the old shard was split, so we need to consume this new shard from the beginning
                        shardIterators.put(shard, getOldestShardIterator(client, stail.stream, shard));
                    } else if (!Strings.isNullOrEmpty(shard.getAdjacentParentShardId())
                            && reshardedShards.contains(shard.getAdjacentParentShardId())) {
                        // the old shards were merged into a new shard
                        shardIterators.put(shard, getOldestShardIterator(client, stail.stream, shard));
                    }
                }

                reshardedShards.clear();
            }

            for (Shard shard : Lists.newArrayList(shardIterators.keySet())) {
                String shardIterator = shardIterators.remove(shard);

                GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
                getRecordsRequest.setShardIterator(shardIterator);
                getRecordsRequest.setLimit(BATCH_SIZE);

                try {
                    GetRecordsResult getRecordsResult = client.getRecords(getRecordsRequest);
                    List<Record> records = getRecordsResult.getRecords();
                    processor.processRecords(records, null);

                    shardIterator = getRecordsResult.getNextShardIterator();

                    if (records.size() <= 0) {
                        // nothing on the stream yet, so lets wait a bit to see if something appears
                        TimeUnit.SECONDS.sleep(1);
                    } else {
                        int bytesRead = records.stream().map(record -> record.getData().position())
                                .reduce((_1, _2) -> _1 + _2).get();

                        sequenceNumbers.put(shard, records.get(records.size() - 1).getSequenceNumber());

                        // optionally sleep if we have hit the limit for this shard
                        rateLimiters.get(shard).acquire(bytesRead);
                    }

                    if (!Strings.isNullOrEmpty(shardIterator)) {
                        shardIterators.put(shard, shardIterator);
                    } else {
                        reshardedShards.add(shard.getShardId());
                    }
                } catch (ProvisionedThroughputExceededException e) {
                    logger.warn("tripped the max throughput.  Backing off: {}", e.getMessage());
                    TimeUnit.SECONDS.sleep(6); // we tripped the max throughput.  Back off

                    // add the original iterator back into the map so we can try it again
                    shardIterators.put(shard, shardIterator);
                } catch (ExpiredIteratorException e) {
                    logger.debug("Iterator expired", e);

                    String sequenceNumber = sequenceNumbers.get(shard);
                    if (sequenceNumber == null) {
                        logger.warn("No previously known sequence number for {}.  Moving to LATEST",
                                shard.getShardId());
                        shardIterators.put(shard, getShardIterator(client, stail.stream, shard, null));
                    } else {
                        shardIterators.put(shard,
                                getShardIteratorAtSequenceNumber(client, stail.stream, shard, sequenceNumber));
                    }
                }
            }
        }
    } catch (ParameterException e) {
        jct.usage();
        System.exit(1);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        System.exit(2);
    }
}

---

From source file:org.apache.beam.sdk.io.kinesis.ShardRecordsIterator.java

License:Apache License

List<ShardRecordsIterator> findSuccessiveShardRecordIterators() throws TransientKinesisException {
    List<Shard> shards = kinesis.listShards(streamName);
    List<ShardRecordsIterator> successiveShardRecordIterators = new ArrayList<>();
    for (Shard shard : shards) {
        if (shardId.equals(shard.getParentShardId())) {
            ShardCheckpoint shardCheckpoint = new ShardCheckpoint(streamName, shard.getShardId(),
                    new StartingPoint(InitialPositionInStream.TRIM_HORIZON));
            successiveShardRecordIterators
                    .add(new ShardRecordsIterator(shardCheckpoint, kinesis, watermarkPolicyFactory));
        }/*w  w  w  .j  av a 2s. c o  m*/
    }
    return successiveShardRecordIterators;
}

---

From source file:org.apache.beam.sdk.io.kinesis.StartingPointShardsFinder.java

License:Apache License

private Set<Shard> findNextShards(List<Shard> allShards, Set<Shard> expiredShards) {
    Set<Shard> nextShards = new HashSet<>();
    for (Shard expiredShard : expiredShards) {
        boolean successorFound = false;
        for (Shard shard : allShards) {
            if (Objects.equals(expiredShard.getShardId(), shard.getParentShardId())) {
                nextShards.add(shard);//from  w  w w . j  a  v a  2 s .c  o m
                successorFound = true;
            } else if (Objects.equals(expiredShard.getShardId(), shard.getAdjacentParentShardId())) {
                successorFound = true;
            }
        }
        if (!successorFound) {
            // This can potentially happen during split/merge operation. Newly created shards might be
            // not listed in the allShards list and their predecessor is already considered expired.
            // Retrying should solve the issue.
            throw new IllegalStateException("No successors were found for shard: " + expiredShard);
        }
    }
    return nextShards;
}

---

From source file:org.apache.beam.sdk.io.kinesis.StartingPointShardsFinder.java

License:Apache License

/**
 * Finds the initial set of shards (the oldest ones). These shards do not have their parents in
 * the shard list.//from  w  w  w  .ja  v  a  2 s.c o m
 */
private Set<Shard> findInitialShardsWithoutParents(String streamName, List<Shard> allShards) {
    Set<String> shardIds = new HashSet<>();
    for (Shard shard : allShards) {
        shardIds.add(shard.getShardId());
    }
    LOGGER.info("Stream {} has following shards: {}", streamName, shardIds);
    Set<Shard> shardsWithoutParents = new HashSet<>();
    for (Shard shard : allShards) {
        if (!shardIds.contains(shard.getParentShardId())) {
            shardsWithoutParents.add(shard);
        }
    }
    return shardsWithoutParents;
}

---