List of usage examples for com.amazonaws.services.kinesis.model Shard getShardId
public String getShardId()
The unique identifier of the shard within the stream.
From source file:StreamUtils.java
License:Open Source License
/** * Split a shard by dividing the hash key space in half. * * @param streamName Name of the stream that contains the shard to split. * @param shardId The id of the shard to split. * * @throws IllegalArgumentException When either streamName or shardId are null or empty. * @throws LimitExceededException Shard limit for the account has been reached. * @throws ResourceNotFoundException The stream or shard cannot be found. * @throws InvalidArgumentException If the shard is closed and no eligible for splitting. * @throws AmazonClientException Error communicating with Amazon Kinesis. * *//*from ww w . j a v a 2s.co m*/ public void splitShardEvenly(String streamName, String shardId) throws LimitExceededException, ResourceNotFoundException, AmazonClientException, InvalidArgumentException, IllegalArgumentException { if (streamName == null || streamName.isEmpty()) { throw new IllegalArgumentException("stream name is required"); } if (shardId == null || shardId.isEmpty()) { throw new IllegalArgumentException("shard id is required"); } DescribeStreamResult result = kinesis.describeStream(streamName); StreamDescription description = result.getStreamDescription(); // Find the shard we want to split Shard shardToSplit = null; for (Shard shard : description.getShards()) { if (shardId.equals(shard.getShardId())) { shardToSplit = shard; break; } } if (shardToSplit == null) { throw new ResourceNotFoundException( "Could not find shard with id '" + shardId + "' in stream '" + streamName + "'"); } // Check if the shard is still open. Open shards do not have an ending sequence number. if (shardToSplit.getSequenceNumberRange().getEndingSequenceNumber() != null) { throw new InvalidArgumentException("Shard is CLOSED and is not eligible for splitting"); } // Calculate the median hash key to use as the new starting hash key for the shard. BigInteger startingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getStartingHashKey()); BigInteger endingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getEndingHashKey()); BigInteger[] medianHashKey = startingHashKey.add(endingHashKey).divideAndRemainder(new BigInteger("2")); BigInteger newStartingHashKey = medianHashKey[0]; if (!BigInteger.ZERO.equals(medianHashKey[1])) { // In order to more evenly distributed the new hash key ranges across the new shards we will "round up" to // the next integer when our current hash key range is not evenly divisible by 2. newStartingHashKey = newStartingHashKey.add(BigInteger.ONE); } // Submit the split shard request kinesis.splitShard(streamName, shardId, newStartingHashKey.toString()); }
From source file:com.datatorrent.contrib.kinesis.AbstractKinesisInputOperator.java
License:Open Source License
@Override public Collection<Partition<AbstractKinesisInputOperator>> definePartitions( Collection<Partition<AbstractKinesisInputOperator>> partitions, PartitioningContext context) { boolean isInitialParitition = partitions.iterator().next().getStats() == null; // Set the credentials to get the list of shards if (isInitialParitition) { try {//from ww w . j a va 2 s .c o m KinesisUtil.getInstance().createKinesisClient(accessKey, secretKey, endPoint); } catch (Exception e) { throw new RuntimeException("[definePartitions]: Unable to load credentials. ", e); } } List<Shard> shards = KinesisUtil.getInstance().getShardList(getStreamName()); // Operator partitions List<Partition<AbstractKinesisInputOperator>> newPartitions = null; // initialize the shard positions Map<String, String> initShardPos = null; if (isInitialParitition && shardManager != null) { initShardPos = shardManager.loadInitialShardPositions(); } switch (strategy) { // For the 1 to 1 mapping The framework will create number of operator partitions based on kinesis shards // Each operator partition will consume from only one kinesis shard case ONE_TO_ONE: if (isInitialParitition) { lastRepartitionTime = System.currentTimeMillis(); logger.info("[ONE_TO_ONE]: Initializing partition(s)"); // initialize the number of operator partitions according to number of shards newPartitions = new ArrayList<Partition<AbstractKinesisInputOperator>>(shards.size()); for (int i = 0; i < shards.size(); i++) { logger.info("[ONE_TO_ONE]: Create operator partition for kinesis partition: " + shards.get(i).getShardId() + ", StreamName: " + this.getConsumer().streamName); newPartitions.add(createPartition(Sets.newHashSet(shards.get(i).getShardId()), initShardPos)); } } else if (newWaitingPartition.size() != 0) { // Remove the partitions for the closed shards removePartitionsForClosedShards(partitions); // add partition for new kinesis shard for (String pid : newWaitingPartition) { logger.info("[ONE_TO_ONE]: Add operator partition for kinesis partition " + pid); partitions.add(createPartition(Sets.newHashSet(pid), null)); } newWaitingPartition.clear(); return partitions; } break; // For the N to 1 mapping The initial partition number is defined by stream application // Afterwards, the framework will dynamically adjust the partition case MANY_TO_ONE: /* This case was handled into two ways. 1. Dynamic Partition: Number of DT partitions is depends on the number of open shards. 2. Static Partition: Number of DT partitions is fixed, whether the number of shards are increased/decreased. */ int size = initialPartitionCount; if (newWaitingPartition.size() != 0) { // Get the list of open shards shards = getOpenShards(partitions); if (shardsPerPartition > 1) size = (int) Math.ceil(shards.size() / (shardsPerPartition * 1.0)); initShardPos = shardManager.loadInitialShardPositions(); } Set<String>[] pIds = (Set<String>[]) Array.newInstance((new HashSet<String>()).getClass(), size); newPartitions = new ArrayList<Partition<AbstractKinesisInputOperator>>(size); for (int i = 0; i < shards.size(); i++) { Shard pm = shards.get(i); if (pIds[i % size] == null) { pIds[i % size] = new HashSet<String>(); } pIds[i % size].add(pm.getShardId()); } if (isInitialParitition) { lastRepartitionTime = System.currentTimeMillis(); logger.info("[MANY_TO_ONE]: Initializing partition(s)"); } else { logger.info("[MANY_TO_ONE]: Add operator partition for kinesis partition(s): " + StringUtils.join(newWaitingPartition, ", ") + ", StreamName: " + this.getConsumer().streamName); newWaitingPartition.clear(); } for (int i = 0; i < pIds.length; i++) { logger.info("[MANY_TO_ONE]: Create operator partition for kinesis partition(s): " + StringUtils.join(pIds[i], ", ") + ", StreamName: " + this.getConsumer().streamName); if (pIds[i] != null) newPartitions.add(createPartition(pIds[i], initShardPos)); } break; default: break; } return newPartitions; }
From source file:com.datatorrent.contrib.kinesis.AbstractKinesisInputOperator.java
License:Open Source License
private boolean isPartitionRequired(int opid, List<KinesisConsumer.KinesisShardStats> kstats) { long t = System.currentTimeMillis(); if (t - lastCheckTime < repartitionCheckInterval) { // return false if it's within repartitionCheckInterval since last time it check the stats return false; }/*from ww w.ja v a2s.co m*/ logger.debug("Use ShardManager to update the Shard Positions"); updateShardPositions(kstats); if (repartitionInterval < 0) { // if repartition is disabled return false; } if (t - lastRepartitionTime < repartitionInterval) { // return false if it's still within repartitionInterval since last (re)partition return false; } try { // monitor if shards are repartitioned Set<String> existingIds = new HashSet<String>(); for (PartitionInfo pio : currentPartitionInfo) { existingIds.addAll(pio.kpids); } List<Shard> shards = KinesisUtil.getInstance().getShardList(getStreamName()); for (Shard shard : shards) { if (!existingIds.contains(shard.getShardId())) { newWaitingPartition.add(shard.getShardId()); } } if (newWaitingPartition.size() != 0) { // found new kinesis partition lastRepartitionTime = t; return true; } return false; } finally { // update last check time lastCheckTime = System.currentTimeMillis(); } }
From source file:com.datatorrent.contrib.kinesis.KinesisConsumer.java
License:Open Source License
/** * This method is called in setup method of the operator *///from w w w . j a va 2s . c om public void create() { holdingBuffer = new ArrayBlockingQueue<Pair<String, Record>>(bufferSize); boolean defaultSelect = (shardIds == null) || (shardIds.size() == 0); final List<Shard> pms = KinesisUtil.getInstance().getShardList(streamName); for (final Shard shId : pms) { if ((shardIds.contains(shId.getShardId()) || defaultSelect) && !closedShards.contains(shId.getShardId())) { simpleConsumerThreads.add(shId); } } }
From source file:com.datatorrent.contrib.kinesis.KinesisConsumer.java
License:Open Source License
/** * This method is called in the activate method of the operator *//*w w w. j ava 2s . c o m*/ public void start() { isAlive = true; int realNumStream = simpleConsumerThreads.size(); if (realNumStream == 0) return; consumerThreadExecutor = Executors.newFixedThreadPool(realNumStream); for (final Shard shd : simpleConsumerThreads) { consumerThreadExecutor.submit(new Runnable() { @Override public void run() { logger.debug("Thread " + Thread.currentThread().getName() + " start consuming Records..."); while (isAlive) { Shard shard = shd; try { List<Record> records = KinesisUtil.getInstance().getRecords(streamName, recordsLimit, shard, getIteratorType(shard.getShardId()), shardPosition.get(shard.getShardId())); if (records == null || records.isEmpty()) { if (shard.getSequenceNumberRange().getEndingSequenceNumber() != null) { closedShards.add(shard); break; } try { Thread.sleep(recordsCheckInterval); } catch (Exception e) { throw new RuntimeException(e); } } else { String seqNo = ""; for (Record rc : records) { seqNo = rc.getSequenceNumber(); putRecord(shd.getShardId(), rc); } shardPosition.put(shard.getShardId(), new String(seqNo)); } } catch (Exception e) { throw new RuntimeException(e); } } logger.debug("Thread " + Thread.currentThread().getName() + " stop consuming Records..."); } }); } }
From source file:com.datatorrent.contrib.kinesis.KinesisTestConsumer.java
License:Open Source License
@Override public void run() { DescribeStreamRequest describeRequest = new DescribeStreamRequest(); describeRequest.setStreamName(streamName); DescribeStreamResult describeResponse = client.describeStream(describeRequest); final List<Shard> shards = describeResponse.getStreamDescription().getShards(); logger.debug("Inside consumer::run receiveCount= {}", receiveCount); while (isAlive) { Shard shId = shards.get(0); GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId.getShardId()); iteratorRequest.setShardIteratorType("TRIM_HORIZON"); GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); String iterator = iteratorResponse.getShardIterator(); GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(1000);//from ww w .j ava2 s .c om getRequest.setShardIterator(iterator); //call "get" operation and get everything in this shard range GetRecordsResult getResponse = client.getRecords(getRequest); //get reference to next iterator for this shard //retrieve records List<Record> records = getResponse.getRecords(); if (records == null || records.isEmpty()) { try { Thread.sleep(1000); } catch (InterruptedException e) { throw new RuntimeException(e); } } else { String seqNo = ""; for (Record rc : records) { if (latch != null) { latch.countDown(); } seqNo = rc.getSequenceNumber(); if (getData(rc).equals(KinesisOperatorTestBase.END_TUPLE)) break; holdingBuffer.add(rc); receiveCount++; logger.debug("Consuming {}, receiveCount= {}", getData(rc), receiveCount); } } } logger.debug("DONE consuming"); }
From source file:com.datatorrent.contrib.kinesis.KinesisUtil.java
License:Open Source License
/** * Get the records from the particular shard * @param streamName Name of the stream from where the records to be accessed * @param recordsLimit Number of records to return from shard * @param shId Shard Id of the shard//from w w w. j a va2s .co m * @param iteratorType Shard iterator type * @param seqNo Record sequence number * @return the list of records from the given shard * @throws AmazonClientException */ public List<Record> getRecords(String streamName, Integer recordsLimit, Shard shId, ShardIteratorType iteratorType, String seqNo) throws AmazonClientException { assert client != null : "Illegal client"; try { // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType to it GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest(); iteratorRequest.setStreamName(streamName); iteratorRequest.setShardId(shId.getShardId()); iteratorRequest.setShardIteratorType(iteratorType); // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType)) iteratorRequest.setStartingSequenceNumber(seqNo); // Get the Response from the getShardIterator service method & get the shardIterator from that response GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest); // getShardIterator() specifies the position in the shard String iterator = iteratorResponse.getShardIterator(); // Create the GetRecordsRequest instance and set the recordsLimit and iterator GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(recordsLimit); getRequest.setShardIterator(iterator); // Get the Response from the getRecords service method and get the data records from that response. GetRecordsResult getResponse = client.getRecords(getRequest); return getResponse.getRecords(); } catch (AmazonClientException e) { throw new RuntimeException(e); } }
From source file:com.kinesis.datavis.utils.StreamUtils.java
License:Open Source License
/** * Split a shard by dividing the hash key space in half. * * @param streamName Name of the stream that contains the shard to split. * @param shardId The id of the shard to split. * * @throws IllegalArgumentException When either streamName or shardId are null or empty. * @throws LimitExceededException Shard limit for the account has been reached. * @throws ResourceNotFoundException The stream or shard cannot be found. * @throws InvalidArgumentException If the shard is closed and no eligible for splitting. * @throws AmazonClientException Error communicating with Amazon Kinesis. * *//*from w w w. ja v a2 s . c o m*/ public void splitShardEvenly(String streamName, String shardId) throws LimitExceededException, ResourceNotFoundException, AmazonClientException, InvalidArgumentException, IllegalArgumentException { if (streamName == null || streamName.isEmpty()) { throw new IllegalArgumentException("stream name is required"); } if (shardId == null || shardId.isEmpty()) { throw new IllegalArgumentException("shard id is required"); } DescribeStreamResult result = kinesis.describeStream(streamName); StreamDescription description = result.getStreamDescription(); // Find the shard we want to split Shard shardToSplit = null; for (Shard shard : description.getShards()) { if (shardId.equals(shard.getShardId())) { shardToSplit = shard; break; } } if (shardToSplit == null) { throw new ResourceNotFoundException( "Could not find shard with id '" + shardId + "' in stream '" + streamName + "'"); } // Check if the shard is still open. Open shards do not have an ending sequence number. if (shardToSplit.getSequenceNumberRange().getEndingSequenceNumber() != null) { throw new InvalidArgumentException("Shard is CLOSED and is not eligible for splitting"); } // Calculate the median hash key to use as the new starting hash key for the shard. BigInteger startingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getStartingHashKey()); BigInteger endingHashKey = new BigInteger(shardToSplit.getHashKeyRange().getEndingHashKey()); BigInteger[] medianHashKey = startingHashKey.add(endingHashKey).divideAndRemainder(new BigInteger("2")); BigInteger newStartingHashKey = medianHashKey[0]; if (!BigInteger.ZERO.equals(medianHashKey[1])) { // In order to more evenly distributed the new hash key ranges across the new shards we will "round up" to // the next integer when our current hash key range is not evenly divisible by 2. newStartingHashKey = newStartingHashKey.add(BigInteger.ONE); } // Submit the split shard request kinesis.splitShard(streamName, shardId, newStartingHashKey.toString()); }
From source file:com.qubole.presto.kinesis.KinesisSplitManager.java
License:Apache License
@Override public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout) { KinesisTableLayoutHandle kinesislayout = handleResolver.convertLayout(layout); KinesisTableHandle kinesisTableHandle = kinesislayout.getTable(); InternalStreamDescription desc = this.getStreamDescription(kinesisTableHandle.getStreamName()); ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder(); for (Shard shard : desc.getShards()) { KinesisSplit split = new KinesisSplit(connectorId, kinesisTableHandle.getStreamName(), kinesisTableHandle.getMessageDataFormat(), shard.getShardId(), shard.getSequenceNumberRange().getStartingSequenceNumber(), shard.getSequenceNumberRange().getEndingSequenceNumber()); builder.add(split);//from w w w . j a va2s . c o m } return new FixedSplitSource(connectorId, builder.build()); }
From source file:com.trulia.stail.Stail.java
License:Apache License
private static String getShardIterator(AmazonKinesis client, String stream, Shard shard, String start) { GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(stream); getShardIteratorRequest.setShardId(shard.getShardId()); if (!Strings.isNullOrEmpty(start)) { getShardIteratorRequest.setShardIteratorType(ShardIteratorType.AT_TIMESTAMP); getShardIteratorRequest/*from w w w . j a va 2s . c om*/ .setTimestamp(new Date(System.currentTimeMillis() - Duration.parse(start).toMillis())); } else { getShardIteratorRequest.setShardIteratorType(ShardIteratorType.LATEST); } GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest); return getShardIteratorResult.getShardIterator(); }