List of usage examples for com.amazonaws.services.kinesis AmazonKinesisClient shutdown
@Override public void shutdown()
From source file:com.streamsets.pipeline.stage.destination.kinesis.KinesisTarget.java
License:Apache License
private void checkStreamExists(List<ConfigIssue> issues) { ClientConfiguration kinesisConfiguration = new ClientConfiguration(); AmazonKinesisClient kinesisClient = new AmazonKinesisClient(kinesisConfiguration); kinesisClient.setRegion(Region.getRegion(region)); try {/*from w w w .j a v a 2s . c o m*/ DescribeStreamResult result = kinesisClient.describeStream(streamName); LOG.info("Connected successfully to stream: {} with description: {}", streamName, result.getStreamDescription().toString()); } catch (Exception e) { issues.add(getContext().createConfigIssue( com.streamsets.pipeline.stage.origin.kinesis.Groups.KINESIS.name(), "streamName", Errors.KINESIS_01, e.toString())); } finally { kinesisClient.shutdown(); } }
From source file:com.streamsets.pipeline.stage.lib.kinesis.KinesisUtil.java
License:Apache License
public static long getShardCount(Regions region, AWSConfig awsConfig, String streamName) throws AmazonClientException { ClientConfiguration kinesisConfiguration = new ClientConfiguration(); AmazonKinesisClient kinesisClient = new AmazonKinesisClient(AWSUtil.getCredentialsProvider(awsConfig), kinesisConfiguration);//from w ww . ja v a2s . co m kinesisClient.setRegion(Region.getRegion(region)); try { long numShards = 0; String lastShardId = null; StreamDescription description; do { if (lastShardId == null) { description = kinesisClient.describeStream(streamName).getStreamDescription(); } else { description = kinesisClient.describeStream(streamName, lastShardId).getStreamDescription(); } for (Shard shard : description.getShards()) { if (shard.getSequenceNumberRange().getEndingSequenceNumber() == null) { // Then this shard is open, so we should count it. Shards with an ending sequence number // are closed and cannot be written to, so we skip counting them. ++numShards; } } int pageSize = description.getShards().size(); lastShardId = description.getShards().get(pageSize - 1).getShardId(); } while (description.getHasMoreShards()); LOG.debug("Connected successfully to stream: '{}' with '{}' shards.", streamName, numShards); return numShards; } finally { kinesisClient.shutdown(); } }
From source file:com.streamsets.pipeline.stage.origin.kinesis.KinesisSource.java
License:Apache License
private void checkStreamExists(List<ConfigIssue> issues) { ClientConfiguration kinesisConfiguration = new ClientConfiguration(); AmazonKinesisClient kinesisClient = new AmazonKinesisClient(kinesisConfiguration); kinesisClient.setRegion(Region.getRegion(region)); try {/*from w w w.j a v a 2 s . c o m*/ DescribeStreamResult result = kinesisClient.describeStream(streamName); LOG.info("Connected successfully to stream: {} with description: {}", streamName, result.getStreamDescription().toString()); } catch (Exception e) { issues.add(getContext().createConfigIssue(Groups.KINESIS.name(), "streamName", Errors.KINESIS_01, e.toString())); } finally { kinesisClient.shutdown(); } }
From source file:dbtucker.connect.kinesis.KinesisSourceConnector.java
License:Apache License
@Override public void start(Map<String, String> map) { config = new KinesisSourceConnectorConfig(map); streamShards = new HashMap<>(); List<String> streamNames; final Set<String> ignoredStreams = new HashSet<>(); final Set<String> consumedStreams = new HashSet<>(); final AmazonKinesisClient client = new AmazonKinesisClient(); client.configureRegion(config.getRegionId()); ListStreamsResult listResult;//from ww w .j a va2 s . c o m ListStreamsRequest lsr = new ListStreamsRequest(); lsr.setLimit(32); String lastEvaluatedStreamName = null; do { lsr.setExclusiveStartStreamName(lastEvaluatedStreamName); listResult = client.listStreams(lsr); streamNames = listResult.getStreamNames(); for (String streamName : streamNames) { if (config.getStreamsPrefix() == null) { if ((config.getStreamsBlacklist() == null || config.getStreamsBlacklist().contains(streamName)) && (config.getStreamsWhitelist() == null || !config.getStreamsWhitelist().contains(streamName))) { ignoredStreams.add(streamName); continue; } } else { if (streamName.startsWith(config.getStreamsPrefix())) { if (config.getStreamsBlacklist() != null && config.getStreamsBlacklist().contains(streamName)) { ignoredStreams.add(streamName); continue; } } else { ignoredStreams.add(streamName); continue; } } final DescribeStreamResult streamDesc = client.describeStream(streamName); if (streamDesc.getStreamDescription().getStreamStatus().equals(StreamStatus.DELETING.toString())) { log.warn("Stream '{}' is being deleted and cannot be consumed", streamName); ignoredStreams.add(streamName); continue; } for (Shard shard : streamDesc.getStreamDescription().getShards()) { streamShards.put(shard, streamDesc); } consumedStreams.add(streamName); } if (streamNames.size() > 0) { lastEvaluatedStreamName = streamNames.get(streamNames.size() - 1); } } while (listResult.getHasMoreStreams()); log.info("Streams to ingest: {}", consumedStreams); log.info("Streams to ignore: {}", ignoredStreams); client.shutdown(); if (consumedStreams.isEmpty()) { throw new ConnectException("No matching Kinesis Streams found. Exiting connector"); } }
From source file:org.apache.samza.system.kinesis.KinesisSystemAdmin.java
License:Apache License
private SystemStreamMetadata createSystemStreamMetadata(String stream) { LOG.info("create stream metadata for stream {} based on aws stream", stream); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> metadata = new HashMap<>(); AmazonKinesisClient client = null; try {//from ww w . java 2s. c om ClientConfiguration clientConfig = kConfig.getAWSClientConfig(system); AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard() .withCredentials(kConfig.credentialsProviderForStream(system, stream)) .withClientConfiguration(clientConfig); builder.setRegion(kConfig.getRegion(system, stream).getName()); client = (AmazonKinesisClient) builder.build(); StreamDescription desc = client.describeStream(stream).getStreamDescription(); IntStream.range(0, desc.getShards().size()) .forEach(i -> metadata.put(new Partition(i), SYSTEM_STREAM_PARTITION_METADATA)); } catch (Exception e) { String errMsg = "couldn't load metadata for stream " + stream; LOG.error(errMsg, e); throw new SamzaException(errMsg, e); } finally { if (client != null) { client.shutdown(); } } return new SystemStreamMetadata(stream, metadata); }