Example usage for com.amazonaws.services.kinesis.model PutRecordsRequestEntry PutRecordsRequestEntry

List of usage examples for com.amazonaws.services.kinesis.model PutRecordsRequestEntry PutRecordsRequestEntry

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.kinesis.model.*
  5. PutRecordsRequestEntry
  6. PutRecordsRequestEntry

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model PutRecordsRequestEntry PutRecordsRequestEntry.

Prototype

PutRecordsRequestEntry

Source Link

Usage

From source file:com.amazon.kinesis.streaming.agent.tailing.KinesisSender.java

License:Open Source License

@Override
protected BufferSendResult<KinesisRecord> attemptSend(RecordBuffer<KinesisRecord> buffer) {
    activePutRecordsCalls.incrementAndGet();
    IMetricsScope metrics = agentContext.beginScope();
    metrics.addDimension(Metrics.DESTINATION_DIMENSION, "KinesisStream:" + getDestination());
    try {//from www  .  ja v a  2 s . c o  m
        BufferSendResult<KinesisRecord> sendResult = null;
        List<PutRecordsRequestEntry> requestRecords = new ArrayList<>();
        for (KinesisRecord data : buffer) {
            PutRecordsRequestEntry record = new PutRecordsRequestEntry();
            record.setData(data.data());
            record.setPartitionKey(data.partitionKey());
            requestRecords.add(record);
        }
        PutRecordsRequest request = new PutRecordsRequest();
        request.setStreamName(getDestination());
        request.setRecords(requestRecords);
        PutRecordsResult result = null;
        Stopwatch timer = Stopwatch.createStarted();
        totalPutRecordsCalls.incrementAndGet();
        try {
            logger.trace("{}: Sending buffer {} to kinesis stream {}...", flow.getId(), buffer,
                    getDestination());
            metrics.addCount(RECORDS_ATTEMPTED_METRIC, requestRecords.size());
            result = agentContext.getKinesisClient().putRecords(request);
            metrics.addCount(SERVICE_ERRORS_METRIC, 0);
        } catch (AmazonServiceException e) {
            metrics.addCount(SERVICE_ERRORS_METRIC, 1);
            totalPutRecordsServiceErrors.incrementAndGet();
            throw e;
        } catch (Exception e) {
            metrics.addCount(SERVICE_ERRORS_METRIC, 1);
            totalPutRecordsOtherErrors.incrementAndGet();
            throw e;
        } finally {
            totalPutRecordsLatency.addAndGet(timer.elapsed(TimeUnit.MILLISECONDS));
        }
        if (sendResult == null) {
            List<Integer> sentRecords = new ArrayList<>(requestRecords.size());
            Multiset<String> errors = HashMultiset.<String>create();
            int index = 0;
            long totalBytesSent = 0;
            for (final PutRecordsResultEntry responseEntry : result.getRecords()) {
                final PutRecordsRequestEntry record = requestRecords.get(index);
                if (responseEntry.getErrorCode() == null) {
                    sentRecords.add(index);
                    totalBytesSent += record.getData().limit();
                } else {
                    logger.trace("{}:{} Record {} returned error code {}: {}", flow.getId(), buffer, index,
                            responseEntry.getErrorCode(), responseEntry.getErrorMessage());
                    errors.add(responseEntry.getErrorCode());
                }
                ++index;
            }
            if (sentRecords.size() == requestRecords.size()) {
                sendResult = BufferSendResult.succeeded(buffer);
            } else {
                buffer = buffer.remove(sentRecords);
                sendResult = BufferSendResult.succeeded_partially(buffer, requestRecords.size());
            }
            metrics.addData(BYTES_SENT_METRIC, totalBytesSent, StandardUnit.Bytes);
            int failedRecordCount = requestRecords.size() - sentRecords.size();
            metrics.addCount(RECORD_ERRORS_METRIC, failedRecordCount);
            logger.debug("{}:{} Records sent to kinesis stream {}: {}. Failed records: {}", flow.getId(),
                    buffer, getDestination(), sentRecords.size(), failedRecordCount);
            totalRecordsAttempted.addAndGet(requestRecords.size());
            totalRecordsSent.addAndGet(sentRecords.size());
            totalRecordsFailed.addAndGet(failedRecordCount);

            if (logger.isDebugEnabled() && !errors.isEmpty()) {
                synchronized (totalErrors) {
                    StringBuilder strErrors = new StringBuilder();
                    for (Multiset.Entry<String> err : errors.entrySet()) {
                        AtomicLong counter = totalErrors.get(err.getElement());
                        if (counter == null)
                            totalErrors.put(err.getElement(), counter = new AtomicLong());
                        counter.addAndGet(err.getCount());
                        if (strErrors.length() > 0)
                            strErrors.append(", ");
                        strErrors.append(err.getElement()).append(": ").append(err.getCount());
                    }
                    logger.debug("{}:{} Errors from kinesis stream {}: {}", flow.getId(), buffer,
                            flow.getDestination(), strErrors.toString());
                }
            }
        }
        return sendResult;
    } finally {
        metrics.commit();
        activePutRecordsCalls.decrementAndGet();
    }
}

From source file:com.datatorrent.contrib.kinesis.AbstractKinesisOutputOperator.java

License:Open Source License

private void addRecord(T tuple) {
    try {//ww w  .  j a  va 2  s.  c o m
        Pair<String, V> keyValue = tupleToKeyValue(tuple);
        PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
        putRecordsEntry.setData(ByteBuffer.wrap(getRecord(keyValue.second)));
        putRecordsEntry.setPartitionKey(keyValue.first);
        putRecordsRequestEntryList.add(putRecordsEntry);
    } catch (AmazonClientException e) {
        throw new RuntimeException(e);
    }
}

From source file:com.datatorrent.contrib.kinesis.KinesisTestProducer.java

License:Open Source License

private void generateRecords() {
    // Create dummy message
    int recordNo = 1;
    while (recordNo <= sendCount) {
        String dataStr = "Record_" + recordNo;
        PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
        putRecordsEntry.setData(ByteBuffer.wrap(dataStr.getBytes()));
        putRecordsEntry.setPartitionKey(dataStr);
        putRecordsRequestEntryList.add(putRecordsEntry);
        if ((putRecordsRequestEntryList.size() == batchSize) || (recordNo == sendCount)) {
            PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
            putRecordsRequest.setStreamName(streamName);
            putRecordsRequest.setRecords(putRecordsRequestEntryList);
            PutRecordsResult putRecordsResult = client.putRecords(putRecordsRequest);
            putRecordsRequestEntryList.clear();
        }/*from  w w  w  . ja va2 s  . c  o m*/
        recordNo++;
    }
}

From source file:com.srotya.flume.kinesis.sink.KinesisSink.java

License:Apache License

@Override
public Status process() throws EventDeliveryException {
    Status status = Status.READY;//from  w w w. j av a 2 s . c  o m
    Channel ch = getChannel();
    Transaction tx = ch.getTransaction();
    tx.begin();
    try {
        PutRecordsRequest request = new PutRecordsRequest();
        List<PutRecordsRequestEntry> records = new ArrayList<>();
        for (int i = 0; i < putSize; i++) {
            Event event = ch.take();
            PutRecordsRequestEntry entry = new PutRecordsRequestEntry();
            if (partitionKey != null) {
                entry.setPartitionKey(event.getHeaders().get(partitionKey));
            } else {
                entry.setPartitionKey(String.format("partitionKey-%d", i));
            }
            if (hashKey != null) {
                entry.setExplicitHashKey(event.getHeaders().get(hashKey));
            }
            entry.setData(serializer.serialize(event));
            records.add(entry);
        }
        request.setRecords(records);
        request.setStreamName(streamName);
        client.putRecords(request);
        tx.commit();
    } catch (Throwable e) {
        e.printStackTrace();
        tx.rollback();
        status = Status.BACKOFF;
    } finally {
        try {
            tx.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    return status;
}

From source file:com.streamsets.pipeline.stage.destination.kinesis.KinesisTarget.java

License:Apache License

private void processBulkPut(List<Record> records) throws StageException {
    PutRecordsRequest request = new PutRecordsRequest();
    request.setStreamName(streamName);/*from w  w w .  jav  a2s . c om*/

    List<PutRecordsRequestEntry> requestEntries = new ArrayList<>();

    int i = 0;
    for (Record record : records) {
        final PutRecordsRequestEntry entry = new PutRecordsRequestEntry();

        ByteArrayOutputStream bytes = new ByteArrayOutputStream(1024 * records.size());
        try {
            DataGenerator generator = generatorFactory.getGenerator(bytes);
            generator.write(record);
            generator.close();

            entry.setData(ByteBuffer.wrap(bytes.toByteArray()));
            entry.setPartitionKey(getPartitionKey(i));

            requestEntries.add(entry);
            ++i;
        } catch (IOException e) {
            handleFailedRecord(record, "Failed to serialize record");
        }
    }

    request.setRecords(requestEntries);
    try {
        PutRecordsResult result = kinesisClient.putRecords(request);

        final Integer failedRecordCount = result.getFailedRecordCount();
        if (failedRecordCount > 0) {
            List<PutRecordsResultEntry> resultEntries = result.getRecords();
            i = 0;
            for (PutRecordsResultEntry resultEntry : resultEntries) {
                final String errorCode = resultEntry.getErrorCode();
                if (null != errorCode) {
                    switch (errorCode) {
                    case "ProvisionedThroughputExceededException":
                    case "InternalFailure":
                        // Records are processed in the order you submit them,
                        // so this will align with the initial record batch
                        handleFailedRecord(records.get(i), errorCode + ":" + resultEntry.getErrorMessage());
                        break;
                    default:
                        validateSuccessfulRecord(records.get(i), resultEntry);
                        break;
                    }
                } else {
                    validateSuccessfulRecord(records.get(i), resultEntry);
                }
                ++i;
            }
        }
    } catch (AmazonClientException e) {
        // Unrecoverable exception -- invalidate the entire batch
        LOG.debug("Exception while putting records", e);
        for (Record record : records) {
            handleFailedRecord(record, "Batch failed due to Amazon service exception: " + e.getMessage());
        }
    }
}

From source file:dbtucker.connect.kinesis.KinesisSinkTask.java

License:Apache License

@Override
public void put(Collection<SinkRecord> collection) {
    if (collection.isEmpty())
        return;//w  w  w. j ava 2  s  .c o  m
    final Map<String, List<PutRecordsRequestEntry>> writesByStream = new HashMap<>();
    for (SinkRecord record : collection) {
        final String streamName = config.getStreamFormat().replace("${topic}", record.topic());
        List<PutRecordsRequestEntry> writes = writesByStream.get(streamName);
        if (writes == null) {
            writes = new ArrayList<>();
            writesByStream.put(streamName, writes);
        }

        final PutRecordsRequestEntry put = new PutRecordsRequestEntry();
        put.setData(ByteBuffer.wrap(record.value().toString().getBytes()));
        if (record.key() != null) {
            if (record.keySchema() != null) {
                // TODO: correctly parse schema'ed key
                put.setPartitionKey(record.key().toString()); // assume toString handles real Strings correctly
            } else {
                put.setPartitionKey(record.key().toString()); // assume toString handles real Strings correctly
            }
        } else {
            put.setPartitionKey("Partition_1");
        }

        writes.add(put.clone());
    }

    writesByStream.forEach((stream, reqEntries) -> {
        try {
            putRecords(stream, reqEntries);
        } catch (Exception e) {
            log.warn("Write to stream {} failed; remaining retries={}", stream, remainingRetries);
            if (remainingRetries == 0) {
                throw new ConnectException(e);
            } else {
                remainingRetries--;
                context.timeout(config.getRetryBackoffMs());
                throw new RetriableException(e);
            }
        }
    });

    remainingRetries = config.getMaxRetries();
}

From source file:gov.pnnl.cloud.producer.kinesis.ProducerBase.java

License:Open Source License

public void run() {

    while (true) {
        try {// www .  java2 s  .  c om
            // get message from queue - blocking so code will wait here for work to do
            Event event = eventsQueue.take();

            List<PutRecordsRequestEntry> puts = new ArrayList<PutRecordsRequestEntry>();

            // bump up the volume!
            for (int i = 0; i < 150; i++) {

                PutRecordsRequestEntry put = new PutRecordsRequestEntry();

                put.setData(event.getData());
                put.setPartitionKey(String.valueOf(System.currentTimeMillis()));
                puts.add(put);

                stats.increment(Key.KINESIS_MESSAGE_GENERATED);

            }

            PutRecordsRequest put = new PutRecordsRequest();
            put.setRecords(puts);
            put.setStreamName(this.streamName);

            PutRecordsResult result = kinesisClient.putRecords(put);
            //logger.info(result.getSequenceNumber() + ": {}", this);   
            stats.increment(Key.KINESIS_MESSAGE_WRITTEN);

            if (stats.getStatValue(Key.KINESIS_MESSAGE_WRITTEN) > 10000) {
                stats.outStats();
                System.exit(0);
            }

        } catch (Exception e) {
            // didn't get record - move on to next\
            e.printStackTrace();

            stats.increment(Key.KINESIS_WRITE_ERROR);
        }
    }

}

From source file:lumbermill.internal.aws.SimpleRetryableKinesisClient.java

License:Apache License

/**
 * Converts event to actual kinesis entry type
 *///w  w w .j av  a 2s. c om
private PutRecordsRequestEntry toRecordEntries(T event) {
    //Optional<String> partitionKey = partitionKeyTemplate.format(event);
    return new PutRecordsRequestEntry().withData(event.raw().asByteBuffer())
            // FIXME: If partitionkey does not return a value, what approach is best?
            .withPartitionKey(
                    partitionKeySupplier.isPresent() ? partitionKeySupplier.get().get().format(event).get()
                            : UUID.randomUUID().toString());
}

From source file:org.apache.apex.malhar.contrib.kinesis.KinesisTestProducer.java

License:Apache License

private void generateRecords() {
    // Create dummy message
    int recordNo = 1;
    while (recordNo <= sendCount) {
        String dataStr = "Record_" + recordNo;
        PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
        putRecordsEntry.setData(ByteBuffer.wrap(dataStr.getBytes()));
        putRecordsEntry.setPartitionKey(dataStr);
        putRecordsRequestEntryList.add(putRecordsEntry);
        if ((putRecordsRequestEntryList.size() == batchSize) || (recordNo == sendCount)) {
            PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
            putRecordsRequest.setStreamName(streamName);
            putRecordsRequest.setRecords(putRecordsRequestEntryList);
            client.putRecords(putRecordsRequest);
            putRecordsRequestEntryList.clear();
        }/*from w  w  w  .  j a va  2  s  . c  o m*/
        recordNo++;
    }
}

From source file:org.apache.beam.sdk.io.kinesis.KinesisUploader.java

License:Apache License

public static void uploadAll(List<String> data, KinesisTestOptions options) {
    AmazonKinesisClient client = new AmazonKinesisClient(new StaticCredentialsProvider(
            new BasicAWSCredentials(options.getAwsAccessKey(), options.getAwsSecretKey())))
                    .withRegion(Regions.fromName(options.getAwsKinesisRegion()));

    List<List<String>> partitions = Lists.partition(data, MAX_NUMBER_OF_RECORDS_IN_BATCH);

    for (List<String> partition : partitions) {
        List<PutRecordsRequestEntry> allRecords = newArrayList();
        for (String row : partition) {
            allRecords.add(new PutRecordsRequestEntry().withData(ByteBuffer.wrap(row.getBytes(Charsets.UTF_8)))
                    .withPartitionKey(Integer.toString(row.hashCode()))

            );//from www . j  av a 2  s. com
        }

        PutRecordsResult result;
        do {
            result = client.putRecords(new PutRecordsRequest().withStreamName(options.getAwsKinesisStream())
                    .withRecords(allRecords));
            List<PutRecordsRequestEntry> failedRecords = newArrayList();
            int i = 0;
            for (PutRecordsResultEntry row : result.getRecords()) {
                if (row.getErrorCode() != null) {
                    failedRecords.add(allRecords.get(i));
                }
                ++i;
            }
            allRecords = failedRecords;
        }

        while (result.getFailedRecordCount() > 0);
    }
}