Example usage for com.amazonaws.services.kinesis.model PutRecordsRequest PutRecordsRequest

List of usage examples for com.amazonaws.services.kinesis.model PutRecordsRequest PutRecordsRequest

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.kinesis.model.*
  5. PutRecordsRequest
  6. PutRecordsRequest

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model PutRecordsRequest PutRecordsRequest.

Prototype

PutRecordsRequest

Source Link

Usage

From source file:com.amazon.kinesis.streaming.agent.tailing.KinesisSender.java

License:Open Source License

@Override
protected BufferSendResult<KinesisRecord> attemptSend(RecordBuffer<KinesisRecord> buffer) {
    activePutRecordsCalls.incrementAndGet();
    IMetricsScope metrics = agentContext.beginScope();
    metrics.addDimension(Metrics.DESTINATION_DIMENSION, "KinesisStream:" + getDestination());
    try {/*from  ww w  .j a  va  2 s  .  c  om*/
        BufferSendResult<KinesisRecord> sendResult = null;
        List<PutRecordsRequestEntry> requestRecords = new ArrayList<>();
        for (KinesisRecord data : buffer) {
            PutRecordsRequestEntry record = new PutRecordsRequestEntry();
            record.setData(data.data());
            record.setPartitionKey(data.partitionKey());
            requestRecords.add(record);
        }
        PutRecordsRequest request = new PutRecordsRequest();
        request.setStreamName(getDestination());
        request.setRecords(requestRecords);
        PutRecordsResult result = null;
        Stopwatch timer = Stopwatch.createStarted();
        totalPutRecordsCalls.incrementAndGet();
        try {
            logger.trace("{}: Sending buffer {} to kinesis stream {}...", flow.getId(), buffer,
                    getDestination());
            metrics.addCount(RECORDS_ATTEMPTED_METRIC, requestRecords.size());
            result = agentContext.getKinesisClient().putRecords(request);
            metrics.addCount(SERVICE_ERRORS_METRIC, 0);
        } catch (AmazonServiceException e) {
            metrics.addCount(SERVICE_ERRORS_METRIC, 1);
            totalPutRecordsServiceErrors.incrementAndGet();
            throw e;
        } catch (Exception e) {
            metrics.addCount(SERVICE_ERRORS_METRIC, 1);
            totalPutRecordsOtherErrors.incrementAndGet();
            throw e;
        } finally {
            totalPutRecordsLatency.addAndGet(timer.elapsed(TimeUnit.MILLISECONDS));
        }
        if (sendResult == null) {
            List<Integer> sentRecords = new ArrayList<>(requestRecords.size());
            Multiset<String> errors = HashMultiset.<String>create();
            int index = 0;
            long totalBytesSent = 0;
            for (final PutRecordsResultEntry responseEntry : result.getRecords()) {
                final PutRecordsRequestEntry record = requestRecords.get(index);
                if (responseEntry.getErrorCode() == null) {
                    sentRecords.add(index);
                    totalBytesSent += record.getData().limit();
                } else {
                    logger.trace("{}:{} Record {} returned error code {}: {}", flow.getId(), buffer, index,
                            responseEntry.getErrorCode(), responseEntry.getErrorMessage());
                    errors.add(responseEntry.getErrorCode());
                }
                ++index;
            }
            if (sentRecords.size() == requestRecords.size()) {
                sendResult = BufferSendResult.succeeded(buffer);
            } else {
                buffer = buffer.remove(sentRecords);
                sendResult = BufferSendResult.succeeded_partially(buffer, requestRecords.size());
            }
            metrics.addData(BYTES_SENT_METRIC, totalBytesSent, StandardUnit.Bytes);
            int failedRecordCount = requestRecords.size() - sentRecords.size();
            metrics.addCount(RECORD_ERRORS_METRIC, failedRecordCount);
            logger.debug("{}:{} Records sent to kinesis stream {}: {}. Failed records: {}", flow.getId(),
                    buffer, getDestination(), sentRecords.size(), failedRecordCount);
            totalRecordsAttempted.addAndGet(requestRecords.size());
            totalRecordsSent.addAndGet(sentRecords.size());
            totalRecordsFailed.addAndGet(failedRecordCount);

            if (logger.isDebugEnabled() && !errors.isEmpty()) {
                synchronized (totalErrors) {
                    StringBuilder strErrors = new StringBuilder();
                    for (Multiset.Entry<String> err : errors.entrySet()) {
                        AtomicLong counter = totalErrors.get(err.getElement());
                        if (counter == null)
                            totalErrors.put(err.getElement(), counter = new AtomicLong());
                        counter.addAndGet(err.getCount());
                        if (strErrors.length() > 0)
                            strErrors.append(", ");
                        strErrors.append(err.getElement()).append(": ").append(err.getCount());
                    }
                    logger.debug("{}:{} Errors from kinesis stream {}: {}", flow.getId(), buffer,
                            flow.getDestination(), strErrors.toString());
                }
            }
        }
        return sendResult;
    } finally {
        metrics.commit();
        activePutRecordsCalls.decrementAndGet();
    }
}

From source file:com.datatorrent.contrib.kinesis.AbstractKinesisOutputOperator.java

License:Open Source License

private void flushRecords() {
    try {//ww  w . j a  v  a  2s. c  o m
        PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
        putRecordsRequest.setStreamName(streamName);
        putRecordsRequest.setRecords(putRecordsRequestEntryList);
        PutRecordsResult putRecordsResult = client.putRecords(putRecordsRequest);
        putRecordsRequestEntryList.clear();
    } catch (AmazonClientException e) {
        throw new RuntimeException(e);
    }
}

From source file:com.datatorrent.contrib.kinesis.KinesisTestProducer.java

License:Open Source License

private void generateRecords() {
    // Create dummy message
    int recordNo = 1;
    while (recordNo <= sendCount) {
        String dataStr = "Record_" + recordNo;
        PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
        putRecordsEntry.setData(ByteBuffer.wrap(dataStr.getBytes()));
        putRecordsEntry.setPartitionKey(dataStr);
        putRecordsRequestEntryList.add(putRecordsEntry);
        if ((putRecordsRequestEntryList.size() == batchSize) || (recordNo == sendCount)) {
            PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
            putRecordsRequest.setStreamName(streamName);
            putRecordsRequest.setRecords(putRecordsRequestEntryList);
            PutRecordsResult putRecordsResult = client.putRecords(putRecordsRequest);
            putRecordsRequestEntryList.clear();
        }/* ww  w  .ja v a  2 s .  c  o m*/
        recordNo++;
    }
}

From source file:com.srotya.flume.kinesis.sink.KinesisSink.java

License:Apache License

@Override
public Status process() throws EventDeliveryException {
    Status status = Status.READY;//from  w  w w. ja va2  s  . c  om
    Channel ch = getChannel();
    Transaction tx = ch.getTransaction();
    tx.begin();
    try {
        PutRecordsRequest request = new PutRecordsRequest();
        List<PutRecordsRequestEntry> records = new ArrayList<>();
        for (int i = 0; i < putSize; i++) {
            Event event = ch.take();
            PutRecordsRequestEntry entry = new PutRecordsRequestEntry();
            if (partitionKey != null) {
                entry.setPartitionKey(event.getHeaders().get(partitionKey));
            } else {
                entry.setPartitionKey(String.format("partitionKey-%d", i));
            }
            if (hashKey != null) {
                entry.setExplicitHashKey(event.getHeaders().get(hashKey));
            }
            entry.setData(serializer.serialize(event));
            records.add(entry);
        }
        request.setRecords(records);
        request.setStreamName(streamName);
        client.putRecords(request);
        tx.commit();
    } catch (Throwable e) {
        e.printStackTrace();
        tx.rollback();
        status = Status.BACKOFF;
    } finally {
        try {
            tx.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    return status;
}

From source file:com.streamsets.pipeline.stage.destination.kinesis.KinesisTarget.java

License:Apache License

private void processBulkPut(List<Record> records) throws StageException {
    PutRecordsRequest request = new PutRecordsRequest();
    request.setStreamName(streamName);//from w  w w.  j  ava 2s .  c o m

    List<PutRecordsRequestEntry> requestEntries = new ArrayList<>();

    int i = 0;
    for (Record record : records) {
        final PutRecordsRequestEntry entry = new PutRecordsRequestEntry();

        ByteArrayOutputStream bytes = new ByteArrayOutputStream(1024 * records.size());
        try {
            DataGenerator generator = generatorFactory.getGenerator(bytes);
            generator.write(record);
            generator.close();

            entry.setData(ByteBuffer.wrap(bytes.toByteArray()));
            entry.setPartitionKey(getPartitionKey(i));

            requestEntries.add(entry);
            ++i;
        } catch (IOException e) {
            handleFailedRecord(record, "Failed to serialize record");
        }
    }

    request.setRecords(requestEntries);
    try {
        PutRecordsResult result = kinesisClient.putRecords(request);

        final Integer failedRecordCount = result.getFailedRecordCount();
        if (failedRecordCount > 0) {
            List<PutRecordsResultEntry> resultEntries = result.getRecords();
            i = 0;
            for (PutRecordsResultEntry resultEntry : resultEntries) {
                final String errorCode = resultEntry.getErrorCode();
                if (null != errorCode) {
                    switch (errorCode) {
                    case "ProvisionedThroughputExceededException":
                    case "InternalFailure":
                        // Records are processed in the order you submit them,
                        // so this will align with the initial record batch
                        handleFailedRecord(records.get(i), errorCode + ":" + resultEntry.getErrorMessage());
                        break;
                    default:
                        validateSuccessfulRecord(records.get(i), resultEntry);
                        break;
                    }
                } else {
                    validateSuccessfulRecord(records.get(i), resultEntry);
                }
                ++i;
            }
        }
    } catch (AmazonClientException e) {
        // Unrecoverable exception -- invalidate the entire batch
        LOG.debug("Exception while putting records", e);
        for (Record record : records) {
            handleFailedRecord(record, "Batch failed due to Amazon service exception: " + e.getMessage());
        }
    }
}

From source file:dbtucker.connect.kinesis.KinesisSinkTask.java

License:Apache License

private void putRecords(String streamName, List<PutRecordsRequestEntry> entries) {
    // TODO  Get smarter about cleaning up memory from PutRecordRequestEntries
    log.debug("Sending {} records to stream {}", entries.size(), streamName);
    final Integer maxRecordsPerRequest = 100; // AWS limit is 500, for now
    Integer numEntries;//from w w w . java  2 s  . c om
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    PutRecordsResult putRecordsResult;
    List<PutRecordsRequestEntry> entriesSubset;

    putRecordsRequest.setStreamName(streamName);
    numEntries = entries.size();
    while (numEntries > maxRecordsPerRequest) {
        entriesSubset = entries.subList(0, maxRecordsPerRequest - 1);
        putRecordsRequest.setRecords(entriesSubset);
        putRecordsResult = client.putRecords(putRecordsRequest);
        log.debug("putRecords returns {}", putRecordsResult.toString());

        entries = entries.subList(maxRecordsPerRequest, entries.size() - 1);
        numEntries = entries.size();
    }
    if (numEntries > 0) {
        putRecordsRequest.setRecords(entries);
        putRecordsResult = client.putRecords(putRecordsRequest);
        log.debug("putRecords returns {}", putRecordsResult.toString());
    }
}

From source file:gov.pnnl.cloud.producer.kinesis.ProducerBase.java

License:Open Source License

public void run() {

    while (true) {
        try {/*w  w w  .  j a va 2  s .c  o m*/
            // get message from queue - blocking so code will wait here for work to do
            Event event = eventsQueue.take();

            List<PutRecordsRequestEntry> puts = new ArrayList<PutRecordsRequestEntry>();

            // bump up the volume!
            for (int i = 0; i < 150; i++) {

                PutRecordsRequestEntry put = new PutRecordsRequestEntry();

                put.setData(event.getData());
                put.setPartitionKey(String.valueOf(System.currentTimeMillis()));
                puts.add(put);

                stats.increment(Key.KINESIS_MESSAGE_GENERATED);

            }

            PutRecordsRequest put = new PutRecordsRequest();
            put.setRecords(puts);
            put.setStreamName(this.streamName);

            PutRecordsResult result = kinesisClient.putRecords(put);
            //logger.info(result.getSequenceNumber() + ": {}", this);   
            stats.increment(Key.KINESIS_MESSAGE_WRITTEN);

            if (stats.getStatValue(Key.KINESIS_MESSAGE_WRITTEN) > 10000) {
                stats.outStats();
                System.exit(0);
            }

        } catch (Exception e) {
            // didn't get record - move on to next\
            e.printStackTrace();

            stats.increment(Key.KINESIS_WRITE_ERROR);
        }
    }

}

From source file:lumbermill.internal.aws.SimpleRetryableKinesisClient.java

License:Apache License

/**
 * Asynchronously puts records to kinesis.
 *
 * @param events - List of events to send
 * @return - Observable with same list as parameter
 *//*w  ww  .java2s  . c o m*/
public Observable<List<T>> putRecords(List<T> events) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("putRecords() with {} events", events.size());
    }

    RequestContext request = new RequestContext(events, new PutRecordsRequest()
            .withRecords(events.stream().map(this::toRecordEntries).collect(toList())).withStreamName(stream));
    putRecordsAsync(request);

    return request.subject;

}

From source file:org.apache.apex.malhar.contrib.kinesis.AbstractKinesisOutputOperator.java

License:Apache License

private void flushRecords() {
    try {// w w  w  .j  a  va 2 s .  c  om
        PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
        putRecordsRequest.setStreamName(streamName);
        putRecordsRequest.setRecords(putRecordsRequestEntryList);
        client.putRecords(putRecordsRequest);
        putRecordsRequestEntryList.clear();
        logger.debug("Records flushed.");
    } catch (AmazonClientException e) {
        logger.warn("PutRecordsRequest exception.", e);
        throw new RuntimeException(e);
    }
}

From source file:org.apache.apex.malhar.contrib.kinesis.KinesisTestProducer.java

License:Apache License

private void generateRecords() {
    // Create dummy message
    int recordNo = 1;
    while (recordNo <= sendCount) {
        String dataStr = "Record_" + recordNo;
        PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
        putRecordsEntry.setData(ByteBuffer.wrap(dataStr.getBytes()));
        putRecordsEntry.setPartitionKey(dataStr);
        putRecordsRequestEntryList.add(putRecordsEntry);
        if ((putRecordsRequestEntryList.size() == batchSize) || (recordNo == sendCount)) {
            PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
            putRecordsRequest.setStreamName(streamName);
            putRecordsRequest.setRecords(putRecordsRequestEntryList);
            client.putRecords(putRecordsRequest);
            putRecordsRequestEntryList.clear();
        }//from   w  ww. ja  v  a  2  s  .  c  om
        recordNo++;
    }
}