Example usage for com.amazonaws.services.kinesisfirehose.model PutRecordBatchResult getFailedPutCount

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesisfirehose.model PutRecordBatchResult getFailedPutCount.

Prototype

public Integer getFailedPutCount() 

Source Link

Document

The number of records that might have failed processing.

Usage

From source file:com.streamsets.pipeline.stage.destination.kinesis.FirehoseTarget.java

License:Apache License

private void flush(List<com.amazonaws.services.kinesisfirehose.model.Record> records, List<Record> sdcRecords)
        throws StageException {

    if (records.isEmpty()) {
        return;//from  ww w. jav a  2s .com
    }

    PutRecordBatchRequest batchRequest = new PutRecordBatchRequest().withDeliveryStreamName(conf.streamName)
            .withRecords(records);
    PutRecordBatchResult result = firehoseClient.putRecordBatch(batchRequest);
    int numFailed = result.getFailedPutCount();
    if (numFailed != 0) {
        List<PutRecordBatchResponseEntry> responses = result.getRequestResponses();
        for (int i = 0; i < responses.size(); i++) {
            PutRecordBatchResponseEntry response = responses.get(i);
            if (response.getErrorCode() != null) {
                errorRecordHandler.onError(new OnRecordErrorException(sdcRecords.get(i), Errors.KINESIS_05,
                        sdcRecords.get(i), response.getErrorMessage()));
            }
        }
    }

    recordCounter += records.size();

    records.clear();
    sdcRecords.clear();
}

---

From source file:org.voltdb.exportclient.FirehoseSink.java

License:Open Source License

ListenableFuture<?> asWriteTask(List<Record> recordsList) {
    final int hashed = ThreadLocalRandom.current().nextInt(m_concurrentWriters);
    if (m_executors.get(hashed).isShutdown()) {
        return Futures
                .immediateFailedFuture(new FirehoseExportException("Firehose sink executor is shut down"));
    }// ww  w .  j  a  va  2 s  .c  om
    return m_executors.get(hashed).submit(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
            PutRecordBatchRequest batchRequest = new PutRecordBatchRequest()
                    .withDeliveryStreamName(m_streamName).withRecords(recordsList);
            applyBackPressure();
            PutRecordBatchResult res = m_client.putRecordBatch(batchRequest);
            if (res.getFailedPutCount() > 0) {
                setBackPressure(true);
                String msg = "%d Firehose records failed";
                LOG.warn(msg, res.getFailedPutCount());
                throw new FirehoseExportException(msg, res.getFailedPutCount());
            }
            setBackPressure(false);
            return null;
        }
    });
}

---

From source file:org.voltdb.exportclient.FirehoseSink.java

License:Open Source License

public void syncWrite(Queue<List<Record>> records) {

    for (List<Record> recordsList : records) {
        int retry = MAX_RETRY;
        while (retry > 0) {
            try {
                PutRecordBatchRequest batchRequest = new PutRecordBatchRequest()
                        .withDeliveryStreamName(m_streamName).withRecords(recordsList);
                PutRecordBatchResult res = m_client.putRecordBatch(batchRequest);
                if (res.getFailedPutCount() > 0) {
                    String msg = "Records failed with the batch: %d, retry: #%d";
                    if (retry == 1) {
                        throw new FirehoseExportException(msg, res.getFailedPutCount(),
                                (MAX_RETRY - retry + 1));
                    } else {
                        LOG.warn(msg, res.getFailedPutCount(), (MAX_RETRY - retry + 1));
                        backoffSleep(retry);
                    }//from  www .  jav  a  2s  .c  om
                } else {
                    recordsList.clear();
                    break;
                }
            } catch (ServiceUnavailableException e) {
                if (retry == 1) {
                    throw new FirehoseExportException("Failed to send record batch", e, true);
                } else {
                    LOG.warn("Failed to send record batch: %s. Retry #%d", e.getErrorMessage(),
                            (MAX_RETRY - retry + 1));
                    backoffSleep(retry);
                }
            }
            retry--;
        }
    }
}

---