Example usage for com.amazonaws.services.kinesis.model PutRecordsResult getFailedRecordCount

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis.model PutRecordsResult getFailedRecordCount.

Prototype

public Integer getFailedRecordCount() 

Source Link

Document

The number of unsuccessfully processed records in a PutRecords request.

Usage

From source file:com.streamsets.pipeline.stage.destination.kinesis.KinesisTarget.java

License:Apache License

private void processBulkPut(List<Record> records) throws StageException {
    PutRecordsRequest request = new PutRecordsRequest();
    request.setStreamName(streamName);/*from  w w  w  .j  a  v a2 s. co m*/

    List<PutRecordsRequestEntry> requestEntries = new ArrayList<>();

    int i = 0;
    for (Record record : records) {
        final PutRecordsRequestEntry entry = new PutRecordsRequestEntry();

        ByteArrayOutputStream bytes = new ByteArrayOutputStream(1024 * records.size());
        try {
            DataGenerator generator = generatorFactory.getGenerator(bytes);
            generator.write(record);
            generator.close();

            entry.setData(ByteBuffer.wrap(bytes.toByteArray()));
            entry.setPartitionKey(getPartitionKey(i));

            requestEntries.add(entry);
            ++i;
        } catch (IOException e) {
            handleFailedRecord(record, "Failed to serialize record");
        }
    }

    request.setRecords(requestEntries);
    try {
        PutRecordsResult result = kinesisClient.putRecords(request);

        final Integer failedRecordCount = result.getFailedRecordCount();
        if (failedRecordCount > 0) {
            List<PutRecordsResultEntry> resultEntries = result.getRecords();
            i = 0;
            for (PutRecordsResultEntry resultEntry : resultEntries) {
                final String errorCode = resultEntry.getErrorCode();
                if (null != errorCode) {
                    switch (errorCode) {
                    case "ProvisionedThroughputExceededException":
                    case "InternalFailure":
                        // Records are processed in the order you submit them,
                        // so this will align with the initial record batch
                        handleFailedRecord(records.get(i), errorCode + ":" + resultEntry.getErrorMessage());
                        break;
                    default:
                        validateSuccessfulRecord(records.get(i), resultEntry);
                        break;
                    }
                } else {
                    validateSuccessfulRecord(records.get(i), resultEntry);
                }
                ++i;
            }
        }
    } catch (AmazonClientException e) {
        // Unrecoverable exception -- invalidate the entire batch
        LOG.debug("Exception while putting records", e);
        for (Record record : records) {
            handleFailedRecord(record, "Batch failed due to Amazon service exception: " + e.getMessage());
        }
    }
}

---

From source file:lumbermill.internal.aws.SimpleRetryableKinesisClient.java

License:Apache License

/**
 * Recursively retry until there are no more failed records in response or to many retries
 *//*from   w  w w  . ja va  2  s.  c  o  m*/
private void putRecordsAsync(final RequestContext request) {
    amazonKinesisClient.putRecordsAsync(request.putRecordsRequest,
            new AsyncHandler<PutRecordsRequest, PutRecordsResult>() {

                @Override
                public void onError(Exception exception) {
                    try {
                        Observable<Optional<RequestContext>> observable = request.nextAttempt();
                        observable
                                .doOnNext(requestContext -> LOGGER.warn(
                                        "About to retry request from exception: {}", exception.getMessage()))
                                .doOnNext(context -> {
                                    if (context.isPresent()) {
                                        putRecordsAsync(context.get());
                                    } else {
                                        request.error(new FatalAWSException(
                                                "Too many kinesis retries, root cause:", exception));
                                    }
                                }).doOnError(throwable -> request.error(throwable)).subscribe();
                    } catch (Throwable t) {
                        LOGGER.error("Unexpected exception in onError()", t);
                        request.error(t);
                    }
                }

                @Override
                public void onSuccess(PutRecordsRequest putRecordsRequest, PutRecordsResult putRecordsResult) {
                    // Surround with try/catch to prevent any unexpected exceptions from beeing swallowed
                    try {
                        if (putRecordsResult.getFailedRecordCount() > 0) {
                            LOGGER.debug("Got {} failed records, retrying (attempts = {})",
                                    putRecordsResult.getFailedRecordCount(), request.attempt);
                            // Try again with failing records,
                            //Optional<RequestContext> nextAttempt = request.nextAttempt(putRecordsResult);
                            Observable<Optional<RequestContext>> observable = request
                                    .nextAttempt(putRecordsResult);
                            observable.doOnNext(context -> {
                                if (context.isPresent()) {
                                    putRecordsAsync(context.get());
                                } else {
                                    request.error(new FatalAWSException("Too many kinesis retries"));
                                }
                            }).doOnError(throwable -> request.error(throwable)).subscribe();
                        } else {
                            request.done();
                        }
                    } catch (Throwable t) {
                        LOGGER.error("Unexpected exception in onSuccess()", t);
                        request.error(t);
                    }
                }
            });
}

---

From source file:org.apache.beam.sdk.io.kinesis.KinesisUploader.java

License:Apache License

public static void uploadAll(List<String> data, KinesisTestOptions options) {
    AmazonKinesisClient client = new AmazonKinesisClient(new StaticCredentialsProvider(
            new BasicAWSCredentials(options.getAwsAccessKey(), options.getAwsSecretKey())))
                    .withRegion(Regions.fromName(options.getAwsKinesisRegion()));

    List<List<String>> partitions = Lists.partition(data, MAX_NUMBER_OF_RECORDS_IN_BATCH);

    for (List<String> partition : partitions) {
        List<PutRecordsRequestEntry> allRecords = newArrayList();
        for (String row : partition) {
            allRecords.add(new PutRecordsRequestEntry().withData(ByteBuffer.wrap(row.getBytes(Charsets.UTF_8)))
                    .withPartitionKey(Integer.toString(row.hashCode()))

            );//from www .j  a v a 2  s. c  o m
        }

        PutRecordsResult result;
        do {
            result = client.putRecords(new PutRecordsRequest().withStreamName(options.getAwsKinesisStream())
                    .withRecords(allRecords));
            List<PutRecordsRequestEntry> failedRecords = newArrayList();
            int i = 0;
            for (PutRecordsResultEntry row : result.getRecords()) {
                if (row.getErrorCode() != null) {
                    failedRecords.add(allRecords.get(i));
                }
                ++i;
            }
            allRecords = failedRecords;
        }

        while (result.getFailedRecordCount() > 0);
    }
}

---

From source file:utils.kinesis.OldStyleKinesisUploader.java

License:Apache License

private void uploadAll(List<String> data) {
    List<List<String>> partitions = Lists.partition(data, 499);

    for (List<String> partition : partitions) {
        List<PutRecordsRequestEntry> allRecords = newArrayList();
        for (String row : partition) {
            allRecords.add(new PutRecordsRequestEntry().withData(ByteBuffer.wrap(row.getBytes(Charsets.UTF_8)))
                    .withPartitionKey(Integer.toString(row.hashCode()))

            );//from ww w.j a v  a 2s .co  m
        }

        PutRecordsResult result;
        do {
            result = client.putRecords(new PutRecordsRequest()
                    .withStreamName(TestConfiguration.get().getTestKinesisStream()).withRecords(allRecords));
            List<PutRecordsRequestEntry> failedRecords = newArrayList();
            int i = 0;
            for (PutRecordsResultEntry row : result.getRecords()) {
                if (row.getErrorCode() != null) {
                    failedRecords.add(allRecords.get(i));
                }
                ++i;
            }
            allRecords = failedRecords;
        }

        while (result.getFailedRecordCount() > 0);
    }
}

---