Java tutorial
/* * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Amazon Software License (the "License"). * You may not use this file except in compliance with the License. * A copy of the License is located at * * http://aws.amazon.com/asl/ * * or in the "license" file accompanying this file. This file is distributed * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ package com.amazonaws.services.kinesis.producer; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.channels.FileLock; import java.nio.file.Paths; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import javax.xml.bind.DatatypeConverter; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.amazonaws.services.kinesis.producer.protobuf.Messages; import com.amazonaws.services.kinesis.producer.protobuf.Messages.Flush; import com.amazonaws.services.kinesis.producer.protobuf.Messages.Message; import com.amazonaws.services.kinesis.producer.protobuf.Messages.MetricsRequest; import com.amazonaws.services.kinesis.producer.protobuf.Messages.MetricsResponse; import com.amazonaws.services.kinesis.producer.protobuf.Messages.PutRecord; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.protobuf.ByteString; /** * An interface to the native KPL daemon. This class handles the creation, * destruction and use of the child process. * * <p> * <b>Use a single instance within the application whenever possible:</b> * <p> * <ul> * <li>One child process is spawned per instance of KinesisProducer. Additional * instances introduce overhead and reduce aggregation efficiency.</li> * <li>All streams within a region that can be accessed with the same * credentials can share the same KinesisProducer instance.</li> * <li>The {@link #addUserRecord} methods are thread safe, and be called * concurrently.</li> * <li>Therefore, unless you need to put to multiple regions, or need to use * different credentials for different streams, you should avoid creating * multiple instances of KinesisProducer.</li> * </ul> * <p> * * @author chaodeng * */ public class KinesisProducer { private static final Logger log = LoggerFactory.getLogger(KinesisProducer.class); private static final BigInteger UINT_128_MAX = new BigInteger(StringUtils.repeat("FF", 16), 16); private static final Object EXTRACT_BIN_MUTEX = new Object(); private static final AtomicInteger callbackCompletionPoolNumber = new AtomicInteger(0); private final KinesisProducerConfiguration config; private final Map<String, String> env; private final AtomicLong messageNumber = new AtomicLong(1); private final Map<Long, SettableFuture<?>> futures = new ConcurrentHashMap<>(); private final ExecutorService callbackCompletionExecutor = new ThreadPoolExecutor(1, Runtime.getRuntime().availableProcessors() * 4, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(), new ThreadFactoryBuilder().setDaemon(true) .setNameFormat( "kpl-callback-pool-" + callbackCompletionPoolNumber.getAndIncrement() + "-thread-%d") .build(), new RejectedExecutionHandler() { /** * Execute the runnable inline if we can't submit it to the * executor. This shouldn't happen since we're using a linked * queue which doesn't have a bound; but it's here just in case. */ @Override public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { r.run(); } }); private String pathToExecutable; private String pathToLibDir; private String pathToTmpDir; private volatile Daemon child; private volatile long lastChild = System.nanoTime(); private volatile boolean destroyed = false; private ProcessFailureBehavior processFailureBehavior = ProcessFailureBehavior.AutoRestart; private class MessageHandler implements Daemon.MessageHandler { @Override public void onMessage(final Message m) { callbackCompletionExecutor.submit(new Runnable() { @Override public void run() { if (m.hasPutRecordResult()) { onPutRecordResult(m); } else if (m.hasMetricsResponse()) { onMetricsResponse(m); } else { log.error("Unexpected message type from child process"); } } }); } @Override public void onError(final Throwable t) { // Don't log error if the user called destroy if (!destroyed) { log.error("Error in child process", t); } // Fail all outstanding futures for (final Map.Entry<Long, SettableFuture<?>> entry : futures.entrySet()) { callbackCompletionExecutor.submit(new Runnable() { @Override public void run() { entry.getValue().setException(t); } }); } futures.clear(); if (processFailureBehavior == ProcessFailureBehavior.AutoRestart && !destroyed) { log.info("Restarting native producer process."); child = new Daemon(pathToExecutable, new MessageHandler(), pathToTmpDir, config, env); } else { // Only restart child if it's not an irrecoverable error, and if // there has been some time (3 seconds) between the last child // creation. If the child process crashes almost immediately, we're // going to abort to avoid going into a loop. if (!(t instanceof IrrecoverableError) && System.nanoTime() - lastChild > 3e9) { lastChild = System.nanoTime(); child = new Daemon(pathToExecutable, new MessageHandler(), pathToTmpDir, config, env); } } } /** * Matches up the incoming PutRecordResult to an outstanding future, and * completes that future with the appropriate data. * * <p> * We adapt the protobuf PutRecordResult to another simpler * PutRecordResult class to hide the protobuf stuff from the user. * * @param msg */ private void onPutRecordResult(Message msg) { SettableFuture<UserRecordResult> f = getFuture(msg); UserRecordResult result = UserRecordResult.fromProtobufMessage(msg.getPutRecordResult()); if (result.isSuccessful()) { f.set(result); } else { f.setException(new UserRecordFailedException(result)); } } private void onMetricsResponse(Message msg) { SettableFuture<List<Metric>> f = getFuture(msg); List<Metric> userMetrics = new ArrayList<>(); MetricsResponse res = msg.getMetricsResponse(); for (Messages.Metric metric : res.getMetricsList()) { userMetrics.add(new Metric(metric)); } f.set(userMetrics); } private <T> SettableFuture<T> getFuture(Message msg) { long id = msg.getSourceId(); @SuppressWarnings("unchecked") SettableFuture<T> f = (SettableFuture<T>) futures.remove(id); if (f == null) { throw new RuntimeException("Future for message id " + id + " not found"); } return f; } } /** * Start up a KinesisProducer instance. * * <p> * Since this creates a child process, it is fairly expensive. Avoid * creating more than one per application, unless putting to multiple * regions at the same time. All streams in the same region can share the * same instance. * * <p> * All methods in KinesisProducer are thread-safe. * * @param config * Configuration for the KinesisProducer. See the docs for that * class for details. * * @see KinesisProducerConfiguration */ public KinesisProducer(KinesisProducerConfiguration config) { this.config = config; extractBinaries(); env = new ImmutableMap.Builder<String, String>().put("LD_LIBRARY_PATH", pathToLibDir) .put("DYLD_LIBRARY_PATH", pathToLibDir).put("CA_DIR", pathToTmpDir).build(); child = new Daemon(pathToExecutable, new MessageHandler(), pathToTmpDir, config, env); } /** * Start up a KinesisProducer instance. * * <p> * Since this creates a child process, it is fairly expensive. Avoid * creating more than one per application, unless putting to multiple * regions at the same time. All streams in the same region can share the * same instance. * * <p> * The KPL will use a set of default configurations. You can set custom * configuration using the constructor that takes a {@link KinesisProducerConfiguration} * object. * * <p> * All methods in KinesisProducer are thread-safe. */ public KinesisProducer() { this(new KinesisProducerConfiguration()); } /** * Connect to a running daemon. Does not start a child process. Used for * testing. * * @param inPipe * @param outPipe */ protected KinesisProducer(File inPipe, File outPipe) { this.config = null; this.env = null; child = new Daemon(inPipe, outPipe, new MessageHandler()); } /** * Put a record asynchronously. A {@link ListenableFuture} is returned that * can be used to retrieve the result, either by polling or by registering a * callback. * * <p> * The return value can be disregarded if you do not wish to process the * result. Under the covers, the KPL will automatically re-attempt puts in * case of transient errors (including throttling). A failed result is * generally returned only if an irrecoverable error is detected (e.g. * trying to put to a stream that doesn't exist), or if the record expires. * * <p> * <b>Thread safe.</b> * * <p> * To add a listener to the future: * <p> * <code> * ListenableFuture<PutRecordResult> f = myKinesisProducer.addUserRecord(...); * com.google.common.util.concurrent.Futures.addCallback(f, callback, executor); * </code> * <p> * where <code>callback</code> is an instance of * {@link com.google.common.util.concurrent.FutureCallback} and * <code>executor</code> is an instance of * {@link java.util.concurrent.Executor}. * <p> * <b>Important:</b> * <p> * If long-running tasks are performed in the callbacks, it is recommended * that a custom executor be provided when registering callbacks to ensure * that there are enough threads to achieve the desired level of * parallelism. By default, the KPL will use an internal thread pool to * execute callbacks, but this pool may not have a sufficient number of * threads if a large number is desired. * <p> * Another option would be to hand the result off to a different component * for processing and keep the callback routine fast. * * @param stream * Stream to put to. * @param partitionKey * Partition key. Length must be at least one, and at most 256 * (inclusive). * @param data * Binary data of the record. Maximum size 1MiB. * @return A future for the result of the put. * @throws IllegalArgumentException * if input does not meet stated constraints * @throws DaemonException * if the child process is dead * @see ListenableFuture * @see UserRecordResult * @see KinesisProducerConfiguration#setRecordTtl(long) * @see UserRecordFailedException */ public ListenableFuture<UserRecordResult> addUserRecord(String stream, String partitionKey, ByteBuffer data) { return addUserRecord(stream, partitionKey, null, data); } /** * Put a record asynchronously. A {@link ListenableFuture} is returned that * can be used to retrieve the result, either by polling or by registering a * callback. * * <p> * The return value can be disregarded if you do not wish to process the * result. Under the covers, the KPL will automatically reattempt puts in * case of transient errors (including throttling). A failed result is * generally returned only if an irrecoverable error is detected (e.g. * trying to put to a stream that doesn't exist), or if the record expires. * * <p> * <b>Thread safe.</b> * * <p> * To add a listener to the future: * <p> * <code> * ListenableFuture<PutRecordResult> f = myKinesisProducer.addUserRecord(...); * com.google.common.util.concurrent.Futures.addCallback(f, callback, executor); * </code> * <p> * where <code>callback</code> is an instance of * {@link com.google.common.util.concurrent.FutureCallback} and * <code>executor</code> is an instance of * {@link java.util.concurrent.Executor}. * <p> * <b>Important:</b> * <p> * If long-running tasks are performed in the callbacks, it is recommended * that a custom executor be provided when registering callbacks to ensure * that there are enough threads to achieve the desired level of * parallelism. By default, the KPL will use an internal thread pool to * execute callbacks, but this pool may not have a sufficient number of * threads if a large number is desired. * <p> * Another option would be to hand the result off to a different component * for processing and keep the callback routine fast. * * @param stream * Stream to put to. * @param partitionKey * Partition key. Length must be at least one, and at most 256 * (inclusive). * @param explicitHashKey * The hash value used to explicitly determine the shard the data * record is assigned to by overriding the partition key hash. * Must be a valid string representation of a positive integer * with value between 0 and <tt>2^128 - 1</tt> (inclusive). * @param data * Binary data of the record. Maximum size 1MiB. * @return A future for the result of the put. * @throws IllegalArgumentException * if input does not meet stated constraints * @throws DaemonException * if the child process is dead * @see ListenableFuture * @see UserRecordResult * @see KinesisProducerConfiguration#setRecordTtl(long) * @see UserRecordFailedException */ public ListenableFuture<UserRecordResult> addUserRecord(String stream, String partitionKey, String explicitHashKey, ByteBuffer data) { if (stream == null) { throw new IllegalArgumentException("Stream name cannot be null"); } stream = stream.trim(); if (stream.length() == 0) { throw new IllegalArgumentException("Stream name cannot be empty"); } if (partitionKey == null) { throw new IllegalArgumentException("partitionKey cannot be null"); } if (partitionKey.length() < 1 || partitionKey.length() > 256) { throw new IllegalArgumentException( "Invalid parition key. Length must be at least 1 and at most 256, got " + partitionKey.length()); } try { partitionKey.getBytes("UTF-8"); } catch (Exception e) { throw new IllegalArgumentException("Partition key must be valid UTF-8"); } BigInteger b = null; if (explicitHashKey != null) { explicitHashKey = explicitHashKey.trim(); try { b = new BigInteger(explicitHashKey); } catch (NumberFormatException e) { throw new IllegalArgumentException( "Invalid explicitHashKey, must be an integer, got " + explicitHashKey); } if (b != null) { if (b.compareTo(UINT_128_MAX) > 0 || b.compareTo(BigInteger.ZERO) < 0) { throw new IllegalArgumentException( "Invalid explicitHashKey, must be greater or equal to zero and less than or equal to (2^128 - 1), got " + explicitHashKey); } } } if (data != null && data.remaining() > 1024 * 1024) { throw new IllegalArgumentException( "Data must be less than or equal to 1MB in size, got " + data.remaining() + " bytes"); } long id = messageNumber.getAndIncrement(); SettableFuture<UserRecordResult> f = SettableFuture.create(); futures.put(id, f); PutRecord.Builder pr = PutRecord.newBuilder().setStreamName(stream).setPartitionKey(partitionKey) .setData(data != null ? ByteString.copyFrom(data) : ByteString.EMPTY); if (b != null) { pr.setExplicitHashKey(b.toString(10)); } Message m = Message.newBuilder().setId(id).setPutRecord(pr.build()).build(); child.add(m); return f; } /** * Get the number of unfinished records currently being processed. The * records could either be waiting to be sent to the child process, or have * reached the child process and are being worked on. * * <p> * This is equal to the number of futures returned from {@link #addUserRecord} * that have not finished. * * @return The number of unfinished records currently being processed. */ public int getOutstandingRecordsCount() { return futures.size(); } /** * Get metrics from the KPL. * * <p> * The KPL computes and buffers useful metrics. Use this method to retrieve * them. The same metrics are also uploaded to CloudWatch (unless disabled). * * <p> * Multiple metrics exist for the same name, each with a different list of * dimensions (e.g. stream name). This method will fetch all metrics with * the provided name. * * <p> * See the stand-alone metrics documentation for details about individual * metrics. * * <p> * This method is synchronous and will block while the data is being * retrieved. * * @param metricName * Name of the metrics to fetch. * @param windowSeconds * Fetch data from the last N seconds. The KPL maintains data at * per second granularity for the past minute. To get total * cumulative data since the start of the program, use the * overloads that do not take this argument. * @return A list of metrics with the provided name. * @throws ExecutionException * If an error occurred while fetching metrics from the child * process. * @throws InterruptedException * If the thread is interrupted while waiting for the response * from the child process. * @see Metric */ public List<Metric> getMetrics(String metricName, int windowSeconds) throws InterruptedException, ExecutionException { MetricsRequest.Builder mrb = MetricsRequest.newBuilder(); if (metricName != null) { mrb.setName(metricName); } if (windowSeconds > 0) { mrb.setSeconds(windowSeconds); } long id = messageNumber.getAndIncrement(); SettableFuture<List<Metric>> f = SettableFuture.create(); futures.put(id, f); child.add(Message.newBuilder().setId(id).setMetricsRequest(mrb.build()).build()); return f.get(); } /** * Get metrics from the KPL. * * <p> * The KPL computes and buffers useful metrics. Use this method to retrieve * them. The same metrics are also uploaded to CloudWatch (unless disabled). * * <p> * Multiple metrics exist for the same name, each with a different list of * dimensions (e.g. stream name). This method will fetch all metrics with * the provided name. * * <p> * The retrieved data represents cumulative statistics since the start of * the program. To get data from a smaller window, use * {@link #getMetrics(String, int)}. * * <p> * See the stand-alone metrics documentation for details about individual * metrics. * * <p> * This method is synchronous and will block while the data is being * retrieved. * * @param metricName * Name of the metrics to fetch. * @return A list of metrics with the provided name. * @throws ExecutionException * If an error occurred while fetching metrics from the child * process. * @throws InterruptedException * If the thread is interrupted while waiting for the response * from the child process. * @see Metric */ public List<Metric> getMetrics(String metricName) throws InterruptedException, ExecutionException { return getMetrics(metricName, -1); } /** * Get metrics from the KPL. * * <p> * The KPL computes and buffers useful metrics. Use this method to retrieve * them. The same metrics are also uploaded to CloudWatch (unless disabled). * * <p> * This method fetches all metrics available. To fetch only metrics with a * given name, use {@link #getMetrics(String)}. * * <p> * The retrieved data represents cumulative statistics since the start of * the program. To get data from a smaller window, use * {@link #getMetrics(int)}. * * <p> * See the stand-alone metrics documentation for details about individual * metrics. * * <p> * This method is synchronous and will block while the data is being * retrieved. * * @return A list of all of the metrics maintained by the KPL. * @throws ExecutionException * If an error occurred while fetching metrics from the child * process. * @throws InterruptedException * If the thread is interrupted while waiting for the response * from the child process. * @see Metric */ public List<Metric> getMetrics() throws InterruptedException, ExecutionException { return getMetrics(null); } /** * Get metrics from the KPL. * * <p> * The KPL computes and buffers useful metrics. Use this method to retrieve * them. The same metrics are also uploaded to CloudWatch (unless disabled). * * <p> * This method fetches all metrics available. To fetch only metrics with a * given name, use {@link #getMetrics(String, int)}. * * <p> * See the stand-alone metrics documentation for details about individual * metrics. * * <p> * This method is synchronous and will block while the data is being * retrieved. * * @param windowSeconds * Fetch data from the last N seconds. The KPL maintains data at * per second granularity for the past minute. To get total * cumulative data since the start of the program, use the * overloads that do not take this argument. * @return A list of metrics with the provided name. * @throws ExecutionException * If an error occurred while fetching metrics from the child * process. * @throws InterruptedException * If the thread is interrupted while waiting for the response * from the child process. * @see Metric */ public List<Metric> getMetrics(int windowSeconds) throws InterruptedException, ExecutionException { return getMetrics(null, windowSeconds); } /** * Immediately kill the child process. This will cause all outstanding * futures to fail immediately. * * <p> * To perform a graceful shutdown instead, there are several options: * * <ul> * <li>Call {@link #flush()} and wait (perhaps with a time limit) for all * futures. If you were sending a very high volume of data you may need to * call flush multiple times to clear all buffers.</li> * <li>Poll {@link #getOutstandingRecordsCount()} until it returns 0.</li> * <li>Call {@link #flushSync()}, which blocks until completion.</li> * </ul> * * Once all records are confirmed with one of the above, call destroy to * terminate the child process. If you are terminating the JVM then calling * destroy is unnecessary since it will be done automatically. */ public void destroy() { destroyed = true; child.destroy(); } /** * Instruct the child process to perform a flush, sending some of the * records it has buffered for the specified stream. * * <p> * This does not guarantee that all buffered records will be sent, only that * most of them will; to flush all records and wait for completion, use * {@link #flushSync}. * * <p> * This method returns immediately without blocking. * * @param stream * Stream to flush * @throws DaemonException * if the child process is dead */ public void flush(String stream) { Flush.Builder f = Flush.newBuilder(); if (stream != null) { f.setStreamName(stream); } Message m = Message.newBuilder().setId(messageNumber.getAndIncrement()).setFlush(f.build()).build(); child.add(m); } /** * Instruct the child process to perform a flush, sending some of the * records it has buffered. Applies to all streams. * * <p> * This does not guarantee that all buffered records will be sent, only that * most of them will; to flush all records and wait for completion, use * {@link #flushSync}. * * <p> * This method returns immediately without blocking. * * @throws DaemonException * if the child process is dead */ public void flush() { flush(null); } /** * Instructs the child process to flush all records and waits until all * records are complete (either succeeding or failing). * * <p> * The wait includes any retries that need to be performed. Depending on * your configuration of record TTL and request timeout, this can * potentially take a long time if the library is having trouble delivering * records to the backend, for example due to network problems. * * <p> * This is useful if you need to shutdown your application and want to make * sure all records are delivered before doing so. * * @throws DaemonException * if the child process is dead * * @see KinesisProducerConfiguration#setRecordTtl(long) * @see KinesisProducerConfiguration#setRequestTimeout(long) */ public void flushSync() { while (getOutstandingRecordsCount() > 0) { flush(); try { Thread.sleep(500); } catch (InterruptedException e) { } } } private void extractBinaries() { synchronized (EXTRACT_BIN_MUTEX) { final List<File> watchFiles = new ArrayList<>(2); String os = SystemUtils.OS_NAME; if (SystemUtils.IS_OS_WINDOWS) { os = "windows"; } else if (SystemUtils.IS_OS_LINUX) { os = "linux"; } else if (SystemUtils.IS_OS_MAC_OSX) { os = "osx"; } else { throw new RuntimeException("Your operation system is not supported (" + os + "), the KPL only supports Linux, OSX and Windows"); } String root = "amazon-kinesis-producer-native-binaries"; String tmpDir = config.getTempDirectory(); if (tmpDir.trim().length() == 0) { tmpDir = System.getProperty("java.io.tmpdir"); } tmpDir = Paths.get(tmpDir, root).toString(); pathToTmpDir = tmpDir; String binPath = config.getNativeExecutable(); if (binPath != null && !binPath.trim().isEmpty()) { pathToExecutable = binPath.trim(); log.warn("Using non-default native binary at " + pathToExecutable); pathToLibDir = ""; } else { log.info("Extracting binaries to " + tmpDir); try { File tmpDirFile = new File(tmpDir); if (!tmpDirFile.exists() && !tmpDirFile.mkdirs()) { throw new IOException("Could not create tmp dir " + tmpDir); } String extension = os.equals("windows") ? ".exe" : ""; String executableName = "kinesis_producer" + extension; byte[] bin = IOUtils.toByteArray(this.getClass().getClassLoader() .getResourceAsStream(root + "/" + os + "/" + executableName)); MessageDigest md = MessageDigest.getInstance("SHA1"); String mdHex = DatatypeConverter.printHexBinary(md.digest(bin)).toLowerCase(); pathToExecutable = Paths.get(pathToTmpDir, "kinesis_producer_" + mdHex + extension).toString(); File extracted = new File(pathToExecutable); watchFiles.add(extracted); if (extracted.exists()) { try (FileInputStream fis = new FileInputStream(extracted); FileLock lock = fis.getChannel().lock(0, Long.MAX_VALUE, true)) { boolean contentEqual = false; if (extracted.length() == bin.length) { byte[] existingBin = IOUtils.toByteArray(new FileInputStream(extracted)); contentEqual = Arrays.equals(bin, existingBin); } if (!contentEqual) { throw new SecurityException("The contents of the binary " + extracted.getAbsolutePath() + " is not what it's expected to be."); } } } else { try (FileOutputStream fos = new FileOutputStream(extracted); FileLock lock = fos.getChannel().lock()) { IOUtils.write(bin, fos); } extracted.setExecutable(true); } String certFileName = "b204d74a.0"; File certFile = new File(pathToTmpDir, certFileName); if (!certFile.exists()) { try (FileOutputStream fos = new FileOutputStream(certFile); FileLock lock = fos.getChannel().lock()) { byte[] certs = IOUtils.toByteArray(this.getClass().getClassLoader() .getResourceAsStream("cacerts/" + certFileName)); IOUtils.write(certs, fos); } } watchFiles.add(certFile); pathToLibDir = pathToTmpDir; FileAgeManager.instance().registerFiles(watchFiles); } catch (Exception e) { throw new RuntimeException("Could not copy native binaries to temp directory " + tmpDir, e); } } } } }