Java tutorial
/* * Copyright 2014 CyberVision, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.kaaproject.kaa.server.flume.sink.hdfs; import com.google.common.base.Preconditions; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.cache.RemovalCause; import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.flume.Channel; import org.apache.flume.Context; import org.apache.flume.Event; import org.apache.flume.EventDeliveryException; import org.apache.flume.Transaction; import org.apache.flume.conf.Configurable; import org.apache.flume.instrumentation.SinkCounter; import org.apache.flume.sink.AbstractSink; import org.apache.flume.sink.hdfs.HDFSDataStream; import org.apache.flume.sink.hdfs.HDFSWriter; import org.apache.flume.sink.hdfs.KerberosUser; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.kaaproject.kaa.server.flume.ConfigurationConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; public class KaaHdfsSink extends AbstractSink implements Configurable, ConfigurationConstants, RemovalListener<HdfsSinkKey, BucketWriter> { private static final Logger LOG = LoggerFactory.getLogger(KaaHdfsSink.class); /** * Singleton credential manager that manages static credentials for the * entire JVM. */ private static final AtomicReference<KerberosUser> staticLogin = new AtomicReference<>(); //NOSONAR private boolean started = false; private KaaEventFactory eventFactory; private Context context; private SinkCounter sinkCounter; //writers private LoadingCache<HdfsSinkKey, BucketWriter> writerCache; private BucketWriterLoader bucketWriterLoader; private Map<HdfsSinkKey, BucketWriter> writerFlushMap; private ExecutorService callTimeoutPool; private ScheduledThreadPoolExecutor timedRollerPool; private ScheduledExecutorService statisticsPool; private volatile ScheduledFuture<?> statisticsFuture; private long cacheCleanupStartInterval; private String kerbConfPrincipal; private String kerbKeytab; private String proxyUserName; private UserGroupInformation proxyTicket; //configurable part private String rootHdfsPath; private long txnEventMax; // writers configuration private long callTimeout; private int threadsPoolSize; private int rollTimerPoolSize; private int maxOpenFiles; private long cacheCleanupInterval; private int writerExpirationInterval; private long rollInterval; private long rollSize; private long rollCount; private long batchSize; private long defaultBlockSize; private String filePrefix; private long statisticsInterval; public KaaHdfsSink() { } /** * Static synchronized method for static Kerberos login. <br/> * Static synchronized due to a thundering herd problem when multiple Sinks * attempt to log in using the same principal at the same time with the * intention of impersonating different users (or even the same user). * If this is not controlled, MIT Kerberos v5 believes it is seeing a replay * attach and it returns: * <blockquote>Request is a replay (34) - PROCESS_TGS</blockquote> * In addition, since the underlying Hadoop APIs we are using for * impersonation are static, we define this method as static as well. * * @param principal Fully-qualified principal to use for authentication. * @param keytab Location of keytab file containing credentials for principal. * @return Logged-in user * @throws IOException if login fails. */ private static synchronized UserGroupInformation kerberosLogin(KaaHdfsSink sink, String principal, String keytab) throws IOException { // if we are the 2nd user thru the lock, the login should already be // available statically if login was successful UserGroupInformation curUser = null; try { curUser = UserGroupInformation.getLoginUser(); } catch (IOException ex) { // not a big deal but this shouldn't typically happen because it will // generally fall back to the UNIX user LOG.debug("Unable to get login user before Kerberos auth attempt.", ex); } // we already have logged in successfully if (curUser != null && curUser.getUserName().equals(principal)) { LOG.debug("{}: Using existing principal ({}): {}", new Object[] { sink, principal, curUser }); // no principal found } else { LOG.info("{}: Attempting kerberos login as principal ({}) from keytab file ({})", new Object[] { sink, principal, keytab }); // attempt static kerberos login UserGroupInformation.loginUserFromKeytab(principal, keytab); curUser = UserGroupInformation.getLoginUser(); } return curUser; } @Override public void configure(Context context) { this.context = context; rootHdfsPath = context.getString(CONFIG_ROOT_HDFS_PATH, DEFAULT_ROOT_HDFS_PATH); Preconditions.checkNotNull(rootHdfsPath, "rootHdfsPath is required"); txnEventMax = context.getLong(CONFIG_HDFS_TXN_EVENT_MAX, DEFAULT_HDFS_TXN_EVENT_MAX); statisticsInterval = context.getLong(CONFIG_STATISTICS_INTERVAL, DEFAULT_STATISTICS_INTERVAL); // writers threadsPoolSize = context.getInteger(CONFIG_HDFS_THREAD_POOL_SIZE, DEFAULT_HDFS_THREAD_POOL_SIZE); rollTimerPoolSize = context.getInteger(CONFIG_HDFS_ROLL_TIMER_POOL_SIZE, DEFAULT_HDFS_ROLL_TIMER_POOL_SIZE); maxOpenFiles = context.getInteger(CONFIG_HDFS_MAX_OPEN_FILES, DEFAULT_HDFS_MAX_OPEN_FILES); cacheCleanupInterval = context.getInteger(CONFIG_HDFS_CACHE_CLEANUP_INTERVAL, DEFAULT_HDFS_CACHE_CLEANUP_INTERVAL) * 1000; writerExpirationInterval = context.getInteger(CONFIG_HDFS_WRITER_EXPIRATION_INTERVAL, DEFAULT_HDFS_WRITER_EXPIRATION_INTERVAL); callTimeout = context.getLong(CONFIG_HDFS_CALL_TIMEOUT, DEFAULT_HDFS_CALL_TIMEOUT); rollInterval = context.getLong(CONFIG_HDFS_ROLL_INTERVAL, DEFAULT_HDFS_ROLL_INTERVAL); rollSize = context.getLong(CONFIG_HDFS_ROLL_SIZE, DEFAULT_HDFS_ROLL_SIZE); rollCount = context.getLong(CONFIG_HDFS_ROLL_COUNT, DEFAULT_HDFS_ROLL_COUNT); batchSize = context.getLong(CONFIG_HDFS_BATCH_SIZE, DEFAULT_HDFS_BATCH_SIZE); defaultBlockSize = context.getLong(CONFIG_HDFS_DEFAULT_BLOCK_SIZE, DEFAULT_HDFS_DEFAULT_BLOCK_SIZE); filePrefix = context.getString(CONFIG_HDFS_FILE_PREFIX, DEFAULT_HDFS_FILE_PREFIX); Preconditions.checkArgument(batchSize > 0, "batchSize must be greater than 0"); Preconditions.checkArgument(txnEventMax > 0, "txnEventMax must be greater than 0"); kerbConfPrincipal = context.getString(CONFIG_HDFS_KERBEROS_PRINCIPAL, ""); kerbKeytab = context.getString(CONFIG_HDFS_KERBEROS_KEYTAB, ""); proxyUserName = context.getString(CONFIG_HDFS_PROXY_USER, ""); if (!authenticate()) { LOG.error("Failed to authenticate!"); } if (sinkCounter == null) { sinkCounter = new SinkCounter(getName()); } } /** * Execute the callable on a separate thread and wait for the completion * for the specified amount of time in milliseconds. In case of timeout * cancel the callable and throw an IOException */ private <T> T callWithTimeout(Callable<T> callable) throws IOException, InterruptedException { Future<T> future = callTimeoutPool.submit(callable); try { if (callTimeout > 0) { return future.get(callTimeout, TimeUnit.MILLISECONDS); } else { return future.get(); } } catch (TimeoutException ex) { future.cancel(true); sinkCounter.incrementConnectionFailedCount(); throw new IOException("Callable timed out after " + callTimeout + " ms", ex); } catch (ExecutionException e1) { sinkCounter.incrementConnectionFailedCount(); Throwable cause = e1.getCause(); if (cause instanceof IOException) { throw (IOException) cause; } else if (cause instanceof InterruptedException) { throw (InterruptedException) cause; } else if (cause instanceof RuntimeException) { throw (RuntimeException) cause; } else if (cause instanceof Error) { throw (Error) cause; } else { throw new RuntimeException(e1); } } catch (CancellationException ce) { LOG.error("Exception catched: ", ce); throw new InterruptedException("Blocked callable interrupted by rotation event"); } catch (InterruptedException ex) { LOG.warn("Unexpected Exception " + ex.getMessage(), ex); throw ex; } } @Override public Status process() throws EventDeliveryException { Channel channel = getChannel(); Transaction transaction = channel.getTransaction(); transaction.begin(); try { Event event = null; int txnEventCount = 0; int sinkEventCount = 0; for (txnEventCount = 0; txnEventCount < txnEventMax; txnEventCount++) { event = channel.take(); if (event == null) { if ((System.currentTimeMillis() - cacheCleanupStartInterval) >= cacheCleanupInterval) { LOG.info("Starting Writer cache cleanup."); writerCache.cleanUp(); timedRollerPool.purge(); cacheCleanupStartInterval = System.currentTimeMillis(); } break; } //else{ // cacheCleanupStartInterval = System.currentTimeMillis(); //} Map<KaaSinkKey, List<KaaRecordEvent>> incomingEventsMap = eventFactory .processIncomingFlumeEvent(event); if (incomingEventsMap == null || incomingEventsMap.isEmpty()) { if (LOG.isWarnEnabled()) { LOG.warn("Unable to parse incoming event: " + event); } continue; } for (KaaSinkKey key : incomingEventsMap.keySet()) { HdfsSinkKey hdfsSinkKey = new HdfsSinkKey(rootHdfsPath, key); BucketWriter bucketWriter; // track the buckets getting written in this transaction bucketWriter = writerCache.get(hdfsSinkKey); writerFlushMap.put(hdfsSinkKey, bucketWriter); // Write the data to HDFS List<KaaRecordEvent> events = incomingEventsMap.get(key); sinkEventCount += events.size(); appendBatch(bucketWriter, events); } } if (txnEventCount == 0) { sinkCounter.incrementBatchEmptyCount(); } else if (txnEventCount == txnEventMax) { sinkCounter.incrementBatchCompleteCount(); } else { sinkCounter.incrementBatchUnderflowCount(); } // flush all pending buckets before committing the transaction for (BucketWriter bucketWriter : writerFlushMap.values()) { flush(bucketWriter); } writerFlushMap.clear(); transaction.commit(); if (sinkEventCount > 0) { sinkCounter.addToEventDrainSuccessCount(sinkEventCount); } if (event == null) { return Status.BACKOFF; } return Status.READY; } catch (IOException ex) { transaction.rollback(); LOG.warn("HDFS IO error", ex); return Status.BACKOFF; } catch (Throwable th) { //NOSONAR transaction.rollback(); LOG.error("process failed", th); if (th instanceof Error) { throw (Error) th; } else { throw new EventDeliveryException(th); } } finally { transaction.close(); } } @Override public void start() { LOG.info("Starting {}...", this); eventFactory = new KaaEventFactory(); String timeoutName = "hdfs-" + getName() + "-call-runner-%d"; callTimeoutPool = Executors.newFixedThreadPool(threadsPoolSize, new ThreadFactoryBuilder().setNameFormat(timeoutName).build()); String rollerName = "hdfs-" + getName() + "-roll-timer-%d"; timedRollerPool = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(rollTimerPoolSize, new ThreadFactoryBuilder().setNameFormat(rollerName).build()); if (statisticsInterval > 0) { String statisticsName = "hdfs-" + getName() + "-statistics-%d"; statisticsPool = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat(statisticsName).build()); Runnable action = new Runnable() { @Override public void run() { LOG.info("Statistics: Drain attempt events: " + sinkCounter.getEventDrainAttemptCount() + "; " + "Drain success events: " + sinkCounter.getEventDrainSuccessCount()); } }; statisticsFuture = statisticsPool.scheduleWithFixedDelay(action, 0, statisticsInterval, TimeUnit.SECONDS); } cacheCleanupStartInterval = System.currentTimeMillis(); bucketWriterLoader = new BucketWriterLoader(rollInterval, rollSize, rollCount, batchSize, defaultBlockSize, context, filePrefix, timedRollerPool, proxyTicket, sinkCounter); writerCache = CacheBuilder.newBuilder().maximumSize(maxOpenFiles) .expireAfterWrite(writerExpirationInterval, TimeUnit.SECONDS).removalListener(this) .build(bucketWriterLoader); writerFlushMap = new HashMap<>(); sinkCounter.start(); started = true; super.start(); LOG.info("Kaa Hdfs Sink {} started.", getName()); } @Override public void stop() { started = false; // do not constrain close() calls with a timeout Map<HdfsSinkKey, BucketWriter> writers = writerCache.asMap(); for (Entry<HdfsSinkKey, BucketWriter> entry : writers.entrySet()) { LOG.info("Closing {}", entry.getKey()); try { close(entry.getValue()); } catch (Exception ex) { LOG.warn("Exception while closing " + entry.getKey() + ". " + "Exception follows.", ex); if (ex instanceof InterruptedException) { Thread.currentThread().interrupt(); } } } if (statisticsFuture != null && !statisticsFuture.isDone()) { statisticsFuture.cancel(false); // do not cancel myself if running! statisticsFuture = null; } // shut down all our thread pools ExecutorService[] toShutdown = { callTimeoutPool, timedRollerPool, statisticsPool }; for (ExecutorService execService : toShutdown) { if (execService != null) { execService.shutdown(); try { while (!execService.isTerminated()) { execService.awaitTermination( Math.max(ConfigurationConstants.DEFAULT_HDFS_CALL_TIMEOUT, callTimeout), TimeUnit.MILLISECONDS); } } catch (InterruptedException ex) { LOG.warn("shutdown interrupted on " + execService, ex); } } } callTimeoutPool = null; timedRollerPool = null; statisticsPool = null; writerCache.cleanUp(); writerCache = null; sinkCounter.stop(); super.stop(); } @Override public String toString() { return "{ Sink type:" + getClass().getSimpleName() + ", name:" + getName() + " }"; } public long getEventDrainSuccessCount() { return sinkCounter.getEventDrainSuccessCount(); } @Override public void onRemoval(RemovalNotification<HdfsSinkKey, BucketWriter> entry) { if (started) { RemovalCause cause = entry.getCause(); HdfsSinkKey key = entry.getKey(); BucketWriter writer = entry.getValue(); LOG.info("Stopping removed writer because of " + cause + " for key: " + entry.getKey()); try { writerFlushMap.remove(key); writer.close(); } catch (IOException ex) { LOG.warn(entry.getKey().toString(), ex); } catch (InterruptedException ex) { LOG.warn(entry.getKey().toString(), ex); Thread.currentThread().interrupt(); } } } private boolean authenticate() { // logic for kerberos login boolean useSecurity = UserGroupInformation.isSecurityEnabled(); LOG.info("Hadoop Security enabled: " + useSecurity); if (useSecurity) { // sanity checking if (kerbConfPrincipal.isEmpty()) { LOG.error("Hadoop running in secure mode, but Flume config doesn't " + "specify a principal to use for Kerberos auth."); return false; } if (kerbKeytab.isEmpty()) { LOG.error("Hadoop running in secure mode, but Flume config doesn't " + "specify a keytab to use for Kerberos auth."); return false; } else { //If keytab is specified, user should want it take effect. //HDFSSink will halt when keytab file is non-exist or unreadable File kfile = new File(kerbKeytab); if (!(kfile.isFile() && kfile.canRead())) { throw new IllegalArgumentException( "The keyTab file: " + kerbKeytab + " is nonexistent or can't read. " + "Please specify a readable keytab file for Kerberos auth."); } } String principal; try { // resolves _HOST pattern using standard Hadoop search/replace // via DNS lookup when 2nd argument is empty principal = SecurityUtil.getServerPrincipal(kerbConfPrincipal, ""); } catch (IOException ex) { LOG.error("Host lookup error resolving kerberos principal (" + kerbConfPrincipal + "). Exception follows.", ex); return false; } Preconditions.checkNotNull(principal, "Principal must not be null"); KerberosUser prevUser = staticLogin.get(); KerberosUser newUser = new KerberosUser(principal, kerbKeytab); // be cruel and unusual when user tries to login as multiple principals // this isn't really valid with a reconfigure but this should be rare // enough to warrant a restart of the agent JVM // TODO: find a way to interrogate the entire current config state, // since we don't have to be unnecessarily protective if they switch all // HDFS sinks to use a different principal all at once. Preconditions.checkState(prevUser == null || prevUser.equals(newUser), "Cannot use multiple kerberos principals in the same agent. " + " Must restart agent to use new principal or keytab. " + "Previous = %s, New = %s", prevUser, newUser); // attempt to use cached credential if the user is the same // this is polite and should avoid flooding the KDC with auth requests UserGroupInformation curUser = null; if (prevUser != null && prevUser.equals(newUser)) { try { curUser = UserGroupInformation.getLoginUser(); } catch (IOException ex) { LOG.warn("User unexpectedly had no active login. Continuing with " + "authentication", ex); } } if (curUser == null || !curUser.getUserName().equals(principal)) { try { // static login kerberosLogin(this, principal, kerbKeytab); } catch (IOException ex) { LOG.error("Authentication or file read error while attempting to " + "login as kerberos principal (" + principal + ") using " + "keytab (" + kerbKeytab + "). Exception follows.", ex); return false; } } else { LOG.debug("{}: Using existing principal login: {}", this, curUser); } // we supposedly got through this unscathed... so store the static user staticLogin.set(newUser); } // hadoop impersonation works with or without kerberos security proxyTicket = null; if (!proxyUserName.isEmpty()) { try { proxyTicket = UserGroupInformation.createProxyUser(proxyUserName, UserGroupInformation.getLoginUser()); } catch (IOException ex) { LOG.error("Unable to login as proxy user. Exception follows.", ex); return false; } } UserGroupInformation ugi = null; if (proxyTicket != null) { ugi = proxyTicket; } else if (useSecurity) { try { ugi = UserGroupInformation.getLoginUser(); } catch (IOException ex) { LOG.error("Unexpected error: Unable to get authenticated user after " + "apparent successful login! Exception follows.", ex); return false; } } if (ugi != null) { // dump login information AuthenticationMethod authMethod = ugi.getAuthenticationMethod(); LOG.info("Auth method: {}", authMethod); LOG.info(" User name: {}", ugi.getUserName()); LOG.info(" Using keytab: {}", ugi.isFromKeytab()); if (authMethod == AuthenticationMethod.PROXY) { UserGroupInformation superUser; try { superUser = UserGroupInformation.getLoginUser(); LOG.info(" Superuser auth: {}", superUser.getAuthenticationMethod()); LOG.info(" Superuser name: {}", superUser.getUserName()); LOG.info(" Superuser using keytab: {}", superUser.isFromKeytab()); } catch (IOException ex) { LOG.error("Unexpected error: unknown superuser impersonating proxy.", ex); return false; } } LOG.info("Logged in as user {}", ugi.getUserName()); return true; } return true; } /** * Append to bucket writer with timeout enforced. */ private void appendBatch(final BucketWriter bucketWriter, final List<KaaRecordEvent> events) throws IOException, InterruptedException { // Write the data to HDFS callWithTimeout(new Callable<Void>() { @Override public Void call() throws Exception { bucketWriter.appendBatch(events); return null; } }); } /** * Flush bucket writer with timeout enforced. */ private void flush(final BucketWriter bucketWriter) throws IOException, InterruptedException { callWithTimeout(new Callable<Void>() { @Override public Void call() throws Exception { bucketWriter.flush(); return null; } }); } /** * Close bucket writer with timeout enforced. */ private void close(final BucketWriter bucketWriter) throws IOException, InterruptedException { callWithTimeout(new Callable<Void>() { @Override public Void call() throws Exception { bucketWriter.close(); return null; } }); } static class BucketWriterLoader extends CacheLoader<HdfsSinkKey, BucketWriter> { private final long rollInterval; private final long rollSize; private final long rollCount; private final long batchSize; private final long defaultBlockSize; private final Context context; private final String filePrefix; private final ScheduledThreadPoolExecutor timedRollerPool; private final UserGroupInformation proxyTicket; private final SinkCounter sinkCounter; public BucketWriterLoader(long rollInterval, long rollSize, long rollCount, long batchSize, long defaultBlockSize, Context context, String filePrefix, ScheduledThreadPoolExecutor timedRollerPool, UserGroupInformation proxyTicket, SinkCounter sinkCounter) { this.rollInterval = rollInterval; this.rollSize = rollSize; this.rollCount = rollCount; this.batchSize = batchSize; this.defaultBlockSize = defaultBlockSize; this.context = context; this.filePrefix = filePrefix; this.timedRollerPool = timedRollerPool; this.proxyTicket = proxyTicket; this.sinkCounter = sinkCounter; } @Override public BucketWriter load(HdfsSinkKey key) throws Exception { HDFSWriter hdfsWriter = new HDFSDataStream(); String path = key.getPath() + Path.SEPARATOR + filePrefix; context.put("serializer", AvroKaaEventSerializer.Builder.class.getName()); LOG.info("Creating new writer for key: " + key); return new BucketWriter(rollInterval, rollSize, rollCount, batchSize, defaultBlockSize, context, path, hdfsWriter, timedRollerPool, proxyTicket, sinkCounter); } } }