Java tutorial
/** * Copyright (C) 2015 Greg Brandt (brandt.greg@gmail.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.brandtg.switchboard; import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import static java.nio.file.StandardWatchEventKinds.OVERFLOW; import io.dropwizard.lifecycle.Managed; import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream; import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.nio.file.FileSystems; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; public class HdfsLogIndexer implements Managed { private static final Logger LOG = LoggerFactory.getLogger(HdfsLogIndexer.class); private enum EditLogType { IMMUTABLE, IN_PROGRESS } private final HdfsLogServerConfig config; private final LogIndex logIndex; private final File rootDir; private final ExecutorService executorService; private final AtomicBoolean isShutdown; private final ConcurrentMap<WatchKey, Path> keys; private WatchService watchService; /** * Creates an agent to index HDFS edit log segments. * * @param config * Server configuration * @param logIndex * Log index in which to write the log segment indices */ public HdfsLogIndexer(HdfsLogServerConfig config, LogIndex logIndex) { this.config = config; this.logIndex = logIndex; this.rootDir = new File(config.getRootDir()); this.executorService = Executors.newSingleThreadExecutor(); this.isShutdown = new AtomicBoolean(true); this.keys = new ConcurrentHashMap<WatchKey, Path>(); } @Override public void start() throws Exception { if (isShutdown.getAndSet(false)) { watchService = FileSystems.getDefault().newWatchService(); // TODO: Only index segments less than high water mark in index // TODO: If inprogress rolled over while server was down, rename log entries w/ new immutable segment name indexSegments(EditLogType.IMMUTABLE); LOG.info("Indexed immutable log segments"); indexSegments(EditLogType.IN_PROGRESS); LOG.info("Indexed in-progress log segments"); Path dir = Paths.get(rootDir.getAbsolutePath()); WatchKey key = dir.register(watchService, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY, OVERFLOW); keys.put(key, dir); LOG.info("Registered watch key on {}", dir); executorService.submit(new Watcher()); LOG.info("Started file system watcher"); } } @Override public void stop() throws Exception { if (!isShutdown.getAndSet(true)) { executorService.shutdown(); watchService.close(); LOG.info("Stopped HDFS log indexer"); } } private void indexSegments(final EditLogType editLogType) throws IOException { File[] files = rootDir.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { if (EditLogType.IN_PROGRESS.equals(editLogType)) { return name.startsWith("edits_inprogress"); } else { return !name.startsWith("edits_inprogress") && name.startsWith("edits_"); } } }); if (files != null) { for (File file : files) { EditLogInputStream editLog; if (EditLogType.IN_PROGRESS.equals(editLogType)) { long startTxid = HdfsLogUtils.getInProgressLowWatermarkTxid(file.getName()); long endTxid = -12345; editLog = new EditLogFileInputStream(file, startTxid, endTxid, true); } else { long startTxid = HdfsLogUtils.getLogHighWatermarkTxid(file.getName()); long endTxid = HdfsLogUtils.getLogHighWatermarkTxid(file.getName()); editLog = new EditLogFileInputStream(file, startTxid, endTxid, false); } FSEditLogOp op = null; long position = 0; while ((op = editLog.readOp()) != null) { LogRegion logRegion = new LogRegion(op.getTransactionId(), file.getAbsolutePath(), position, editLog.getPosition()); position = editLog.getPosition(); logIndex.putLogRegion("*", logRegion); } LOG.info("Scanned {}", file); } } } private class Watcher implements Runnable { @Override @SuppressWarnings("unchecked") public void run() { while (!isShutdown.get()) { WatchKey key = null; try { // Get next event try { key = watchService.take(); } catch (InterruptedException e) { LOG.warn("Could not take next key", e); continue; } // Are we watching? Path dir = keys.get(key); if (dir == null) { LOG.warn("Watch key not recognized {}", key); key.reset(); continue; } for (WatchEvent<?> event : key.pollEvents()) { if (event.kind() == OVERFLOW) { LOG.warn("Received an OVERFLOW event"); continue; } WatchEvent<Path> pathEvent = (WatchEvent<Path>) event; Path path = dir.resolve(pathEvent.context()); File file = path.toFile(); LOG.info("{} :: {}", pathEvent.kind(), path); if (file.getName().startsWith("edits_inprogress_")) { if (pathEvent.kind() == ENTRY_DELETE) { // Rename existing log index entries long txid = HdfsLogUtils.getInProgressLowWatermarkTxid(file.getName()); final String editLogPrefix = "edits_" + HdfsLogUtils.getLogFileTxidComponent(txid); File[] files = rootDir.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.startsWith(editLogPrefix); } }); if (files == null || files.length != 1) { throw new IllegalStateException( "No rolled edit log with prefix " + editLogPrefix); } logIndex.rename(file.getAbsolutePath(), files[0].getAbsolutePath()); LOG.info("Renamed file {} to {} in index", file.getAbsolutePath(), files[0].getAbsolutePath()); } else { // ENTRY_CREATE || ENTRY_MODIFY LogUtils.waitForWrites(file, config.getWaitSleepMillis(), config.getWaitTimeoutMillis()); indexSegments(EditLogType.IN_PROGRESS); } } } } catch (Exception e) { LOG.error("Exception during watch event key={}", key, e); } if (key != null) { boolean valid = key.reset(); if (!valid) { keys.remove(key); if (keys.isEmpty()) { break; } } } } } } }