Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.pinterest.terrapin.tools; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.pinterest.terrapin.Constants; import com.pinterest.terrapin.TerrapinUtil; import com.pinterest.terrapin.zookeeper.ClusterInfo; import com.pinterest.terrapin.zookeeper.FileSetInfo; import com.pinterest.terrapin.zookeeper.ViewInfo; import com.pinterest.terrapin.zookeeper.ZooKeeperManager; import com.twitter.common.zookeeper.ZooKeeperClient; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.lang3.tuple.Pair; import org.apache.helix.HelixAdmin; import org.apache.helix.manager.zk.ZKHelixAdmin; import org.apache.helix.model.ExternalView; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.InputStream; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.List; import java.util.Map; import java.util.Scanner; /** * Admin tool for running administrative actions against a terrapin * cluster. */ public class TerrapinAdmin { private static final Logger LOG = LoggerFactory.getLogger(TerrapinAdmin.class); private final PropertiesConfiguration configuration; private final String zkQuorum; private final ZooKeeperManager zkManager; public TerrapinAdmin(PropertiesConfiguration configuration) throws Exception { this.configuration = configuration; String zkQuorum = TerrapinUtil.getZKQuorumFromConf(configuration); ZooKeeperClient zkClient = TerrapinUtil.getZooKeeperClient(zkQuorum, 30); ZooKeeperManager zkManager = new ZooKeeperManager(zkClient, configuration.getString(Constants.HELIX_CLUSTER)); zkManager.registerWatchAllFileSets(); this.zkQuorum = zkQuorum; this.zkManager = zkManager; } // Set the namenode address for the terrapin cluster. public void setNamenode(String[] args) throws Exception { ClusterInfo clusterInfo = zkManager.getClusterInfo(); if (clusterInfo == null) { clusterInfo = new ClusterInfo("", Constants.DEFAULT_HDFS_REPLICATION); } clusterInfo.hdfsNameNode = args[1]; zkManager.setClusterInfo(clusterInfo); } // Override the HDFS replication number. public void setDfsReplication(String[] args) throws Exception { ClusterInfo clusterInfo = zkManager.getClusterInfo(); if (clusterInfo == null) { clusterInfo = new ClusterInfo("", Constants.DEFAULT_HDFS_REPLICATION); } clusterInfo.hdfsReplicationFactor = Integer.parseInt(args[1]); zkManager.setClusterInfo(clusterInfo); } // Check the cluster health. public void checkClusterHealth(String[] args) throws Exception { this.zkManager.registerWatchAllFileSets(); Thread.sleep(10000); int totalMissing = 0, totalPartitions = 0; List<String> fileSetsNotFullyServing = Lists.newArrayList(); List<String> fileSetsWithInconsistentView = Lists.newArrayList(); Map<String, Pair<FileSetInfo, FileSetInfo>> fileSetInfoMap = this.zkManager.getCandidateHdfsDirMap(); HelixAdmin helixAdmin = new ZKHelixAdmin(zkQuorum); for (Map.Entry<String, Pair<FileSetInfo, FileSetInfo>> entry : fileSetInfoMap.entrySet()) { String fileSet = "\"" + entry.getKey() + "\""; LOG.info(""); LOG.info("Checking file set " + fileSet); if (entry.getValue().getRight() != null) { LOG.info(fileSet + " is locked (either UPLOADING or a maintenance operation)."); } FileSetInfo fileSetInfo = entry.getValue().getLeft(); if (fileSetInfo == null) { LOG.info(fileSet + " is not serving."); } else { int numPartitions = fileSetInfo.servingInfo.numPartitions; totalPartitions += numPartitions; ExternalView externalView = helixAdmin.getResourceExternalView( configuration.getString(Constants.HELIX_CLUSTER), fileSetInfo.servingInfo.helixResource); int numServing = 0; for (String partition : externalView.getPartitionSet()) { Map<String, String> stateMap = externalView.getStateMap(partition); if (stateMap == null) { continue; } for (Map.Entry<String, String> stateEntry : stateMap.entrySet()) { if (stateEntry.getValue().equals("ONLINE")) { numServing++; break; } } } LOG.info(fileSet + " serving at " + numServing + "/" + numPartitions); totalMissing += (numPartitions - numServing); if (numServing < numPartitions) { fileSetsNotFullyServing.add(fileSet); } ViewInfo viewInfo = new ViewInfo(externalView); ViewInfo viewInfoInZk = zkManager.getViewInfo(fileSetInfo.servingInfo.helixResource); if (viewInfoInZk == null || !viewInfoInZk.equals(viewInfo)) { LOG.info("Compressed view inconsistent for " + fileSet); fileSetsWithInconsistentView.add(fileSet); } else { LOG.info("Compressed view consistent with helix external view."); } } } if (totalMissing > 0) { LOG.info(""); LOG.info("TOTAL MISSING PARTITIONS " + totalMissing + " out of " + totalPartitions); LOG.info("Unhealthy filesets."); for (String fileSet : fileSetsNotFullyServing) { LOG.info(fileSet); } } else { LOG.info(""); LOG.info("ALL SHARDS SERVING :)"); } if (!fileSetsWithInconsistentView.isEmpty()) { LOG.info(""); LOG.info("Filesets with inconsistent compressed/external view."); for (String fileSet : fileSetsWithInconsistentView) { LOG.info(fileSet); } } } @VisibleForTesting protected static int selectFileSetRollbackVersion(FileSetInfo fileSetInfo, InputStream inputStream) { int size = fileSetInfo.oldServingInfoList.size(); if (fileSetInfo.numVersionsToKeep < 2 || fileSetInfo.oldServingInfoList.size() < 1) { throw new IllegalArgumentException("no available version for rollback"); } System.out.println("available versions for rollback:"); for (int i = 0; i < size; i++) { FileSetInfo.ServingInfo servingInfo = fileSetInfo.oldServingInfoList.get(i); System.out.println(String.format("[%d] %s", i, servingInfo.hdfsPath)); } if (size > 1) { System.out.print(String.format("choose a version [0-%d]: ", size - 1)); } else { System.out.print("choose a version [0]: "); } Scanner scanner = new Scanner(inputStream); if (scanner.hasNextLine()) { String input = scanner.nextLine(); if (input.length() > 0) { int index = Integer.valueOf(input); if (index >= 0 && index < size) { return index; } throw new IllegalArgumentException( String.format("version index should between 0 and %d", size - 1)); } } throw new IllegalArgumentException("version index not specified"); } @VisibleForTesting protected static boolean confirmFileSetRollbackVersion(String fileSet, FileSetInfo fileSetInfo, int versionIndex, InputStream inputStream) { String rollbackHdfs = fileSetInfo.oldServingInfoList.get(versionIndex).hdfsPath; System.out.print(String.format("are you sure to rollback %s from %s to %s? [y/n]: ", fileSet, fileSetInfo.servingInfo.hdfsPath, rollbackHdfs)); Scanner scanner = new Scanner(inputStream); if (scanner.hasNextLine()) { String input = scanner.nextLine(); if (input.length() > 0 && input.equalsIgnoreCase("y")) { return true; } } return false; } @VisibleForTesting protected static boolean confirmFileSetDeletion(String fileSet, InputStream inputStream) { System.out.print(String.format("are you sure to delete %s? [y/n]: ", fileSet)); Scanner scanner = new Scanner(inputStream); if (scanner.hasNextLine()) { String input = scanner.nextLine(); if (input.length() > 0 && input.equalsIgnoreCase("y")) { return true; } } return false; } /** * Rollback specific file set to last kept version. * This function is for command line tool usage. * * @param args command line arguments containing file set * @throws Exception if communication with ZooKeeper fail */ public void rollbackFileSet(String[] args) throws Exception { if (args.length > 1) { final String fileSet = args[1]; FileSetInfo fileSetInfo = lockFileSet(zkManager, fileSet); // Add shutdown hook to gracefully catch SIGKILL signal Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { @Override public void run() { try { unlockFileSet(zkManager, fileSet); } catch (Exception e) { e.printStackTrace(); } } })); try { int versionIndex = selectFileSetRollbackVersion(fileSetInfo, System.in); if (confirmFileSetRollbackVersion(fileSet, fileSetInfo, versionIndex, System.in)) { rollbackFileSet(zkManager, fileSet, fileSetInfo, versionIndex); } else { System.out.println("exiting rollback..."); } } catch (IllegalArgumentException exception) { LOG.error(exception.getMessage()); } } else { System.err.println("missing file set argument"); } } /** * Delete file set from command line * * @param args command line arguments containing file set * @throws Exception if communication with ZooKeeper fail */ public void deleteFileSet(String[] args) throws Exception { if (args.length > 1) { String fileSet = args[1]; if (confirmFileSetDeletion(fileSet, System.in)) { deleteFileSet(zkManager, fileSet); System.out.println("file set is successfully marked to be deleted. It will be " + "physically deleted after short period of time."); } } else { System.err.println("missing file set argument"); } } /** * Lock file set * * @param zkManager ZooKeeperManager instance * @param fileSet name of file set * @param mode create mode for lock * @return file set information * @throws Exception if communication with ZooKeeper goes wrong */ public static FileSetInfo lockFileSet(ZooKeeperManager zkManager, String fileSet, CreateMode mode) throws Exception { FileSetInfo fileSetInfo = zkManager.getFileSetInfo(fileSet); if (fileSetInfo == null) { throw new IllegalArgumentException("no such file set or it is invalid"); } zkManager.lockFileSet(fileSet, fileSetInfo, mode); //re-read the file set information in case it is changed during the above process return zkManager.getFileSetInfo(fileSet); } /** * Lock file set * * @param zkManager ZooKeeperManager instance * @param fileSet name of file set * @return file set information * @throws Exception if communication with ZooKeeper goes wrong */ public static FileSetInfo lockFileSet(ZooKeeperManager zkManager, String fileSet) throws Exception { return lockFileSet(zkManager, fileSet, CreateMode.EPHEMERAL); } /** * Unlock file set * * @param zkManager ZooKeeperManager instance * @param fileSet name of file set * @throws Exception if communication with ZooKeeper goes wrong */ public static void unlockFileSet(ZooKeeperManager zkManager, String fileSet) throws Exception { zkManager.unlockFileSet(fileSet); LOG.info(String.format("release %s lock", fileSet)); } /** * Rollback file set to a specific version * * @param zkManager ZooKeeperManager instance * @param fileSet name of file set * @param fileSetInfo file set information * @param versionIndex index of version to be rolled back to * @throws IllegalArgumentException if version index is out of bound * @throws Exception if communication with ZooKeeper goes wrong */ public static void rollbackFileSet(ZooKeeperManager zkManager, String fileSet, FileSetInfo fileSetInfo, int versionIndex) throws Exception { List<FileSetInfo.ServingInfo> oldServingInfoList = fileSetInfo.oldServingInfoList; if (versionIndex >= 0 && versionIndex < oldServingInfoList.size()) { String currentHdfsPath = fileSetInfo.servingInfo.hdfsPath; String rollbackHdfsPath = oldServingInfoList.get(versionIndex).hdfsPath; fileSetInfo.servingInfo = oldServingInfoList.get(versionIndex); fileSetInfo.oldServingInfoList = oldServingInfoList.subList(versionIndex + 1, oldServingInfoList.size()); zkManager.setFileSetInfo(fileSet, fileSetInfo); LOG.info(String.format("successfully rollback %s from %s to %s", fileSet, currentHdfsPath, rollbackHdfsPath)); } else { throw new IllegalArgumentException("version index is out of bound"); } } /** * Delete file set * * @param zkManager ZooKeeperManager instance * @param fileSet name of file set * @throws Exception if communication with ZooKeeper goes wrong */ public static void deleteFileSet(ZooKeeperManager zkManager, String fileSet) throws Exception { LOG.info(String.format("deleting file set %s", fileSet)); FileSetInfo fileSetInfo = lockFileSet(zkManager, fileSet, CreateMode.PERSISTENT); fileSetInfo.deleted = true; zkManager.setFileSetInfo(fileSet, fileSetInfo); } public static void main(String[] args) throws Exception { PropertiesConfiguration configuration = TerrapinUtil .readPropertiesExitOnFailure(System.getProperties().getProperty("terrapin.config")); TerrapinAdmin admin = new TerrapinAdmin(configuration); String action = args[0]; Method[] methods = admin.getClass().getMethods(); boolean done = false; for (Method method : methods) { if (method.getName().equals(action) && !Modifier.isStatic(method.getModifiers())) { method.invoke(admin, (Object) args); done = true; } } if (!done) { LOG.error("Could not find a function call for " + args[0]); System.exit(1); } } }