Java tutorial
/* * Copyright [2013-2015] PayPal Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package ml.shifu.guagua.hadoop; import java.io.IOException; import java.io.PrintWriter; import java.net.InetAddress; import ml.shifu.guagua.GuaguaConstants; import ml.shifu.guagua.GuaguaRuntimeException; import ml.shifu.guagua.coordinator.zk.ZooKeeperUtils; import ml.shifu.guagua.io.Bytable; import ml.shifu.guagua.master.BasicMasterInterceptor; import ml.shifu.guagua.master.MasterContext; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * To start zookeeper instance in cluster environment. * * <p> * Compare with start zookeeper instance in client process, {@link ZooKeeperMasterInterceptor} can be used in cluster * and even to start zookeeper ensemble in cluster to void single point failure issue. */ public class ZooKeeperMasterInterceptor<MASTER_RESULT extends Bytable, WORKER_RESULT extends Bytable> extends BasicMasterInterceptor<MASTER_RESULT, WORKER_RESULT> { private static final Logger LOG = LoggerFactory.getLogger(ZooKeeperMasterInterceptor.class); /** * FileSystem instance to store zookeeper server info into hdfs. */ private FileSystem fileSystem; /** * Zookeeper server file path to store server address and port info. */ private Path zookeeperServerPath; /** * Do we need to start zookeeper instance in cluster env. */ private boolean isNeedStartZookeeper = false; @Override public void preApplication(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { String zkServers = context.getProps().getProperty(GuaguaConstants.GUAGUA_ZK_SERVERS); if (zkServers == null || zkServers.length() == 0 || !ZooKeeperUtils.checkServers(zkServers)) { this.isNeedStartZookeeper = true; if (fileSystem == null) { try { fileSystem = FileSystem.get(new Configuration()); } catch (IOException e) { throw new GuaguaRuntimeException(e); } } String localHostName = getLocalHostName(); LOG.warn("No valid zookeeper servers, start one in ZooKeeperMaster {}", localHostName); // 1. start embed zookeeper server in one thread. String zkJavaOpts = context.getProps().getProperty(GuaguaConstants.GUAGUA_CHILD_ZKSERVER_OPTS, GuaguaConstants.GUAGUA_CHILD_DEFAULT_ZKSERVER_OPTS); String zookeeperServer; try { zookeeperServer = ZooKeeperUtils.startChildZooKeeperProcess(zkJavaOpts); } catch (IOException e) { LOG.error("Error in start child zookeeper process.", e); // set to null to try start zookeeper with tread. zookeeperServer = null; } if (zookeeperServer == null) { // if started failed zookeeperServer = startZookeeperServer(localHostName); LOG.info("Zookeeper server is stated with thread: {}", zookeeperServer); } else { LOG.info("Zookeeper server is stated with child process: {}", zookeeperServer); } // 2. write such server info to HDFS file, no any place for us to communicate server address with worker. writeServerInfoToHDFS(context, zookeeperServer); // 3. set server info to context for next intercepters. context.getProps().setProperty(GuaguaConstants.GUAGUA_ZK_SERVERS, zookeeperServer); } } /** * Write zookeeper server info to HDFS. Then worker can get such info and connect to such server.. */ private void writeServerInfoToHDFS(MasterContext<MASTER_RESULT, WORKER_RESULT> context, String embededZooKeeperServer) { String hdfsZookeeperServerFolder = getZookeeperServerFolder(context); PrintWriter pw = null; try { if (!this.fileSystem.exists(new Path(hdfsZookeeperServerFolder))) { this.fileSystem.mkdirs(new Path(hdfsZookeeperServerFolder)); } this.zookeeperServerPath = fileSystem.makeQualified( new Path(hdfsZookeeperServerFolder, GuaguaConstants.GUAGUA_CLUSTER_ZOOKEEPER_SERVER_FILE)); LOG.info("Writing hdfs zookeeper server info to {}", this.zookeeperServerPath); FSDataOutputStream fos = fileSystem.create(this.zookeeperServerPath); pw = new PrintWriter(fos); pw.println(embededZooKeeperServer); pw.flush(); } catch (IOException e) { LOG.error("Error in writing output.", e); } catch (Exception e) { LOG.error("Error in writing output.", e); } finally { IOUtils.closeStream(pw); } } private String getZookeeperServerFolder(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { String defaultZooKeeperServePath = new StringBuilder(200).append("tmp").append(Path.SEPARATOR) .append("_guagua").append(Path.SEPARATOR).append(context.getAppId()).append(Path.SEPARATOR) .toString(); String hdfsZookeeperServerPath = context.getProps() .getProperty(GuaguaConstants.GUAGUA_ZK_CLUSTER_SERVER_FOLDER, defaultZooKeeperServePath); return hdfsZookeeperServerPath; } @Override public void postApplication(MasterContext<MASTER_RESULT, WORKER_RESULT> context) { if (this.isNeedStartZookeeper) { try { this.fileSystem.delete(new Path(this.getZookeeperServerFolder(context)), true); } catch (IOException e) { throw new GuaguaRuntimeException(e); } } } /** * Start zookeeper server in thread of master node. */ private String startZookeeperServer(String localHostName) { int embedZkClientPort = 0; try { embedZkClientPort = ZooKeeperUtils.startEmbedZooKeeper(); } catch (IOException e) { throw new RuntimeException(e); } // 2. check if it is started. ZooKeeperUtils.checkIfEmbedZooKeeperStarted(embedZkClientPort); return localHostName + ":" + embedZkClientPort; } private String getLocalHostName() { try { return InetAddress.getLocalHost().getHostName(); } catch (Exception e) { throw new GuaguaRuntimeException(e); } } }