Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.tajo.yarn; import org.apache.commons.cli.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.StringHelper; import org.apache.tajo.yarn.thrift.TajoYarnService; import org.apache.thrift.TProcessor; import org.apache.thrift.server.TNonblockingServer; import org.apache.thrift.server.TServer; import org.apache.thrift.transport.TNonblockingServerSocket; import org.apache.thrift.transport.TTransportException; import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.nio.ByteBuffer; import java.util.*; /** * Tajo-Yarn Application Master for YARN */ @InterfaceAudience.Public @InterfaceStability.Unstable public class ApplicationMaster { private static final Log LOG = LogFactory.getLog(ApplicationMaster.class); // Configuration private Configuration conf; private AppContext appContext; // Hardcoded path to custom log_properties private static final String log4jPath = "log4j.properties"; private ClusterScheduler scheduler; private TServer server; /** * @param args Command line args */ public static void main(String[] args) { try { ApplicationMaster appMaster = new ApplicationMaster(); LOG.info("Initializing ApplicationMaster"); boolean doRun = appMaster.init(args); if (!doRun) { System.exit(0); } appMaster.run(); } catch (Throwable t) { LOG.fatal("Error running ApplicationMaster", t); System.exit(1); } } public ApplicationMaster() { // Set up the configuration conf = new YarnConfiguration(); } /** * Parse command line options * * @param args Command line args * @return Whether init successful and getLaunchContext should be invoked * @throws org.apache.commons.cli.ParseException * @throws java.io.IOException */ public boolean init(String[] args) throws ParseException, IOException { Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); opts.addOption("qm_memory", true, "Amount of memory in MB to be requested to launch a QueryMaster. Default 512"); opts.addOption("qm_vcores", true, "Amount of virtual cores to be requested to launch a QueryMaster. Default 2"); opts.addOption("tr_memory", true, "Amount of memory in MB to be requested to launch a TaskRunner. Default 1024"); opts.addOption("tr_vcores", true, "Amount of virtual cores to be requested to launch a TaskRunner. Default 4"); opts.addOption("worker_memory", true, "Amount of memory in MB to be requested to launch a worker. Default 2048"); opts.addOption("worker_vcores", true, "Amount of virtual cores to be requested to launch a worker. Default 4"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); // Check whether customer log4j.properties file exists if (fileExist(log4jPath)) { try { Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath); } catch (Exception e) { LOG.warn("Can not set up custom log4j properties. " + e); } } if (cliParser.hasOption("help")) { printUsage(opts); return false; } ApplicationAttemptId appAttemptID = null; Map<String, String> envs = System.getenv(); if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); int qmMemory = Integer.parseInt(cliParser.getOptionValue("qm_memory", "512")); int qmVCores = Integer.parseInt(cliParser.getOptionValue("qm_vcores", "2")); int trMemory = Integer.parseInt(cliParser.getOptionValue("tr_memory", "1024")); int trVCores = Integer.parseInt(cliParser.getOptionValue("tr_vcores", "4")); int workerMemory = Integer.parseInt(cliParser.getOptionValue("worker_memory", "2048")); int workerVCores = Integer.parseInt(cliParser.getOptionValue("worker_vcores", "4")); int requestPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); String appMasterHostName = InetAddress.getLocalHost().getHostName(); this.appContext = new AppContext(conf, appAttemptID, workerMemory, workerVCores, requestPriority, appMasterHostName); return true; } /** * Helper function to print usage * * @param opts Parsed command line options */ private void printUsage(Options opts) { new HelpFormatter().printHelp("ApplicationMaster", opts); } /** * Main getLaunchContext function for the application master * * @throws org.apache.hadoop.yarn.exceptions.YarnException * @throws java.io.IOException */ @SuppressWarnings({ "unchecked" }) public void run() throws YarnException, IOException { LOG.info("Starting ApplicationMaster"); startTajoMaster(); TNonblockingServerSocket serverSocket; int appMasterRpcPort; try { serverSocket = new TNonblockingServerSocket(0); appMasterRpcPort = ThriftHelper.getServerSocketFor(serverSocket).getLocalPort(); } catch (TTransportException tte) { LOG.error("Error throws when starting rpc server", tte); throw new IOException(tte); } LOG.info("Starting cluster scheduler"); String appMasterTrackingUrl = StringHelper.join("http://", appContext.getMasterHost(), ":26080"); this.scheduler = new ClusterScheduler(appContext, appMasterTrackingUrl, appMasterRpcPort); scheduler.init(conf); scheduler.service(); LOG.info("Starting rpc server, listening on" + appContext.getMasterHost() + ":" + appMasterRpcPort); TProcessor processor = new TajoYarnService.Processor<TajoYarnService.Iface>( new TajoYarnServiceImpl(scheduler, appContext)); this.server = new TNonblockingServer(new TNonblockingServer.Args(serverSocket).processor(processor)); this.server.serve(); } public void startTajoMaster() throws IOException { LOG.info("Current working dir:" + System.getProperty("user.dir")); String tajoHome = System.getenv("TAJO_HOME"); List<String> script = new ArrayList<String>(); script.add("bin/tajo-daemon.sh"); script.add("start"); script.add("master"); Shell.ShellCommandExecutor shell = new Shell.ShellCommandExecutor(script.toArray(new String[script.size()]), new File(tajoHome)); try { shell.execute(); LOG.info(shell.getOutput()); } catch (Shell.ExitCodeException e) { throw new IOException(e); } } private boolean fileExist(String filePath) { return new File(filePath).exists(); } }