List of usage examples for org.apache.hadoop.yarn.api.records ContainerId getApplicationAttemptId
@Public @Stable public abstract ApplicationAttemptId getApplicationAttemptId();
ApplicationAttemptId of the application to which the Container was assigned. From source file:com.toy.TOYMaster.java
License:Apache License
void init() throws Exception { if (System.getenv().containsKey(ApplicationConstants.Environment.CONTAINER_ID.name())) { ContainerId containerId = ConverterUtils .toContainerId(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); }/*from w ww . j a v a2 s.c o m*/ Preconditions.checkNotNull(appAttemptID, "No attempt id"); zookeeper.start(); if (!zookeeper.getZookeeperClient().blockUntilConnectedOrTimedOut()) { LOG.error("Unable to connect to Zookeeper @{}", System.getenv(Constants.ZOOKEEPER_QUORUM)); System.exit(1); } // Create ROOT znode try { zookeeper.create().creatingParentsIfNeeded().forPath(ZK_ROOT); } catch (KeeperException.NodeExistsException e) { //Ignore } // Check if there is no other master alive if (zookeeper.checkExists().forPath(ZK_MASTER) != null) { throw new Exception("Master is already alive"); } // Create master znode final String masterPath = zookeeper.create().withMode(CreateMode.EPHEMERAL).forPath(ZK_MASTER, getMasterInfo().getBytes(Charsets.UTF_8)); final Stat masterPathExist = zookeeper.checkExists().usingWatcher(new ZKMasterWatcher()) .forPath(masterPath); if (masterPathExist == null) { throw new Exception("Master znode is not created"); } // Start order queue ordersQueue = Orders.getQueueAsConsumer(zookeeper, this, ZK_ORDERS); ordersQueue.start(); }
From source file:com.yahoo.storm.yarn.MasterServer.java
License:Open Source License
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { LOG.info("Starting the AM!!!!"); Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used " + "unless for testing purposes"); CommandLine cl = new GnuParser().parse(opts, args); ApplicationAttemptId appAttemptID;/*from w w w .j a va 2s. c o m*/ Map<String, String> envs = System.getenv(); if (cl.hasOption("app_attempt_id")) { String appIdStr = cl.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else if (envs.containsKey(ApplicationConstants.Environment.CONTAINER_ID.name())) { ContainerId containerId = ConverterUtils .toContainerId(envs.get(ApplicationConstants.Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); LOG.info("appAttemptID from env:" + appAttemptID.toString()); } else { LOG.error("appAttemptID is not specified for storm master"); throw new Exception("appAttemptID is not specified for storm master"); } @SuppressWarnings("rawtypes") Map storm_conf = Config.readStormConfig(null); Util.rmNulls(storm_conf); YarnConfiguration hadoopConf = new YarnConfiguration(); final String host = InetAddress.getLocalHost().getHostName(); storm_conf.put("nimbus.host", host); StormAMRMClient rmClient = new StormAMRMClient(appAttemptID, storm_conf, hadoopConf); rmClient.init(hadoopConf); rmClient.start(); BlockingQueue<Container> launcherQueue = new LinkedBlockingQueue<Container>(); MasterServer server = new MasterServer(storm_conf, rmClient); try { final int port = Utils.getInt(storm_conf.get(Config.MASTER_THRIFT_PORT)); final String target = host + ":" + port; InetSocketAddress addr = NetUtils.createSocketAddr(target); RegisterApplicationMasterResponse resp = rmClient.registerApplicationMaster(addr.getHostName(), port, null); LOG.info("Got a registration response " + resp); LOG.info("Max Capability " + resp.getMaximumResourceCapability()); rmClient.setMaxResource(resp.getMaximumResourceCapability()); LOG.info("Starting HB thread"); server.initAndStartHeartbeat(rmClient, launcherQueue, (Integer) storm_conf.get(Config.MASTER_HEARTBEAT_INTERVAL_MILLIS)); LOG.info("Starting launcher"); initAndStartLauncher(rmClient, launcherQueue); rmClient.startAllSupervisors(); LOG.info("Starting Master Thrift Server"); server.serve(); LOG.info("StormAMRMClient::unregisterApplicationMaster"); rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "AllDone", null); } finally { if (server.isServing()) { LOG.info("Stop Master Thrift Server"); server.stop(); } LOG.info("Stop RM client"); rmClient.stop(); } System.exit(0); }
From source file:com.zqh.hadoop.moya.core.yarn.ApplicationMaster.java
License:Apache License
/** * Parse command line options// w w w.j av a 2 s . c o m * * @param args Command line args * @return Whether init successful and run should be invoked * @throws org.apache.commons.cli.ParseException * @throws java.io.IOException */ public boolean init(String[] args) throws ParseException, IOException { Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); opts.addOption("lib", true, "libs required to run Moya"); opts.addOption("shell_script", true, "Location of the shell script to be executed"); opts.addOption("shell_args", true, "Command line args for the shell script"); opts.addOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"); opts.addOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command"); opts.addOption("num_containers", true, "No. of containers on which the shell command needs to be executed"); opts.addOption("priority", true, "Application Priority. Default 0"); opts.addOption("debug", false, "Dump out debug information"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { printUsage(opts); throw new IllegalArgumentException("No args specified for application master to initialize"); } if (cliParser.hasOption("help")) { printUsage(opts); return false; } if (cliParser.hasOption("debug")) { dumpOutDebugInfo(); } Map<String, String> envs = System.getenv(); if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); // if (!cliParser.hasOption("shell_command")) { // throw new IllegalArgumentException( // "No shell command specified to be executed by application master"); // } // shellCommand = cliParser.getOptionValue("shell_command"); if (cliParser.hasOption("shell_args")) { shellArgs = cliParser.getOptionValue("shell_args"); } if (cliParser.hasOption("shell_env")) { String shellEnvs[] = cliParser.getOptionValues("shell_env"); for (String env : shellEnvs) { env = env.trim(); int index = env.indexOf('='); if (index == -1) { shellEnv.put(env, ""); continue; } String key = env.substring(0, index); String val = ""; if (index < (env.length() - 1)) { val = env.substring(index + 1); } shellEnv.put(key, val); } } if (envs.containsKey(MConstants.APPLICATIONMASTERJARLOCATION)) { shellScriptPath = envs.get(MConstants.APPLICATIONMASTERJARLOCATION); if (envs.containsKey(MConstants.APPLICATIONMASTERJARTIMESTAMP)) { shellScriptPathTimestamp = Long.valueOf(envs.get(MConstants.APPLICATIONMASTERJARTIMESTAMP)); } if (envs.containsKey(MConstants.APPLICATIONMASTERJARLEN)) { shellScriptPathLen = Long.valueOf(envs.get(MConstants.APPLICATIONMASTERJARLEN)); } if (envs.containsKey(MConstants.LIBSLOCATION)) { libsPath = envs.get(MConstants.LIBSLOCATION); if (envs.containsKey(MConstants.LIBSTIMESTAMP)) { libsTimestamp = Long.valueOf(envs.get(MConstants.LIBSTIMESTAMP)); } if (envs.containsKey(MConstants.LIBSLEN)) { libsPathLen = Long.valueOf(envs.get(MConstants.LIBSLEN)); } } //TODO if (envs.containsKey(MConstants.ZOOKEEPERHOSTS)) { ZKHosts = envs.get(MConstants.ZOOKEEPERHOSTS); } if (!shellScriptPath.isEmpty() && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) { LOG.error("Illegal values in env for shell script path" + ", path=" + shellScriptPath + ", len=" + shellScriptPathLen + ", timestamp=" + shellScriptPathTimestamp); throw new IllegalArgumentException("Illegal values in env for shell script path"); } } containerMemory = Integer.parseInt(cliParser.getOptionValue("container_memory", "10")); numTotalContainers = Integer.parseInt(cliParser.getOptionValue("num_containers", "1")); if (numTotalContainers == 0) { throw new IllegalArgumentException("Cannot run distributed shell with no containers"); } requestPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); return true; }
From source file:de.huberlin.wbi.hiway.am.HiWay.java
License:Apache License
/** * Parse command line options.//from w w w .j av a 2 s . co m * * @param args * Command line arguments. * @return Whether init successful and run should be invoked. * @throws ParseException * ParseException */ public boolean init(String[] args) throws ParseException { DefaultMetricsSystem.initialize("ApplicationMaster"); Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); opts.addOption("workflow", true, "The workflow file to be executed by the Application Master"); opts.addOption("s", "summary", true, "The name of the json summary file. No file is created if this parameter is not specified."); opts.addOption("debug", false, "Dump out debug information"); opts.addOption("appid", true, "Id of this Application Master."); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { printUsage(opts); throw new IllegalArgumentException("No args specified for application master to initialize"); } if (!cliParser.hasOption("appid")) { throw new IllegalArgumentException("No id of Application Master specified"); } appId = cliParser.getOptionValue("appid"); try { statLog = new BufferedWriter(new FileWriter(appId + ".log")); } catch (IOException e) { e.printStackTrace(); System.exit(-1); } if (cliParser.hasOption("help")) { printUsage(opts); return false; } if (cliParser.hasOption("debug")) { dumpOutDebugInfo(); } if (cliParser.hasOption("summary")) { summaryPath = new Path(cliParser.getOptionValue("summary")); } String hdfsBaseDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE, HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE_DEFAULT); String hdfsSandboxDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE, HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE_DEFAULT); Path hdfsBaseDirectory = new Path(new Path(hdfs.getUri()), hdfsBaseDirectoryName); Data.setHdfsBaseDirectory(hdfsBaseDirectory); Path hdfsSandboxDirectory = new Path(hdfsBaseDirectory, hdfsSandboxDirectoryName); hdfsApplicationDirectory = new Path(hdfsSandboxDirectory, appId); Data.setHdfsApplicationDirectory(hdfsApplicationDirectory); Data.setHdfs(hdfs); Map<String, String> envs = System.getenv(); if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } System.out.println("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); String shellEnvs[] = conf.getStrings(HiWayConfiguration.HIWAY_WORKER_SHELL_ENV, HiWayConfiguration.HIWAY_WORKER_SHELL_ENV_DEFAULT); for (String env : shellEnvs) { env = env.trim(); int index = env.indexOf('='); if (index == -1) { shellEnv.put(env, ""); continue; } String key = env.substring(0, index); String val = ""; if (index < (env.length() - 1)) { val = env.substring(index + 1); } shellEnv.put(key, val); } if (!cliParser.hasOption("workflow")) { throw new IllegalArgumentException("No workflow file specified to be executed by application master"); } workflowPath = new Path(cliParser.getOptionValue("workflow")); workflowFile = new Data(workflowPath); schedulerName = HiWayConfiguration.HIWAY_SCHEDULER_OPTS.valueOf(conf.get(HiWayConfiguration.HIWAY_SCHEDULER, HiWayConfiguration.HIWAY_SCHEDULER_DEFAULT.toString())); containerMemory = conf.getInt(HiWayConfiguration.HIWAY_WORKER_MEMORY, HiWayConfiguration.HIWAY_WORKER_MEMORY_DEFAULT); containerCores = conf.getInt(HiWayConfiguration.HIWAY_WORKER_VCORES, HiWayConfiguration.HIWAY_WORKER_VCORES_DEFAULT); requestPriority = conf.getInt(HiWayConfiguration.HIWAY_WORKER_PRIORITY, HiWayConfiguration.HIWAY_WORKER_PRIORITY_DEFAULT); return true; }
From source file:de.huberlin.wbi.hiway.am.WorkflowDriver.java
License:Apache License
/** Parse command line arguments, initialize HDFS, manage environment variables. */ public boolean init(String[] args) throws ParseException, IOException, JSONException { DefaultMetricsSystem.initialize("ApplicationMaster"); Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); opts.addOption("u", "summary", true, "The name of the json summary file. No file is created if this parameter is not specified."); opts.addOption("m", "memory", true, "The amount of memory (in MB) to be allocated per worker container. Overrides settings in hiway-site.xml."); opts.addOption("c", "custom", true, "The name of an (optional) JSON file, in which custom amounts of memory can be specified per task."); opts.addOption("s", "scheduler", true, "The scheduling policy that is to be employed. Valid arguments: " + Arrays.toString(HiWayConfiguration.HIWAY_SCHEDULERS.values()) + ". Overrides settings in hiway-site.xml."); opts.addOption("d", "debug", false, "Provide additional logs and information for debugging"); opts.addOption("v", "verbose", false, "Increase verbosity of output / reporting."); opts.addOption("appid", true, "Id of this Application Master."); opts.addOption("h", "help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { Logger.printUsage(opts);/* w w w . j ava 2 s.c o m*/ throw new IllegalArgumentException("No args specified for application master to initialize"); } if (cliParser.getArgs().length == 0) { Logger.printUsage(opts); throw new IllegalArgumentException("No workflow file specified."); } if (!cliParser.hasOption("appid")) { throw new IllegalArgumentException("No id of Application Master specified"); } if (cliParser.hasOption("verbose")) { HiWayConfiguration.verbose = true; } appId = cliParser.getOptionValue("appid"); try { logger.statLog = new BufferedWriter(new FileWriter(appId + ".log")); } catch (IOException e) { e.printStackTrace(System.out); System.exit(-1); } if (cliParser.hasOption("help")) { Logger.printUsage(opts); return false; } if (cliParser.hasOption("debug")) { Logger.dumpOutDebugInfo(); HiWayConfiguration.debug = true; } if (cliParser.hasOption("summary")) { summaryPath = new Path(cliParser.getOptionValue("summary")); } String hdfsBaseDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE, HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE_DEFAULT); String hdfsSandboxDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE, HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE_DEFAULT); Path hdfsBaseDirectory = new Path(new Path(hdfs.getUri()), hdfsBaseDirectoryName); Data.setHdfsBaseDirectory(hdfsBaseDirectory); Path hdfsSandboxDirectory = new Path(hdfsBaseDirectory, hdfsSandboxDirectoryName); hdfsApplicationDirectory = new Path(hdfsSandboxDirectory, appId); Data.setHdfsApplicationDirectory(hdfsApplicationDirectory); Data.setHdfs(hdfs); if (cliParser.hasOption("custom")) { Data customMemPath = new Data(cliParser.getOptionValue("custom")); customMemPath.stageIn(); StringBuilder sb = new StringBuilder(); try (BufferedReader in = new BufferedReader(new FileReader(customMemPath.getLocalPath().toString()))) { String line; while ((line = in.readLine()) != null) { sb.append(line); } } JSONObject obj = new JSONObject(sb.toString()); Iterator<?> keys = obj.keys(); while (keys.hasNext()) { String key = (String) keys.next(); int minMem = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); int desiredMem = obj.getInt(key); customMemoryMap.put(key, (desiredMem % minMem) == 0 ? desiredMem : (desiredMem / minMem + 1) * minMem); } } Map<String, String> envs = System.getenv(); /* this application's attempt id (combination of attemptId and fail count) */ ApplicationAttemptId appAttemptID; if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } Logger.writeToStdout("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); String shellEnvs[] = conf.getStrings(HiWayConfiguration.HIWAY_WORKER_SHELL_ENV, HiWayConfiguration.HIWAY_WORKER_SHELL_ENV_DEFAULT); for (String env : shellEnvs) { env = env.trim(); int index = env.indexOf('='); if (index == -1) { shellEnv.put(env, ""); continue; } String key = env.substring(0, index); String val = ""; if (index < (env.length() - 1)) { val = env.substring(index + 1); } shellEnv.put(key, val); } String workflowParam = cliParser.getArgs()[0]; try { workflowPath = new Path(new URI(workflowParam).getPath()); } catch (URISyntaxException e) { workflowPath = new Path(workflowParam); } schedulerEnumValue = HiWayConfiguration.HIWAY_SCHEDULERS.valueOf(conf .get(HiWayConfiguration.HIWAY_SCHEDULER, HiWayConfiguration.HIWAY_SCHEDULER_DEFAULT.toString())); if (cliParser.hasOption("scheduler")) { schedulerEnumValue = HiWayConfiguration.HIWAY_SCHEDULERS.valueOf(cliParser.getOptionValue("scheduler")); } containerMemory = conf.getInt(HiWayConfiguration.HIWAY_WORKER_MEMORY, HiWayConfiguration.HIWAY_WORKER_MEMORY_DEFAULT); if (cliParser.hasOption("memory")) { containerMemory = Integer.parseInt(cliParser.getOptionValue("memory")); } containerCores = conf.getInt(HiWayConfiguration.HIWAY_WORKER_VCORES, HiWayConfiguration.HIWAY_WORKER_VCORES_DEFAULT); requestPriority = conf.getInt(HiWayConfiguration.HIWAY_WORKER_PRIORITY, HiWayConfiguration.HIWAY_WORKER_PRIORITY_DEFAULT); // Create and start the Timeline timelineClient if (conf.getBoolean("yarn.timeline-service.enabled", false)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); Logger.writeToStdout("Started TimeLineClient."); } else { Logger.writeToStdErr("TimeLineClient disabled."); } return true; }
From source file:edu.cmu.graphchi.toolkits.collaborative_filtering.yarn.ApplicationMaster.java
License:Apache License
/** * Parse command line options//from w w w .ja v a2s . co m * * @param args Command line args * @return Whether init successful and run should be invoked * @throws ParseException * @throws IOException */ public boolean init(String[] args) throws ParseException, IOException { LOG.info("Inside Init"); this.setup = new ProblemSetup(args); Configuration conf = new YarnConfiguration(); this.yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); Map<String, String> envs = System.getenv(); ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); DataSetDescription dataDesc = new DataSetDescription(); dataDesc.loadFromJsonFile(setup.dataMetadataFile); this.recommenders = RecommenderFactory.buildRecommenders(dataDesc, setup.paramFile, null, setup); return true; }
From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java
License:Apache License
public void setEnvs(CommandLine cliParser) { Map<String, String> envs = System.getenv(); if (envs.containsKey("HADOOP_CONF_DIR")) { File hadoopConfDir = new File(envs.get("HADOOP_CONF_DIR")); if (hadoopConfDir.isDirectory()) { for (File config : hadoopConfDir.listFiles()) { if (config.getName().matches("^.*(xml)$")) { conf.addResource(new Path(config.getAbsolutePath())); }//from ww w . ja va 2 s .c o m } } } //the containerID might be in the arguments or the environment if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException( "Environment is not set correctly- please check client submission settings"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV) || !envs.containsKey(Environment.NM_HOST.name()) || !envs.containsKey(Environment.NM_HTTP_PORT.name()) || !envs.containsKey(Environment.NM_PORT.name())) { throw new IllegalArgumentException( "Environment is not set correctly- please check client submission settings"); } System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, envs.get("PWD") + File.separator + "bin" + File.separator + ASTERIX_CONF_NAME); LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); asterixZipPath = envs.get(AConstants.TARLOCATION); asterixZipTimestamp = Long.parseLong(envs.get(AConstants.TARTIMESTAMP)); asterixZipLen = Long.parseLong(envs.get(AConstants.TARLEN)); asterixConfPath = envs.get(AConstants.CONFLOCATION); asterixConfTimestamp = Long.parseLong(envs.get(AConstants.CONFTIMESTAMP)); asterixConfLen = Long.parseLong(envs.get(AConstants.CONFLEN)); instanceConfPath = envs.get(AConstants.INSTANCESTORE); //the only time this is null is during testing, when asterix-yarn isn't packaged in a JAR yet. if (envs.get(AConstants.APPLICATIONMASTERJARLOCATION) != null && !envs.get(AConstants.APPLICATIONMASTERJARLOCATION).endsWith(File.separator)) { appMasterJar = new Path(envs.get(AConstants.APPLICATIONMASTERJARLOCATION)); } else { appMasterJar = null; } dfsBasePath = envs.get(AConstants.DFS_BASE); //If the NM has an odd environment where the proper hadoop XML configs dont get imported, we can end up not being able to talk to the RM // this solves that! //in a testing environment these can be null however. if (envs.get(AConstants.RMADDRESS) != null) { conf.set("yarn.resourcemanager.address", envs.get(AConstants.RMADDRESS)); LOG.info("RM Address: " + envs.get(AConstants.RMADDRESS)); } if (envs.get(AConstants.RMADDRESS) != null) { conf.set("yarn.resourcemanager.scheduler.address", envs.get(AConstants.RMSCHEDULERADDRESS)); } ccJavaOpts = envs.get(AConstants.CC_JAVA_OPTS); //set defaults if no special given options if (ccJavaOpts == null) { ccJavaOpts = EXTERNAL_CC_JAVA_OPTS_DEFAULT; } ncJavaOpts = envs.get(AConstants.NC_JAVA_OPTS); if (ncJavaOpts == null) { ncJavaOpts = EXTERNAL_NC_JAVA_OPTS_DEFAULT; } LOG.info("Path suffix: " + instanceConfPath); }
From source file:edu.uci.ics.hyracks.yarn.am.HyracksYarnApplicationMaster.java
License:Apache License
private HyracksYarnApplicationMaster(Options options) { this.options = options; timer = new Timer(true); asks = new ArrayList<ResourceRequest>(); resource2AskMap = new HashMap<Resource, Set<AskRecord>>(); proc2AskMap = new HashMap<AbstractProcess, AskRecord>(); lastResponseId = new AtomicInteger(); String containerIdStr = System.getenv(ApplicationConstants.AM_CONTAINER_ID_ENV); ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); appAttemptId = containerId.getApplicationAttemptId(); }
From source file:edu.uci.ics.hyracks.yarn.common.protocols.amrm.AMRMConnection.java
License:Apache License
public AMRMConnection(YarnConfiguration config) { this.config = config; Map<String, String> envs = System.getenv(); String containerIdString = envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV); if (containerIdString == null) { throw new IllegalArgumentException("ContainerId not set in the environment"); }//from ww w . j av a 2s . c om ContainerId containerId = ConverterUtils.toContainerId(containerIdString); appAttemptId = containerId.getApplicationAttemptId(); InetSocketAddress rmAddress = NetUtils.createSocketAddr( config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS)); YarnRPC rpc = YarnRPC.create(config); amrmp = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, config); }
From source file:gobblin.yarn.ContainerMetrics.java
License:Open Source License
private static List<Tag<?>> tagsForContainer(State containerState, String applicationName, ContainerId containerId) { ImmutableList.Builder<Tag<?>> tags = new ImmutableList.Builder<>(); tags.add(new Tag<>(GobblinYarnMetricTagNames.YARN_APPLICATION_NAME, applicationName)); tags.add(new Tag<>(GobblinYarnMetricTagNames.YARN_APPLICATION_ID, containerId.getApplicationAttemptId().getApplicationId().toString())); tags.add(new Tag<>(GobblinYarnMetricTagNames.YARN_APPLICATION_ATTEMPT_ID, containerId.getApplicationAttemptId().toString())); tags.add(new Tag<>(GobblinYarnMetricTagNames.CONTAINER_ID, containerId.toString())); tags.addAll(getCustomTagsFromState(containerState)); return tags.build(); }