List of usage examples for org.apache.hadoop.yarn.api.records ContainerId getApplicationAttemptId
@Public @Stable public abstract ApplicationAttemptId getApplicationAttemptId();
ApplicationAttemptId of the application to which the Container was assigned. From source file:gobblin.yarn.GobblinApplicationMaster.java
License:Apache License
public GobblinApplicationMaster(String applicationName, ContainerId containerId, Config config, YarnConfiguration yarnConfiguration) throws Exception { super(applicationName, containerId.getApplicationAttemptId().getApplicationId().toString(), config, Optional.<Path>absent()); GobblinYarnLogSource gobblinYarnLogSource = new GobblinYarnLogSource(); if (gobblinYarnLogSource.isLogSourcePresent()) { this.applicationLauncher .addService(gobblinYarnLogSource.buildLogCopier(config, containerId, this.fs, this.appWorkDir)); }// w w w. ja v a 2 s. c om this.applicationLauncher.addService( buildYarnService(config, applicationName, this.applicationId, yarnConfiguration, this.fs)); if (UserGroupInformation.isSecurityEnabled()) { LOGGER.info("Adding YarnContainerSecurityManager since security is enabled"); this.applicationLauncher.addService(buildYarnContainerSecurityManager(config, this.fs)); } }
From source file:gobblin.yarn.GobblinYarnTaskRunner.java
License:Apache License
private static String getApplicationId(ContainerId containerId) { return containerId.getApplicationAttemptId().getApplicationId().toString(); }
From source file:hadoop.yarn.distributedshell.DshellApplicationMaster.java
License:Apache License
/** * Parse command line options/*from w ww. j av a 2 s . c o m*/ * * @param args * Command line args * @return Whether init successful and run should be invoked * @throws ParseException * @throws IOException */ public boolean init(String[] args) throws ParseException, IOException { Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); opts.addOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"); opts.addOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command"); opts.addOption("container_vcores", true, "Amount of virtual cores to be requested to run the shell command"); opts.addOption("num_containers", true, "No. of containers on which the shell command needs to be executed"); opts.addOption("priority", true, "Application Priority. Default 0"); opts.addOption("debug", false, "Dump out debug information"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { printUsage(opts); throw new IllegalArgumentException("No args specified for application master to initialize"); } // Check whether customer log4j.properties file exists if (fileExist(log4jPath)) { try { Log4jPropertyHelper.updateLog4jConfiguration(DshellApplicationMaster.class, log4jPath); } catch (Exception e) { LOG.warn("Can not set up custom log4j properties. " + e); } } if (cliParser.hasOption("help")) { printUsage(opts); return false; } if (cliParser.hasOption("debug")) { dumpOutDebugInfo(); } Map<String, String> envs = System.getenv(); if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); if (!fileExist(shellCommandPath) && envs.get(DshellDSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION).isEmpty()) { throw new IllegalArgumentException( "No shell command or shell script specified to be executed by application master"); } if (fileExist(shellCommandPath)) { shellCommand = readContent(shellCommandPath); } if (fileExist(shellArgsPath)) { shellArgs = readContent(shellArgsPath); } if (cliParser.hasOption("shell_env")) { String shellEnvs[] = cliParser.getOptionValues("shell_env"); for (String env : shellEnvs) { env = env.trim(); int index = env.indexOf('='); if (index == -1) { shellEnv.put(env, ""); continue; } String key = env.substring(0, index); String val = ""; if (index < (env.length() - 1)) { val = env.substring(index + 1); } shellEnv.put(key, val); } } // ??envsjarsHDFSLEN?jarPaths for (String key : envs.keySet()) { if (key.contains(DshellDSConstants.DISTRIBUTEDJARLOCATION)) { DshellFile scistorJar = new DshellFile(); scistorJar.setJarPath(envs.get(key)); String num = key.split(DshellDSConstants.DISTRIBUTEDJARLOCATION)[1]; scistorJar.setTimestamp(Long.parseLong(envs.get(DshellDSConstants.DISTRIBUTEDJARTIMESTAMP + num))); scistorJar.setSize(Long.parseLong(envs.get(DshellDSConstants.DISTRIBUTEDJARLEN + num))); scistorJars.add(scistorJar); } } // // ??envsarchiveHDFSLEN?archivePaths for (String key : envs.keySet()) { if (key.contains(DshellDSConstants.DISTRIBUTEDARCHIVELOCATION)) { DshellArchive scistorArchive = new DshellArchive(); scistorArchive.setArchivePath(envs.get(key)); String num = key.split(DshellDSConstants.DISTRIBUTEDARCHIVELOCATION)[1]; scistorArchive.setTimestamp( Long.parseLong(envs.get(DshellDSConstants.DISTRIBUTEDARCHIVETIMESTAMP + num))); scistorArchive.setSize(Long.parseLong(envs.get(DshellDSConstants.DISTRIBUTEDARCHIVELEN + num))); scistorArchives.add(scistorArchive); } } // if (envs.containsKey(DshellDSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) { scriptPath = envs.get(DshellDSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION); if (envs.containsKey(DshellDSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) { shellScriptPathTimestamp = Long .valueOf(envs.get(DshellDSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)); } if (envs.containsKey(DshellDSConstants.DISTRIBUTEDSHELLSCRIPTLEN)) { shellScriptPathLen = Long.valueOf(envs.get(DshellDSConstants.DISTRIBUTEDSHELLSCRIPTLEN)); } if (!scriptPath.isEmpty() && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) { LOG.error("Illegal values in env for shell script path" + ", path=" + scriptPath + ", len=" + shellScriptPathLen + ", timestamp=" + shellScriptPathTimestamp); throw new IllegalArgumentException("Illegal values in env for shell script path"); } } containerMemory = Integer.parseInt(cliParser.getOptionValue("container_memory", "10")); containerVirtualCores = Integer.parseInt(cliParser.getOptionValue("container_vcores", "1")); numTotalContainers = Integer.parseInt(cliParser.getOptionValue("num_containers", "1")); if (numTotalContainers == 0) { throw new IllegalArgumentException("Cannot run distributed shell with no containers"); } requestPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); // Creating the Timeline Client timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); return true; }
From source file:io.hops.tensorflow.ApplicationMaster.java
License:Apache License
/** * Parse command line options//from w w w.j a v a2s .c o m * * @param args * Command line args * @return Whether init successful and run should be invoked * @throws ParseException * @throws IOException */ public boolean init(String[] args) throws ParseException, IOException { Options opts = createOptions(); cliParser = new GnuParser().parse(opts, args); containerPython = cliParser.getOptionValue(PYTHON, null); if (args.length == 0) { printUsage(opts); throw new IllegalArgumentException("No args specified for application master to initialize"); } //Check whether customer log4j.properties file exists if (fileExist(LOG4J_PATH)) { try { Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, LOG4J_PATH); } catch (Exception e) { LOG.warn("Can not set up custom log4j properties. " + e); } } if (cliParser.hasOption(HELP)) { printUsage(opts); return false; } if (cliParser.hasOption(DEBUG)) { dumpOutDebugInfo(); } if (!cliParser.hasOption(MAIN_RELATIVE)) { throw new IllegalArgumentException("No main application file specified"); } mainRelative = cliParser.getOptionValue(MAIN_RELATIVE); if (cliParser.hasOption(ARGS)) { arguments = cliParser.getOptionValues(ARGS); } Map<String, String> envs = System.getenv(); if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption(APP_ATTEMPT_ID)) { String appIdStr = cliParser.getOptionValue(APP_ATTEMPT_ID, ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); if (cliParser.hasOption(ENV)) { String shellEnvs[] = cliParser.getOptionValues(ENV); for (String env : shellEnvs) { env = env.trim(); int index = env.indexOf('='); if (index == -1) { environment.put(env, ""); continue; } String key = env.substring(0, index); String val = ""; if (index < (env.length() - 1)) { val = env.substring(index + 1); } environment.put(key, val); } } if (cliParser.hasOption(TENSORBOARD)) { environment.put("YARNTF_TENSORBOARD", "true"); } if (envs.containsKey(Constants.YARNTFTIMELINEDOMAIN)) { domainId = envs.get(Constants.YARNTFTIMELINEDOMAIN); } containerMemory = Integer.parseInt(cliParser.getOptionValue(MEMORY, "1024")); containerVirtualCores = Integer.parseInt(cliParser.getOptionValue(VCORES, "1")); containerGPUs = Integer.parseInt(cliParser.getOptionValue(GPUS, "0")); tfProtocol = cliParser.getOptionValue(PROTOCOL, null); numWorkers = Integer.parseInt(cliParser.getOptionValue(WORKERS, "1")); numPses = Integer.parseInt(cliParser.getOptionValue(PSES, "1")); numTotalContainers = numWorkers + numPses; if (!(numWorkers > 0 && numPses > 0 || numWorkers == 1 && numPses == 0)) { throw new IllegalArgumentException("Invalid no. of workers or parameter server"); } // requestPriority = Integer.parseInt(cliParser.getOptionValue(PRIORITY, "0")); allocationTimeout = Long.parseLong(cliParser.getOptionValue(ALLOCATION_TIMEOUT, "15")) * 1000; environment.put("YARNTF_MEMORY", Integer.toString(containerMemory)); environment.put("YARNTF_VCORES", Integer.toString(containerVirtualCores)); environment.put("YARNTF_GPUS", Integer.toString(containerGPUs)); if (tfProtocol != null) { environment.put("YARNTF_PROTOCOL", tfProtocol); } environment.put("YARNTF_WORKERS", Integer.toString(numWorkers)); environment.put("YARNTF_PSES", Integer.toString(numPses)); environment.put("YARNTF_HOME_DIR", FileSystem.get(conf).getHomeDirectory().toString()); environment.put("PYTHONUNBUFFERED", "true"); DistributedCacheList distCacheList = null; FileInputStream fin = null; ObjectInputStream ois = null; try { fin = new FileInputStream(Constants.DIST_CACHE_PATH); ois = new ObjectInputStream(fin); try { distCacheList = (DistributedCacheList) ois.readObject(); } catch (ClassNotFoundException e) { e.printStackTrace(); } } finally { org.apache.commons.io.IOUtils.closeQuietly(ois); org.apache.commons.io.IOUtils.closeQuietly(fin); } LOG.info("Loaded distribute cache list: " + distCacheList.toString()); for (int i = 0; i < distCacheList.size(); i++) { DistributedCacheList.Entry entry = distCacheList.get(i); LocalResource distRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(entry.uri), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, entry.size, entry.timestamp); localResources.put(entry.relativePath, distRsrc); } return true; }
From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java
License:Apache License
/** * Application entry point//from w w w.j av a 2 s. c o m * * @param args * command-line args (set by GuaguaYarnClient, if any) */ public static void main(final String[] args) { LOG.info("Starting GuaguaAppMaster. "); String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); Configuration conf = new YarnConfiguration(); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); try { UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.info(token.toString()); } UserGroupInformation appMasterUgi = UserGroupInformation.createRemoteUser(jobUserName); appMasterUgi.addCredentials(credentials); // Now remove the AM->RM token so tasks don't have it Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } final GuaguaAppMaster appMaster = new GuaguaAppMaster(containerId, appAttemptId, conf); appMasterUgi.doAs(new PrivilegedAction<Void>() { @Override public Void run() { boolean result = false; try { result = appMaster.run(); } catch (Throwable t) { LOG.error("GuaguaAppMaster caught a top-level exception in main.", t); System.exit(1); } if (result) { LOG.info("Guagua Application Master completed successfully. exiting"); System.exit(0); } else { LOG.info("Guagua Application Master failed. exiting"); System.exit(2); } return null; } }); } catch (Throwable t) { LOG.error("GuaguaAppMaster caught a top-level exception in main.", t); System.exit(1); } }
From source file:ml.shifu.guagua.yarn.GuaguaYarnTask.java
License:Apache License
public static void main(String[] args) { LOG.info("args:{}", Arrays.toString(args)); if (args.length != 7) { throw new IllegalStateException(String.format( "GuaguaYarnTask could not construct a TaskAttemptID for the Guagua job from args: %s", Arrays.toString(args))); }// w ww . j a v a 2 s . c o m String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); try { Configuration conf = new YarnConfiguration(); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.info(token.toString()); } UserGroupInformation appTaskUGI = UserGroupInformation.createRemoteUser(jobUserName); appTaskUGI.addCredentials(credentials); @SuppressWarnings("rawtypes") final GuaguaYarnTask<?, ?> guaguaYarnTask = new GuaguaYarnTask(appAttemptId, containerId, Integer.parseInt(args[args.length - 3]), args[args.length - 2], args[args.length - 1], conf); appTaskUGI.doAs(new PrivilegedAction<Void>() { @Override public Void run() { guaguaYarnTask.run(); return null; } }); } catch (Throwable t) { LOG.error("GuaguaYarnTask threw a top-level exception, failing task", t); System.exit(2); } System.exit(0); }
From source file:org.apache.asterix.aoya.AsterixApplicationMaster.java
License:Apache License
public void setEnvs(CommandLine cliParser) { Map<String, String> envs = System.getenv(); if (envs.containsKey("HADOOP_CONF_DIR")) { File hadoopConfDir = new File(envs.get("HADOOP_CONF_DIR")); if (hadoopConfDir.isDirectory()) { for (File config : hadoopConfDir.listFiles()) { if (config.getName().matches("^.*(xml)$")) { conf.addResource(new Path(config.getAbsolutePath())); }//from w w w.ja v a 2s . co m } } } //the containerID might be in the arguments or the environment if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException( "Environment is not set correctly- please check client submission settings"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV) || !envs.containsKey(Environment.NM_HOST.name()) || !envs.containsKey(Environment.NM_HTTP_PORT.name()) || !envs.containsKey(Environment.NM_PORT.name())) { throw new IllegalArgumentException( "Environment is not set correctly- please check client submission settings"); } System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, envs.get("PWD") + File.separator + "bin" + File.separator + ASTERIX_CONF_NAME); LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); asterixZipPath = envs.get(AConstants.TARLOCATION); asterixZipTimestamp = Long.parseLong(envs.get(AConstants.TARTIMESTAMP)); asterixZipLen = Long.parseLong(envs.get(AConstants.TARLEN)); asterixConfPath = envs.get(AConstants.CONFLOCATION); asterixConfTimestamp = Long.parseLong(envs.get(AConstants.CONFTIMESTAMP)); asterixConfLen = Long.parseLong(envs.get(AConstants.CONFLEN)); instanceConfPath = envs.get(AConstants.INSTANCESTORE); //the only time this is null is during testing, when asterix-yarn isn't packaged in a JAR yet. if (envs.get(AConstants.APPLICATIONMASTERJARLOCATION) != null && !envs.get(AConstants.APPLICATIONMASTERJARLOCATION).endsWith(File.separator)) { appMasterJar = new Path(envs.get(AConstants.APPLICATIONMASTERJARLOCATION)); } else { appMasterJar = null; } dfsBasePath = envs.get(AConstants.DFS_BASE); //If the NM has an odd environment where the proper hadoop XML configs dont get imported, we can end up not being able to talk to the RM // this solves that! //in a testing environment these can be null however. if (envs.get(AConstants.RMADDRESS) != null) { conf.set("yarn.resourcemanager.address", envs.get(AConstants.RMADDRESS)); LOG.info("RM Address: " + envs.get(AConstants.RMADDRESS)); } if (envs.get(AConstants.RMADDRESS) != null) { conf.set("yarn.resourcemanager.scheduler.address", envs.get(AConstants.RMSCHEDULERADDRESS)); } ccJavaOpts = envs.get(AConstants.CC_JAVA_OPTS); //set defaults if no special given options if (ccJavaOpts == null) { ccJavaOpts = EXTERNAL_CC_JAVA_OPTS_DEFAULT; } ncJavaOpts = envs.get(AConstants.NC_JAVA_OPTS); if (ncJavaOpts == null) { ncJavaOpts = EXTERNAL_NC_JAVA_OPTS_DEFAULT; } LOG.info("Path suffix: " + instanceConfPath); }
From source file:org.apache.flink.yarn.YARNSessionCapacitySchedulerITCase.java
License:Apache License
/** * Test TaskManager failure and also if the vcores are set correctly (see issue FLINK-2213). *///from w w w.j ava 2 s . co m @Test(timeout = 100000) // timeout after 100 seconds public void testTaskManagerFailure() { LOG.info("Starting testTaskManagerFailure()"); Runner runner = startWithArgs( new String[] { "-j", flinkUberjar.getAbsolutePath(), "-t", flinkLibFolder.getAbsolutePath(), "-n", "1", "-jm", "768", "-tm", "1024", "-s", "3", // set the slots 3 to check if the vCores are set properly! "-nm", "customName", "-Dfancy-configuration-value=veryFancy", "-Dyarn.maximum-failed-containers=3", "-D" + ConfigConstants.YARN_VCORES + "=2" }, "Number of connected TaskManagers changed to 1. Slots available: 3", RunTypes.YARN_SESSION); Assert.assertEquals(2, getRunningContainers()); // ------------------------ Test if JobManager web interface is accessible ------- YarnClient yc = null; try { yc = YarnClient.createYarnClient(); yc.init(yarnConfiguration); yc.start(); List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING)); Assert.assertEquals(1, apps.size()); // Only one running ApplicationReport app = apps.get(0); Assert.assertEquals("customName", app.getName()); String url = app.getTrackingUrl(); if (!url.endsWith("/")) { url += "/"; } if (!url.startsWith("http://")) { url = "http://" + url; } LOG.info("Got application URL from YARN {}", url); String response = TestBaseUtils.getFromHTTP(url + "taskmanagers/"); JsonNode parsedTMs = new ObjectMapper().readTree(response); ArrayNode taskManagers = (ArrayNode) parsedTMs.get("taskmanagers"); Assert.assertNotNull(taskManagers); Assert.assertEquals(1, taskManagers.size()); Assert.assertEquals(3, taskManagers.get(0).get("slotsNumber").asInt()); // get the configuration from webinterface & check if the dynamic properties from YARN show up there. String jsonConfig = TestBaseUtils.getFromHTTP(url + "jobmanager/config"); Map<String, String> parsedConfig = WebMonitorUtils.fromKeyValueJsonArray(jsonConfig); Assert.assertEquals("veryFancy", parsedConfig.get("fancy-configuration-value")); Assert.assertEquals("3", parsedConfig.get("yarn.maximum-failed-containers")); Assert.assertEquals("2", parsedConfig.get(ConfigConstants.YARN_VCORES)); // -------------- FLINK-1902: check if jobmanager hostname/port are shown in web interface // first, get the hostname/port String oC = outContent.toString(); Pattern p = Pattern.compile("Flink JobManager is now running on ([a-zA-Z0-9.-]+):([0-9]+)"); Matcher matches = p.matcher(oC); String hostname = null; String port = null; while (matches.find()) { hostname = matches.group(1).toLowerCase(); port = matches.group(2); } LOG.info("Extracted hostname:port: {} {}", hostname, port); Assert.assertEquals("unable to find hostname in " + jsonConfig, hostname, parsedConfig.get(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY)); Assert.assertEquals("unable to find port in " + jsonConfig, port, parsedConfig.get(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY)); // test logfile access String logs = TestBaseUtils.getFromHTTP(url + "jobmanager/log"); Assert.assertTrue(logs.contains("Starting YARN ApplicationMaster")); Assert.assertTrue(logs.contains("Starting JobManager")); Assert.assertTrue(logs.contains("Starting JobManager Web Frontend")); } catch (Throwable e) { LOG.warn("Error while running test", e); Assert.fail(e.getMessage()); } // ------------------------ Kill container with TaskManager and check if vcores are set correctly ------- // find container id of taskManager: ContainerId taskManagerContainer = null; NodeManager nodeManager = null; UserGroupInformation remoteUgi = null; NMTokenIdentifier nmIdent = null; try { remoteUgi = UserGroupInformation.getCurrentUser(); } catch (IOException e) { LOG.warn("Unable to get curr user", e); Assert.fail(); } for (int nmId = 0; nmId < NUM_NODEMANAGERS; nmId++) { NodeManager nm = yarnCluster.getNodeManager(nmId); ConcurrentMap<ContainerId, Container> containers = nm.getNMContext().getContainers(); for (Map.Entry<ContainerId, Container> entry : containers.entrySet()) { String command = Joiner.on(" ").join(entry.getValue().getLaunchContext().getCommands()); if (command.contains(YarnTaskManager.class.getSimpleName())) { taskManagerContainer = entry.getKey(); nodeManager = nm; nmIdent = new NMTokenIdentifier(taskManagerContainer.getApplicationAttemptId(), null, "", 0); // allow myself to do stuff with the container // remoteUgi.addCredentials(entry.getValue().getCredentials()); remoteUgi.addTokenIdentifier(nmIdent); } } sleep(500); } Assert.assertNotNull("Unable to find container with TaskManager", taskManagerContainer); Assert.assertNotNull("Illegal state", nodeManager); try { List<NodeReport> nodeReports = yc.getNodeReports(NodeState.RUNNING); // we asked for one node with 2 vcores so we expect 2 vcores int userVcores = 0; for (NodeReport rep : nodeReports) { userVcores += rep.getUsed().getVirtualCores(); } Assert.assertEquals(2, userVcores); } catch (Exception e) { Assert.fail("Test failed: " + e.getMessage()); } yc.stop(); List<ContainerId> toStop = new LinkedList<ContainerId>(); toStop.add(taskManagerContainer); StopContainersRequest scr = StopContainersRequest.newInstance(toStop); try { nodeManager.getNMContext().getContainerManager().stopContainers(scr); } catch (Throwable e) { LOG.warn("Error stopping container", e); Assert.fail("Error stopping container: " + e.getMessage()); } // stateful termination check: // wait until we saw a container being killed and AFTERWARDS a new one launched boolean ok = false; do { LOG.debug("Waiting for correct order of events. Output: {}", errContent.toString()); String o = errContent.toString(); int killedOff = o.indexOf("Container killed by the ApplicationMaster"); if (killedOff != -1) { o = o.substring(killedOff); ok = o.indexOf("Launching TaskManager") > 0; } sleep(1000); } while (!ok); // send "stop" command to command line interface runner.sendStop(); // wait for the thread to stop try { runner.join(1000); } catch (InterruptedException e) { LOG.warn("Interrupted while stopping runner", e); } LOG.warn("stopped"); // ----------- Send output to logger System.setOut(originalStdout); System.setErr(originalStderr); String oC = outContent.toString(); String eC = errContent.toString(); LOG.info("Sending stdout content through logger: \n\n{}\n\n", oC); LOG.info("Sending stderr content through logger: \n\n{}\n\n", eC); // ------ Check if everything happened correctly Assert.assertTrue("Expect to see failed container", eC.contains("New messages from the YARN cluster")); Assert.assertTrue("Expect to see failed container", eC.contains("Container killed by the ApplicationMaster")); Assert.assertTrue("Expect to see new container started", eC.contains("Launching TaskManager") && eC.contains("on host")); // cleanup auth for the subsequent tests. remoteUgi.getTokenIdentifiers().remove(nmIdent); LOG.info("Finished testTaskManagerFailure()"); }
From source file:org.apache.giraph.yarn.GiraphApplicationMaster.java
License:Apache License
/** * Application entry point/*from ww w . j a v a2 s . c o m*/ * @param args command-line args (set by GiraphYarnClient, if any) */ public static void main(final String[] args) { boolean result = false; LOG.info("Starting GitaphAM "); String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); try { GiraphApplicationMaster giraphAppMaster = new GiraphApplicationMaster(containerId, appAttemptId); result = giraphAppMaster.run(); // CHECKSTYLE: stop IllegalCatch } catch (Throwable t) { // CHECKSTYLE: resume IllegalCatch LOG.error("GiraphApplicationMaster caught a " + "top-level exception in main.", t); System.exit(1); } if (result) { LOG.info("Giraph Application Master completed successfully. exiting"); System.exit(0); } else { LOG.info("Giraph Application Master failed. exiting"); System.exit(2); } }
From source file:org.apache.gobblin.yarn.GobblinApplicationMaster.java
License:Apache License
public GobblinApplicationMaster(String applicationName, ContainerId containerId, Config config, YarnConfiguration yarnConfiguration) throws Exception { super(applicationName, containerId.getApplicationAttemptId().getApplicationId().toString(), GobblinClusterUtils.addDynamicConfig(config), Optional.<Path>absent()); String containerLogDir = config.getString(GobblinYarnConfigurationKeys.LOGS_SINK_ROOT_DIR_KEY); GobblinYarnLogSource gobblinYarnLogSource = new GobblinYarnLogSource(); if (gobblinYarnLogSource.isLogSourcePresent()) { Path appWorkDir = PathUtils.combinePaths(containerLogDir, GobblinClusterUtils.getAppWorkDirPath(this.clusterName, this.applicationId), "AppMaster"); this.applicationLauncher.addService( gobblinYarnLogSource.buildLogCopier(this.config, containerId.toString(), this.fs, appWorkDir)); }/*from ww w . j a v a2 s .co m*/ this.yarnService = buildYarnService(this.config, applicationName, this.applicationId, yarnConfiguration, this.fs); this.applicationLauncher.addService(this.yarnService); if (UserGroupInformation.isSecurityEnabled()) { LOGGER.info("Adding YarnContainerSecurityManager since security is enabled"); this.applicationLauncher.addService(buildYarnContainerSecurityManager(this.config, this.fs)); } // Add additional services List<String> serviceClassNames = ConfigUtils.getStringList(this.config, GobblinYarnConfigurationKeys.APP_MASTER_SERVICE_CLASSES); for (String serviceClassName : serviceClassNames) { Class<?> serviceClass = Class.forName(serviceClassName); this.applicationLauncher .addService((Service) GobblinConstructorUtils.invokeLongestConstructor(serviceClass, this)); } }