List of usage examples for org.apache.hadoop.yarn.api.records ApplicationAttemptId getApplicationId
@Public @Stable public abstract ApplicationId getApplicationId();
ApplicationId of the ApplicationAttempId. From source file:UnmanagedAMLauncher.java
License:Apache License
public void launchAM(ApplicationAttemptId attemptId) throws IOException, YarnException { Credentials credentials = new Credentials(); Token<AMRMTokenIdentifier> token = rmClient.getAMRMToken(attemptId.getApplicationId()); // Service will be empty but that's okay, we are just passing down only // AMRMToken down to the real AM which eventually sets the correct // service-address. credentials.addToken(token.getService(), token); File tokenFile = File.createTempFile("unmanagedAMRMToken", "", new File(System.getProperty("user.dir"))); try {/* w w w . j a va 2 s . c o m*/ FileUtil.chmod(tokenFile.getAbsolutePath(), "600"); } catch (InterruptedException ex) { throw new RuntimeException(ex); } tokenFile.deleteOnExit(); DataOutputStream os = new DataOutputStream(new FileOutputStream(tokenFile, true)); credentials.writeTokenStorageToStream(os); os.close(); Map<String, String> env = System.getenv(); ArrayList<String> envAMList = new ArrayList<String>(); boolean setClasspath = false; for (Map.Entry<String, String> entry : env.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); if (key.equals("CLASSPATH")) { setClasspath = true; if (classpath != null) { value = value + File.pathSeparator + classpath; } } envAMList.add(key + "=" + value); } if (!setClasspath && classpath != null) { envAMList.add("CLASSPATH=" + classpath); } ContainerId containerId = ContainerId.newContainerId(attemptId, 0); String hostname = InetAddress.getLocalHost().getHostName(); envAMList.add(Environment.CONTAINER_ID.name() + "=" + containerId); envAMList.add(Environment.NM_HOST.name() + "=" + hostname); envAMList.add(Environment.NM_HTTP_PORT.name() + "=0"); envAMList.add(Environment.NM_PORT.name() + "=0"); envAMList.add(Environment.LOCAL_DIRS.name() + "= /tmp"); envAMList.add(ApplicationConstants.APP_SUBMIT_TIME_ENV + "=" + System.currentTimeMillis()); envAMList.add(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME + "=" + tokenFile.getAbsolutePath()); String[] envAM = new String[envAMList.size()]; Process amProc = Runtime.getRuntime().exec(amCmd, envAMList.toArray(envAM)); final BufferedReader errReader = new BufferedReader(new InputStreamReader(amProc.getErrorStream())); final BufferedReader inReader = new BufferedReader(new InputStreamReader(amProc.getInputStream())); // read error and input streams as this would free up the buffers // free the error stream buffer Thread errThread = new Thread() { @Override public void run() { try { String line = errReader.readLine(); while ((line != null) && !isInterrupted()) { System.err.println(line); line = errReader.readLine(); } } catch (IOException ioe) { LOG.warn("Error reading the error stream", ioe); } } }; Thread outThread = new Thread() { @Override public void run() { try { String line = inReader.readLine(); while ((line != null) && !isInterrupted()) { System.out.println(line); line = inReader.readLine(); } } catch (IOException ioe) { LOG.warn("Error reading the out stream", ioe); } } }; try { errThread.start(); outThread.start(); } catch (IllegalStateException ise) { } // wait for the process to finish and check the exit code try { int exitCode = amProc.waitFor(); LOG.info("AM process exited with value: " + exitCode); } catch (InterruptedException e) { e.printStackTrace(); } finally { amCompleted = true; } try { // make sure that the error thread exits // on Windows these threads sometimes get stuck and hang the execution // timeout and join later after destroying the process. errThread.join(); outThread.join(); errReader.close(); inReader.close(); } catch (InterruptedException ie) { LOG.info("ShellExecutor: Interrupted while reading the error/out stream", ie); } catch (IOException ioe) { LOG.warn("Error while closing the error/out stream", ioe); } amProc.destroy(); }
From source file:com.splicemachine.yarn.test.BareYarnTest.java
License:Apache License
/** * All we really need to do here is to create a yarn client, configure it using the same * yarn-site.xml as was used by the server to start up. * @throws YarnException//from www .j a v a 2 s . c o m * @throws IOException */ @Test(timeout = 60000) @Ignore("Broken by dependency change") public void testAMRMClientMatchingFitInferredRack() throws YarnException, IOException { // create, submit new app ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(0); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext( Collections.<String, LocalResource>emptyMap(), new HashMap<String, String>(), Arrays.asList("sleep", "100"), new HashMap<String, ByteBuffer>(), null, new HashMap<ApplicationAccessType, String>()); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class); appRequest.setApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.submitApplication(appContext); // wait for app to start RMAppAttempt appAttempt; while (true) { ApplicationReport appReport = yarnClient.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); appAttempt = yarnPlatform.getResourceManager().getRMContext().getRMApps() .get(attemptId.getApplicationId()).getCurrentAppAttempt(); while (true) { if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) { break; } } break; } } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.setLoginUser( UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken()); }
From source file:de.huberlin.wbi.hiway.am.WorkflowDriver.java
License:Apache License
/** Parse command line arguments, initialize HDFS, manage environment variables. */ public boolean init(String[] args) throws ParseException, IOException, JSONException { DefaultMetricsSystem.initialize("ApplicationMaster"); Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); opts.addOption("u", "summary", true, "The name of the json summary file. No file is created if this parameter is not specified."); opts.addOption("m", "memory", true, "The amount of memory (in MB) to be allocated per worker container. Overrides settings in hiway-site.xml."); opts.addOption("c", "custom", true, "The name of an (optional) JSON file, in which custom amounts of memory can be specified per task."); opts.addOption("s", "scheduler", true, "The scheduling policy that is to be employed. Valid arguments: " + Arrays.toString(HiWayConfiguration.HIWAY_SCHEDULERS.values()) + ". Overrides settings in hiway-site.xml."); opts.addOption("d", "debug", false, "Provide additional logs and information for debugging"); opts.addOption("v", "verbose", false, "Increase verbosity of output / reporting."); opts.addOption("appid", true, "Id of this Application Master."); opts.addOption("h", "help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { Logger.printUsage(opts);/*from www . ja v a2 s.c om*/ throw new IllegalArgumentException("No args specified for application master to initialize"); } if (cliParser.getArgs().length == 0) { Logger.printUsage(opts); throw new IllegalArgumentException("No workflow file specified."); } if (!cliParser.hasOption("appid")) { throw new IllegalArgumentException("No id of Application Master specified"); } if (cliParser.hasOption("verbose")) { HiWayConfiguration.verbose = true; } appId = cliParser.getOptionValue("appid"); try { logger.statLog = new BufferedWriter(new FileWriter(appId + ".log")); } catch (IOException e) { e.printStackTrace(System.out); System.exit(-1); } if (cliParser.hasOption("help")) { Logger.printUsage(opts); return false; } if (cliParser.hasOption("debug")) { Logger.dumpOutDebugInfo(); HiWayConfiguration.debug = true; } if (cliParser.hasOption("summary")) { summaryPath = new Path(cliParser.getOptionValue("summary")); } String hdfsBaseDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE, HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE_DEFAULT); String hdfsSandboxDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE, HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE_DEFAULT); Path hdfsBaseDirectory = new Path(new Path(hdfs.getUri()), hdfsBaseDirectoryName); Data.setHdfsBaseDirectory(hdfsBaseDirectory); Path hdfsSandboxDirectory = new Path(hdfsBaseDirectory, hdfsSandboxDirectoryName); hdfsApplicationDirectory = new Path(hdfsSandboxDirectory, appId); Data.setHdfsApplicationDirectory(hdfsApplicationDirectory); Data.setHdfs(hdfs); if (cliParser.hasOption("custom")) { Data customMemPath = new Data(cliParser.getOptionValue("custom")); customMemPath.stageIn(); StringBuilder sb = new StringBuilder(); try (BufferedReader in = new BufferedReader(new FileReader(customMemPath.getLocalPath().toString()))) { String line; while ((line = in.readLine()) != null) { sb.append(line); } } JSONObject obj = new JSONObject(sb.toString()); Iterator<?> keys = obj.keys(); while (keys.hasNext()) { String key = (String) keys.next(); int minMem = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); int desiredMem = obj.getInt(key); customMemoryMap.put(key, (desiredMem % minMem) == 0 ? desiredMem : (desiredMem / minMem + 1) * minMem); } } Map<String, String> envs = System.getenv(); /* this application's attempt id (combination of attemptId and fail count) */ ApplicationAttemptId appAttemptID; if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } Logger.writeToStdout("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); String shellEnvs[] = conf.getStrings(HiWayConfiguration.HIWAY_WORKER_SHELL_ENV, HiWayConfiguration.HIWAY_WORKER_SHELL_ENV_DEFAULT); for (String env : shellEnvs) { env = env.trim(); int index = env.indexOf('='); if (index == -1) { shellEnv.put(env, ""); continue; } String key = env.substring(0, index); String val = ""; if (index < (env.length() - 1)) { val = env.substring(index + 1); } shellEnv.put(key, val); } String workflowParam = cliParser.getArgs()[0]; try { workflowPath = new Path(new URI(workflowParam).getPath()); } catch (URISyntaxException e) { workflowPath = new Path(workflowParam); } schedulerEnumValue = HiWayConfiguration.HIWAY_SCHEDULERS.valueOf(conf .get(HiWayConfiguration.HIWAY_SCHEDULER, HiWayConfiguration.HIWAY_SCHEDULER_DEFAULT.toString())); if (cliParser.hasOption("scheduler")) { schedulerEnumValue = HiWayConfiguration.HIWAY_SCHEDULERS.valueOf(cliParser.getOptionValue("scheduler")); } containerMemory = conf.getInt(HiWayConfiguration.HIWAY_WORKER_MEMORY, HiWayConfiguration.HIWAY_WORKER_MEMORY_DEFAULT); if (cliParser.hasOption("memory")) { containerMemory = Integer.parseInt(cliParser.getOptionValue("memory")); } containerCores = conf.getInt(HiWayConfiguration.HIWAY_WORKER_VCORES, HiWayConfiguration.HIWAY_WORKER_VCORES_DEFAULT); requestPriority = conf.getInt(HiWayConfiguration.HIWAY_WORKER_PRIORITY, HiWayConfiguration.HIWAY_WORKER_PRIORITY_DEFAULT); // Create and start the Timeline timelineClient if (conf.getBoolean("yarn.timeline-service.enabled", false)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); Logger.writeToStdout("Started TimeLineClient."); } else { Logger.writeToStdErr("TimeLineClient disabled."); } return true; }
From source file:gobblin.yarn.GobblinWorkUnitRunner.java
License:Open Source License
public GobblinWorkUnitRunner(String applicationName, String helixInstanceName, ContainerId containerId, Config config, Optional<Path> appWorkDirOptional) throws Exception { this.helixInstanceName = helixInstanceName; this.config = config; this.containerId = containerId; ApplicationAttemptId applicationAttemptId = this.containerId.getApplicationAttemptId(); String applicationId = applicationAttemptId.getApplicationId().toString(); Configuration conf = HadoopUtils.newConfiguration(); FileSystem fs = buildFileSystem(this.config, conf); String zkConnectionString = config.getString(GobblinYarnConfigurationKeys.ZK_CONNECTION_STRING_KEY); LOGGER.info("Using ZooKeeper connection string: " + zkConnectionString); this.helixManager = HelixManagerFactory.getZKHelixManager( config.getString(GobblinYarnConfigurationKeys.HELIX_CLUSTER_NAME_KEY), helixInstanceName, InstanceType.PARTICIPANT, zkConnectionString); Properties properties = ConfigUtils.configToProperties(config); TaskExecutor taskExecutor = new TaskExecutor(properties); TaskStateTracker taskStateTracker = new GobblinHelixTaskStateTracker(properties, this.helixManager); Path appWorkDir = appWorkDirOptional.isPresent() ? appWorkDirOptional.get() : YarnHelixUtils.getAppWorkDirPath(fs, applicationName, applicationId); List<Service> services = Lists.newArrayList(); if (isLogSourcePresent()) { services.add(buildLogCopier(this.containerId, fs, appWorkDir)); }/*from w ww .ja v a 2 s .c om*/ services.add(taskExecutor); services.add(taskStateTracker); if (config.hasPath(GobblinYarnConfigurationKeys.KEYTAB_FILE_PATH)) { LOGGER.info("Adding YarnContainerSecurityManager since login is keytab based"); services.add(new YarnContainerSecurityManager(config, fs, this.eventBus)); } this.serviceManager = new ServiceManager(services); this.containerMetrics = buildContainerMetrics(this.config, properties, applicationName, containerId); // Register task factory for the Helix task state model Map<String, TaskFactory> taskFactoryMap = Maps.newHashMap(); taskFactoryMap.put(GOBBLIN_TASK_FACTORY_NAME, new GobblinHelixTaskFactory(this.containerMetrics, taskExecutor, taskStateTracker, fs, appWorkDir)); this.taskStateModelFactory = new TaskStateModelFactory(this.helixManager, taskFactoryMap); this.helixManager.getStateMachineEngine().registerStateModelFactory("Task", this.taskStateModelFactory); this.metricRegistry = new MetricRegistry(); this.jmxReporter = JmxReporter.forRegistry(this.metricRegistry).convertRatesTo(TimeUnit.SECONDS) .convertDurationsTo(TimeUnit.MILLISECONDS).build(); }
From source file:org.apache.helix.provisioning.yarn.AppMasterLauncher.java
License:Apache License
public static void main(String[] args) throws Exception { Map<String, String> env = System.getenv(); LOG.info("Starting app master with the following environment variables"); for (String key : env.keySet()) { LOG.info(key + "\t\t=" + env.get(key)); }// www. ja v a 2s .co m Options opts; opts = new Options(); opts.addOption("num_containers", true, "Number of containers"); // START ZOOKEEPER String dataDir = "dataDir"; String logDir = "logDir"; IDefaultNameSpace defaultNameSpace = new IDefaultNameSpace() { @Override public void createDefaultNameSpace(ZkClient zkClient) { } }; try { FileUtils.deleteDirectory(new File(dataDir)); FileUtils.deleteDirectory(new File(logDir)); } catch (IOException e) { LOG.error(e); } final ZkServer server = new ZkServer(dataDir, logDir, defaultNameSpace); server.start(); // start Generic AppMaster that interacts with Yarn RM AppMasterConfig appMasterConfig = new AppMasterConfig(); String containerIdStr = appMasterConfig.getContainerId(); ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId(); String configFile = AppMasterConfig.AppEnvironment.APP_SPEC_FILE.toString(); String className = appMasterConfig.getApplicationSpecFactory(); GenericApplicationMaster genericApplicationMaster = new GenericApplicationMaster(appAttemptID); try { genericApplicationMaster.start(); } catch (Exception e) { LOG.error("Unable to start application master: ", e); } ApplicationSpecFactory factory = HelixYarnUtil.createInstance(className); // TODO: Avoid setting static variable. YarnProvisioner.applicationMaster = genericApplicationMaster; YarnProvisioner.applicationMasterConfig = appMasterConfig; ApplicationSpec applicationSpec = factory.fromYaml(new FileInputStream(configFile)); YarnProvisioner.applicationSpec = applicationSpec; String zkAddress = appMasterConfig.getZKAddress(); String clusterName = appMasterConfig.getAppName(); // CREATE CLUSTER and setup the resources // connect ZkHelixConnection connection = new ZkHelixConnection(zkAddress); connection.connect(); // create the cluster ClusterId clusterId = ClusterId.from(clusterName); ClusterAccessor clusterAccessor = connection.createClusterAccessor(clusterId); StateModelDefinition statelessService = new StateModelDefinition( StateModelConfigGenerator.generateConfigForStatelessService()); StateModelDefinition taskStateModel = new StateModelDefinition( StateModelConfigGenerator.generateConfigForTaskStateModel()); clusterAccessor.createCluster(new ClusterConfig.Builder(clusterId).addStateModelDefinition(statelessService) .addStateModelDefinition(taskStateModel).build()); for (String service : applicationSpec.getServices()) { String resourceName = service; // add the resource with the local provisioner ResourceId resourceId = ResourceId.from(resourceName); ServiceConfig serviceConfig = applicationSpec.getServiceConfig(resourceName); serviceConfig.setSimpleField("service_name", service); int numContainers = serviceConfig.getIntField("num_containers", 1); YarnProvisionerConfig provisionerConfig = new YarnProvisionerConfig(resourceId); provisionerConfig.setNumContainers(numContainers); AutoRebalanceModeISBuilder idealStateBuilder = new AutoRebalanceModeISBuilder(resourceId); idealStateBuilder.setStateModelDefId(statelessService.getStateModelDefId()); idealStateBuilder.add(PartitionId.from(resourceId, "0")); idealStateBuilder.setNumReplica(1); ResourceConfig.Builder resourceConfigBuilder = new ResourceConfig.Builder( ResourceId.from(resourceName)); ResourceConfig resourceConfig = resourceConfigBuilder.provisionerConfig(provisionerConfig) .idealState(idealStateBuilder.build()) // .build(); clusterAccessor.addResource(resourceConfig); } // start controller ControllerId controllerId = ControllerId.from("controller1"); HelixController controller = connection.createController(clusterId, controllerId); controller.start(); // Start any pre-specified jobs List<TaskConfig> taskConfigs = applicationSpec.getTaskConfigs(); if (taskConfigs != null) { YarnConfiguration conf = new YarnConfiguration(); FileSystem fs; fs = FileSystem.get(conf); for (TaskConfig taskConfig : taskConfigs) { URI yamlUri = taskConfig.getYamlURI(); if (yamlUri != null && taskConfig.name != null) { InputStream is = readFromHDFS(fs, taskConfig.name, yamlUri, applicationSpec, appAttemptID.getApplicationId()); Workflow workflow = Workflow.parse(is); TaskDriver taskDriver = new TaskDriver(new ZKHelixManager(controller)); taskDriver.start(workflow); } } } Thread shutdownhook = new Thread(new Runnable() { @Override public void run() { server.shutdown(); } }); Runtime.getRuntime().addShutdownHook(shutdownhook); Thread.sleep(10000); }
From source file:org.apache.tajo.util.TajoIdUtils.java
License:Apache License
public static QueryId createQueryId(ApplicationAttemptId appAttemptId) { QueryId queryId = new QueryId(); queryId.setApplicationId(appAttemptId.getApplicationId()); queryId.setAttemptId(appAttemptId.getAttemptId()); return queryId; }
From source file:org.apache.tajo.yarn.ApplicationMaster.java
License:Apache License
/** * Parse command line options//from w w w . j a va 2 s. c om * * @param args Command line args * @return Whether init successful and getLaunchContext should be invoked * @throws org.apache.commons.cli.ParseException * @throws java.io.IOException */ public boolean init(String[] args) throws ParseException, IOException { Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); opts.addOption("qm_memory", true, "Amount of memory in MB to be requested to launch a QueryMaster. Default 512"); opts.addOption("qm_vcores", true, "Amount of virtual cores to be requested to launch a QueryMaster. Default 2"); opts.addOption("tr_memory", true, "Amount of memory in MB to be requested to launch a TaskRunner. Default 1024"); opts.addOption("tr_vcores", true, "Amount of virtual cores to be requested to launch a TaskRunner. Default 4"); opts.addOption("worker_memory", true, "Amount of memory in MB to be requested to launch a worker. Default 2048"); opts.addOption("worker_vcores", true, "Amount of virtual cores to be requested to launch a worker. Default 4"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); // Check whether customer log4j.properties file exists if (fileExist(log4jPath)) { try { Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath); } catch (Exception e) { LOG.warn("Can not set up custom log4j properties. " + e); } } if (cliParser.hasOption("help")) { printUsage(opts); return false; } ApplicationAttemptId appAttemptID = null; Map<String, String> envs = System.getenv(); if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); int qmMemory = Integer.parseInt(cliParser.getOptionValue("qm_memory", "512")); int qmVCores = Integer.parseInt(cliParser.getOptionValue("qm_vcores", "2")); int trMemory = Integer.parseInt(cliParser.getOptionValue("tr_memory", "1024")); int trVCores = Integer.parseInt(cliParser.getOptionValue("tr_vcores", "4")); int workerMemory = Integer.parseInt(cliParser.getOptionValue("worker_memory", "2048")); int workerVCores = Integer.parseInt(cliParser.getOptionValue("worker_vcores", "4")); int requestPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); String appMasterHostName = InetAddress.getLocalHost().getHostName(); this.appContext = new AppContext(conf, appAttemptID, workerMemory, workerVCores, requestPriority, appMasterHostName); return true; }
From source file:org.apache.tez.dag.app.rm.ContainerFactory.java
License:Apache License
public ContainerFactory(ApplicationAttemptId appAttemptId, long appIdLong) { this.nextId = new AtomicLong(1); ApplicationId appId = ApplicationId.newInstance(appIdLong, appAttemptId.getApplicationId().getId()); this.customAppAttemptId = ApplicationAttemptId.newInstance(appId, appAttemptId.getAttemptId()); }
From source file:org.apache.tez.dag.history.logging.proto.HistoryEventProtoConverter.java
License:Apache License
private HistoryEventProto.Builder makeBuilderForEvent(HistoryEvent event, long time, TezDAGID dagId, ApplicationId appId, ApplicationAttemptId appAttemptId, TezVertexID vertexId, TezTaskID taskId, TezTaskAttemptID taskAttemptId, String user) { HistoryEventProto.Builder builder = HistoryEventProto.newBuilder(); builder.setEventType(event.getEventType().name()); builder.setEventTime(time);/*w w w . j a va 2s . co m*/ if (taskAttemptId != null) { builder.setTaskAttemptId(taskAttemptId.toString()); taskId = taskAttemptId.getTaskID(); } if (taskId != null) { builder.setTaskId(taskId.toString()); vertexId = taskId.getVertexID(); } if (vertexId != null) { builder.setVertexId(vertexId.toString()); dagId = vertexId.getDAGId(); } if (dagId != null) { builder.setDagId(dagId.toString()); if (appId == null) { appId = dagId.getApplicationId(); } } if (appAttemptId != null) { builder.setAppAttemptId(appAttemptId.toString()); if (appId == null) { appId = appAttemptId.getApplicationId(); } } if (appId != null) { builder.setAppId(appId.toString()); } if (user != null) { builder.setUser(user); } return builder; }
From source file:org.springframework.yarn.am.monitor.DefaultContainerMonitorTests.java
License:Apache License
/** * Mock {@link ApplicationAttemptId}//from w ww . jav a 2 s. c o m * * @param appId the app id * @param attemptId the app attempt id * @return mocked {@link ApplicationAttemptId} */ public static ApplicationAttemptId getMockApplicationAttemptId(int appId, int attemptId) { ApplicationId applicationId = mock(ApplicationId.class); when(applicationId.getClusterTimestamp()).thenReturn(0L); when(applicationId.getId()).thenReturn(appId); ApplicationAttemptId applicationAttemptId = mock(ApplicationAttemptId.class); when(applicationAttemptId.getApplicationId()).thenReturn(applicationId); when(applicationAttemptId.getAttemptId()).thenReturn(attemptId); return applicationAttemptId; }