List of usage examples for org.apache.hadoop.yarn.api.records ContainerLaunchContext getEnvironment
@Public @Stable public abstract Map<String, String> getEnvironment();
From source file:alluxio.yarn.ApplicationMasterTest.java
License:Apache License
/** * @param expectedContext the context to test for matching * @return an argument matcher which tests for matching the given container launch context *//*from w w w . j a v a2s . com*/ private ArgumentMatcher<ContainerLaunchContext> getContextMatcher( final ContainerLaunchContext expectedContext) { return new ArgumentMatcher<ContainerLaunchContext>() { public boolean matches(Object arg) { if (!(arg instanceof ContainerLaunchContext)) { return false; } ContainerLaunchContext ctx = (ContainerLaunchContext) arg; // Compare only keys for local resources because values include timestamps. return ctx.getLocalResources().keySet().equals(expectedContext.getLocalResources().keySet()) && ctx.getCommands().equals(expectedContext.getCommands()) && ctx.getEnvironment().equals(expectedContext.getEnvironment()); } }; }
From source file:org.apache.flink.yarn.YarnResourceManager.java
License:Apache License
private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host) throws Exception { // init the ContainerLaunchContext final String currDir = ENV.get(ApplicationConstants.Environment.PWD.key()); final ContaineredTaskManagerParameters taskManagerParameters = ContaineredTaskManagerParameters .create(flinkConfig, resource.getMemory(), 1); LOG.info(/*from ww w.j av a2 s . c o m*/ "TaskExecutor{} will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB", containerId, taskManagerParameters.taskManagerTotalMemoryMB(), taskManagerParameters.taskManagerHeapSizeMB(), taskManagerParameters.taskManagerDirectMemoryLimitMB()); int timeout = flinkConfig.getInteger(ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION, DEFAULT_TASK_MANAGER_REGISTRATION_DURATION); FiniteDuration teRegistrationTimeout = new FiniteDuration(timeout, TimeUnit.SECONDS); final Configuration taskManagerConfig = BootstrapTools.generateTaskManagerConfiguration(flinkConfig, "", 0, 1, teRegistrationTimeout); LOG.debug("TaskManager configuration: {}", taskManagerConfig); ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext(flinkConfig, yarnConfig, ENV, taskManagerParameters, taskManagerConfig, currDir, YarnTaskExecutorRunner.class, LOG); // set a special environment variable to uniquely identify this container taskExecutorLaunchContext.getEnvironment().put(ENV_FLINK_CONTAINER_ID, containerId); taskExecutorLaunchContext.getEnvironment().put(ENV_FLINK_NODE_ID, host); return taskExecutorLaunchContext; }
From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java
License:Apache License
private ApplicationAttemptId allocateAndLaunchQueryMaster(QueryInProgress queryInProgress) throws IOException, YarnException { QueryId queryId = queryInProgress.getQueryId(); ApplicationId appId = ApplicationIdUtils.queryIdToAppId(queryId); LOG.info("Allocate and launch ApplicationMaster for QueryMaster: queryId=" + queryId + ", appId=" + appId); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId);//from w w w . j av a 2 s. c om // set the application name appContext.setApplicationName("Tajo"); Priority pri = Records.newRecord(Priority.class); pri.setPriority(5); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); ContainerLaunchContext commonContainerLaunchContext = YarnContainerProxy .createCommonContainerLaunchContext(masterContext.getConf(), queryId.toString(), true); // Setup environment by cloning from common env. Map<String, String> env = commonContainerLaunchContext.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env); //////////////////////////////////////////////////////////////////////////// // Set the local resources //////////////////////////////////////////////////////////////////////////// // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command //LOG.info("Setting up app master command"); vargs.add("${JAVA_HOME}" + "/bin/java"); // Set Xmx based on am memory size String jvmOptions = masterContext.getConf().get("tajo.rm.yarn.querymaster.jvm.option", "-Xmx2000m"); for (String eachToken : jvmOptions.split((" "))) { vargs.add(eachToken); } // Set Remote Debugging //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) { //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005"); //} // Set class name vargs.add(TajoWorker.class.getCanonicalName()); vargs.add("qm"); vargs.add(queryId.toString()); // queryId vargs.add(masterContext.getTajoMasterService().getBindAddress().getHostName() + ":" + masterContext.getTajoMasterService().getBindAddress().getPort()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up QueryMasterRunner command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); final Resource resource = Records.newRecord(Resource.class); // TODO - get default value from conf resource.setMemory(2000); resource.setVirtualCores(1); Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>(); ContainerLaunchContext masterContainerContext = BuilderUtils.newContainerLaunchContext( commonContainerLaunchContext.getLocalResources(), myEnv, commands, myServiceData, null, new HashMap<ApplicationAccessType, String>(2)); appContext.setAMContainerSpec(masterContainerContext); LOG.info("Submitting QueryMaster to ResourceManager"); yarnClient.submitApplication(appContext); ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED)); ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); LOG.info("Launching QueryMaster with appAttemptId: " + attemptId); return attemptId; }
From source file:org.apache.tajo.master.YarnContainerProxy.java
License:Apache License
public ContainerLaunchContext createContainerLaunchContext( ContainerLaunchContext commonContainerLaunchContext) { // Setup environment by cloning from common env. Map<String, String> env = commonContainerLaunchContext.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env);/* w ww. j ava 2s . co m*/ // Duplicate the ByteBuffers for access by multiple containers. Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>(); for (Map.Entry<String, ByteBuffer> entry : commonContainerLaunchContext.getServiceData().entrySet()) { myServiceData.put(entry.getKey(), entry.getValue().duplicate()); } //////////////////////////////////////////////////////////////////////////// // Set the local resources //////////////////////////////////////////////////////////////////////////// // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command //LOG.info("Setting up app master command"); vargs.add("${JAVA_HOME}" + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx2000m"); // Set Remote Debugging //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) { //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005"); //} // Set class name //vargs.add(getRunnerClass()); vargs.add(TajoWorker.class.getCanonicalName()); vargs.add("tr"); //workerMode vargs.add(getId()); // subqueryId vargs.add(containerMgrAddress); // nodeId vargs.add(containerID.toString()); // containerId Vector<CharSequence> taskParams = getTaskParams(); if (taskParams != null) { vargs.addAll(taskParams); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up TaskRunner command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); return BuilderUtils.newContainerLaunchContext(commonContainerLaunchContext.getLocalResources(), myEnv, commands, myServiceData, null, new HashMap<ApplicationAccessType, String>()); }
From source file:org.apache.tez.dag.app.rm.container.AMContainerHelpers.java
License:Apache License
@VisibleForTesting public static ContainerLaunchContext createContainerLaunchContext(TezDAGID tezDAGID, Map<String, LocalResource> commonDAGLRs, Map<ApplicationAccessType, String> acls, ContainerId containerId, Map<String, LocalResource> localResources, Map<String, String> vertexEnv, String javaOpts, InetSocketAddress taskAttemptListenerAddress, Credentials credentials, AppContext appContext, Resource containerResource, Configuration conf) { ContainerLaunchContext commonContainerSpec = null; synchronized (commonContainerSpecLock) { if (!commonContainerSpecs.containsKey(tezDAGID)) { commonContainerSpec = createCommonContainerLaunchContext(acls, credentials, commonDAGLRs); commonContainerSpecs.put(tezDAGID, commonContainerSpec); } else {//from www .java2 s .co m commonContainerSpec = commonContainerSpecs.get(tezDAGID); } // Ensure that we remove container specs for previous AMs to reduce // memory footprint if (lastDAGID == null) { lastDAGID = tezDAGID; } else if (!lastDAGID.equals(tezDAGID)) { commonContainerSpecs.remove(lastDAGID); lastDAGID = tezDAGID; } } // Fill in the fields needed per-container that are missing in the common // spec. Map<String, LocalResource> lResources = new TreeMap<String, LocalResource>(); lResources.putAll(commonContainerSpec.getLocalResources()); lResources.putAll(localResources); // Setup environment by cloning from common env. Map<String, String> env = commonContainerSpec.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env); myEnv.putAll(vertexEnv); String modifiedJavaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(javaOpts, containerResource, conf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION, TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT)); if (LOG.isDebugEnabled()) { if (!modifiedJavaOpts.equals(javaOpts)) { LOG.debug("Modified java opts for container" + ", containerId=" + containerId + ", originalJavaOpts=" + javaOpts + ", modifiedJavaOpts=" + modifiedJavaOpts); } } List<String> commands = TezRuntimeChildJVM.getVMCommand(taskAttemptListenerAddress, containerId.toString(), appContext.getApplicationID().toString(), appContext.getApplicationAttemptId().getAttemptId(), modifiedJavaOpts); // Duplicate the ByteBuffers for access by multiple containers. Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>(); for (Entry<String, ByteBuffer> entry : commonContainerSpec.getServiceData().entrySet()) { myServiceData.put(entry.getKey(), entry.getValue().duplicate()); } // Construct the actual Container ContainerLaunchContext container = ContainerLaunchContext.newInstance(lResources, myEnv, commands, myServiceData, commonContainerSpec.getTokens().duplicate(), acls); return container; }
From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ContainerManagerHandler.java
License:Apache License
public StartContainerResponse startContainer(List<String> commands, Map<String, LocalResource> localResources, Map<String, String> env) throws IOException { if (containerManager == null) throw new IllegalStateException("Cannot start a continer before connecting to the container manager!"); ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); ctx.setContainerId(container.getId()); ctx.setResource(container.getResource()); ctx.setLocalResources(localResources); ctx.setCommands(commands);//from ww w . j ava2 s.co m ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); ctx.setEnvironment(env); if (LOG.isDebugEnabled()) { LOG.debug("Using ContainerLaunchContext with" + ", containerId=" + ctx.getContainerId() + ", memory=" + ctx.getResource().getMemory() + ", localResources=" + ctx.getLocalResources().toString() + ", commands=" + ctx.getCommands().toString() + ", env=" + ctx.getEnvironment().toString()); } StartContainerRequest request = Records.newRecord(StartContainerRequest.class); request.setContainerLaunchContext(ctx); LOG.info("Starting container, containerId=" + container.getId().toString() + ", host=" + container.getNodeId().getHost() + ", http=" + container.getNodeHttpAddress()); return containerManager.startContainer(request); }
From source file:org.springframework.yarn.batch.am.AbstractBatchAppmaster.java
License:Apache License
@Override public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) { AppmasterService service = getAppmasterService(); if (log.isDebugEnabled()) { log.debug("Intercept launch context: " + context); }/*from www .j av a 2 s. co m*/ StepExecution stepExecution = containerToStepMap.get(container.getId()); String jobName = remoteStepNames.get(stepExecution); if (service != null) { int port = service.getPort(); String address = service.getHost(); Map<String, String> env = new HashMap<String, String>(context.getEnvironment()); env.put(YarnSystemConstants.FS_ADDRESS, getConfiguration().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)); env.put(YarnSystemConstants.AMSERVICE_PORT, Integer.toString(port)); env.put(YarnSystemConstants.AMSERVICE_HOST, address); env.put(YarnSystemConstants.AMSERVICE_BATCH_STEPNAME, jobName); env.put(YarnSystemConstants.AMSERVICE_BATCH_JOBEXECUTIONID, Long.toString(stepExecution.getJobExecutionId())); env.put(YarnSystemConstants.AMSERVICE_BATCH_STEPEXECUTIONID, Long.toString(stepExecution.getId())); context.setEnvironment(env); return context; } else { return context; } }
From source file:org.springframework.yarn.examples.CustomAppmaster.java
License:Apache License
@Override public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) { ContainerId containerId = container.getId(); Integer attempt = 1;//w ww. j a v a 2 s .c om ContainerId failedContainerId = failed.poll(); Object assignedData = (failedContainerId != null ? getContainerAssign().getAssignedData(failedContainerId) : null); if (assignedData != null) { attempt = (Integer) assignedData; attempt += 1; } getContainerAssign().assign(containerId, attempt); Map<String, String> env = new HashMap<String, String>(context.getEnvironment()); env.put("customappmaster_attempt", attempt.toString()); context.setEnvironment(env); return context; }
From source file:org.springframework.yarn.examples.ThriftHeartbeatAppmaster.java
License:Apache License
@Override public ContainerLaunchContext preLaunch(ContainerLaunchContext context) { log.info("preLaunch: " + context); AppmasterService service = getAppmasterService(); if (service != null) { int port = service.getPort(); String address = service.getHost(); Map<String, String> env = new HashMap<String, String>(context.getEnvironment()); env.put(YarnSystemConstants.AMSERVICE_PORT, Integer.toString(port)); env.put(YarnSystemConstants.AMSERVICE_HOST, address); // for now always use same id env.put("syarn.amservice.nodeid", "1"); context.setEnvironment(env);/*from w ww .j a v a 2 s . c o m*/ return context; } else { return context; } // below uncommented code is from other example which // tracked failed containers by assigning 'some' data // with container id order to have some understanding // what was the actual task container was doing. // // I believe we should add similar info to thrift // so that when heartbeat system notifies that container is // down, we'd know which container was it and what // was it doing. // ContainerId containerId = context.getContainerId(); // Integer attempt = 1; // ContainerId failedContainerId = failed.poll(); // Object assignedData = (failedContainerId != null ? getContainerAssing().getAssignedData(failedContainerId) : null); // if (assignedData != null) { // attempt = (Integer) assignedData; // attempt += 1; // } // getContainerAssing().assign(containerId, attempt); // // Map<String, String> env = new HashMap<String, String>(context.getEnvironment()); // env.put("customappmaster.attempt", attempt.toString()); // context.setEnvironment(env); // return context; }
From source file:org.springframework.yarn.examples.XdAppmaster.java
License:Apache License
@Override public ContainerLaunchContext preLaunch(ContainerLaunchContext context) { if (log.isDebugEnabled()) { log.debug("preLaunch: " + context); }/*from ww w . j a va 2 s .c o m*/ AppmasterService service = getAppmasterService(); if (service != null) { int port = service.getPort(); String address = service.getHost(); Map<String, String> env = new HashMap<String, String>(context.getEnvironment()); env.put(YarnSystemConstants.AMSERVICE_PORT, Integer.toString(port)); env.put(YarnSystemConstants.AMSERVICE_HOST, address); env.put(YarnSystemConstants.SYARN_CONTAINER_ID, ConverterUtils.toString(context.getContainerId())); // String xdGroup = getManagedGroups().findGroupNameByMember(ConverterUtils.toString(context.getContainerId())); String xdGroup = getManagedGroups().getGroupByMember(ConverterUtils.toString(context.getContainerId())) .getId(); env.put("syarn.cg.group", xdGroup != null ? xdGroup : ""); context.setEnvironment(env); // testing try { Credentials credentials = new Credentials(); credentials.addSecretKey(new Text(YarnSystemConstants.SYARN_SEC_SESSIONID), sessionId.getBytes()); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer containerToken = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); context.setContainerTokens(containerToken); } catch (IOException e) { log.error("XXX error setContainerTokens", e); } return context; } else { return context; } }