List of usage examples for org.apache.hadoop.yarn.api.records ContainerLaunchContext getLocalResources
@Public @Stable public abstract Map<String, LocalResource> getLocalResources();
LocalResource required by the container. From source file:alluxio.yarn.ApplicationMasterTest.java
License:Apache License
/** * @param expectedContext the context to test for matching * @return an argument matcher which tests for matching the given container launch context *//*from ww w . j a v a2 s . c o m*/ private ArgumentMatcher<ContainerLaunchContext> getContextMatcher( final ContainerLaunchContext expectedContext) { return new ArgumentMatcher<ContainerLaunchContext>() { public boolean matches(Object arg) { if (!(arg instanceof ContainerLaunchContext)) { return false; } ContainerLaunchContext ctx = (ContainerLaunchContext) arg; // Compare only keys for local resources because values include timestamps. return ctx.getLocalResources().keySet().equals(expectedContext.getLocalResources().keySet()) && ctx.getCommands().equals(expectedContext.getCommands()) && ctx.getEnvironment().equals(expectedContext.getEnvironment()); } }; }
From source file:org.apache.flink.yarn.YarnApplicationMasterRunnerTest.java
License:Apache License
@Test public void testCreateTaskExecutorContext() throws Exception { File root = folder.getRoot(); File home = new File(root, "home"); boolean created = home.mkdir(); assertTrue(created);/*from www . j av a 2 s . c o m*/ Answer<?> getDefault = new Answer<Object>() { @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { return invocationOnMock.getArguments()[1]; } }; Configuration flinkConf = new Configuration(); YarnConfiguration yarnConf = mock(YarnConfiguration.class); doAnswer(getDefault).when(yarnConf).get(anyString(), anyString()); doAnswer(getDefault).when(yarnConf).getInt(anyString(), anyInt()); doAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { return new String[] { (String) invocationOnMock.getArguments()[1] }; } }).when(yarnConf).getStrings(anyString(), Mockito.<String>anyVararg()); Map<String, String> env = ImmutableMap.<String, String>builder().put(ENV_APP_ID, "foo") .put(ENV_CLIENT_HOME_DIR, home.getAbsolutePath()).put(ENV_CLIENT_SHIP_FILES, "") .put(ENV_FLINK_CLASSPATH, "").put(ENV_HADOOP_USER_NAME, "foo") .put(FLINK_JAR_PATH, root.toURI().toString()).build(); ContaineredTaskManagerParameters tmParams = mock(ContaineredTaskManagerParameters.class); Configuration taskManagerConf = new Configuration(); String workingDirectory = root.getAbsolutePath(); Class<?> taskManagerMainClass = YarnApplicationMasterRunnerTest.class; ContainerLaunchContext ctx = Utils.createTaskExecutorContext(flinkConf, yarnConf, env, tmParams, taskManagerConf, workingDirectory, taskManagerMainClass, LOG); assertEquals("file", ctx.getLocalResources().get("flink.jar").getResource().getScheme()); }
From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java
License:Apache License
private ApplicationAttemptId allocateAndLaunchQueryMaster(QueryInProgress queryInProgress) throws IOException, YarnException { QueryId queryId = queryInProgress.getQueryId(); ApplicationId appId = ApplicationIdUtils.queryIdToAppId(queryId); LOG.info("Allocate and launch ApplicationMaster for QueryMaster: queryId=" + queryId + ", appId=" + appId); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId);//ww w. ja v a 2 s. c o m // set the application name appContext.setApplicationName("Tajo"); Priority pri = Records.newRecord(Priority.class); pri.setPriority(5); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); ContainerLaunchContext commonContainerLaunchContext = YarnContainerProxy .createCommonContainerLaunchContext(masterContext.getConf(), queryId.toString(), true); // Setup environment by cloning from common env. Map<String, String> env = commonContainerLaunchContext.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env); //////////////////////////////////////////////////////////////////////////// // Set the local resources //////////////////////////////////////////////////////////////////////////// // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command //LOG.info("Setting up app master command"); vargs.add("${JAVA_HOME}" + "/bin/java"); // Set Xmx based on am memory size String jvmOptions = masterContext.getConf().get("tajo.rm.yarn.querymaster.jvm.option", "-Xmx2000m"); for (String eachToken : jvmOptions.split((" "))) { vargs.add(eachToken); } // Set Remote Debugging //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) { //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005"); //} // Set class name vargs.add(TajoWorker.class.getCanonicalName()); vargs.add("qm"); vargs.add(queryId.toString()); // queryId vargs.add(masterContext.getTajoMasterService().getBindAddress().getHostName() + ":" + masterContext.getTajoMasterService().getBindAddress().getPort()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up QueryMasterRunner command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); final Resource resource = Records.newRecord(Resource.class); // TODO - get default value from conf resource.setMemory(2000); resource.setVirtualCores(1); Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>(); ContainerLaunchContext masterContainerContext = BuilderUtils.newContainerLaunchContext( commonContainerLaunchContext.getLocalResources(), myEnv, commands, myServiceData, null, new HashMap<ApplicationAccessType, String>(2)); appContext.setAMContainerSpec(masterContainerContext); LOG.info("Submitting QueryMaster to ResourceManager"); yarnClient.submitApplication(appContext); ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED)); ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); LOG.info("Launching QueryMaster with appAttemptId: " + attemptId); return attemptId; }
From source file:org.apache.tajo.master.YarnContainerProxy.java
License:Apache License
public ContainerLaunchContext createContainerLaunchContext( ContainerLaunchContext commonContainerLaunchContext) { // Setup environment by cloning from common env. Map<String, String> env = commonContainerLaunchContext.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env);/*from w ww.ja va 2 s.c o m*/ // Duplicate the ByteBuffers for access by multiple containers. Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>(); for (Map.Entry<String, ByteBuffer> entry : commonContainerLaunchContext.getServiceData().entrySet()) { myServiceData.put(entry.getKey(), entry.getValue().duplicate()); } //////////////////////////////////////////////////////////////////////////// // Set the local resources //////////////////////////////////////////////////////////////////////////// // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command //LOG.info("Setting up app master command"); vargs.add("${JAVA_HOME}" + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx2000m"); // Set Remote Debugging //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) { //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005"); //} // Set class name //vargs.add(getRunnerClass()); vargs.add(TajoWorker.class.getCanonicalName()); vargs.add("tr"); //workerMode vargs.add(getId()); // subqueryId vargs.add(containerMgrAddress); // nodeId vargs.add(containerID.toString()); // containerId Vector<CharSequence> taskParams = getTaskParams(); if (taskParams != null) { vargs.addAll(taskParams); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up TaskRunner command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); return BuilderUtils.newContainerLaunchContext(commonContainerLaunchContext.getLocalResources(), myEnv, commands, myServiceData, null, new HashMap<ApplicationAccessType, String>()); }
From source file:org.apache.tez.dag.app.rm.container.AMContainerHelpers.java
License:Apache License
@VisibleForTesting public static ContainerLaunchContext createContainerLaunchContext(TezDAGID tezDAGID, Map<String, LocalResource> commonDAGLRs, Map<ApplicationAccessType, String> acls, ContainerId containerId, Map<String, LocalResource> localResources, Map<String, String> vertexEnv, String javaOpts, InetSocketAddress taskAttemptListenerAddress, Credentials credentials, AppContext appContext, Resource containerResource, Configuration conf) { ContainerLaunchContext commonContainerSpec = null; synchronized (commonContainerSpecLock) { if (!commonContainerSpecs.containsKey(tezDAGID)) { commonContainerSpec = createCommonContainerLaunchContext(acls, credentials, commonDAGLRs); commonContainerSpecs.put(tezDAGID, commonContainerSpec); } else {//from w w w . j a v a2s .c om commonContainerSpec = commonContainerSpecs.get(tezDAGID); } // Ensure that we remove container specs for previous AMs to reduce // memory footprint if (lastDAGID == null) { lastDAGID = tezDAGID; } else if (!lastDAGID.equals(tezDAGID)) { commonContainerSpecs.remove(lastDAGID); lastDAGID = tezDAGID; } } // Fill in the fields needed per-container that are missing in the common // spec. Map<String, LocalResource> lResources = new TreeMap<String, LocalResource>(); lResources.putAll(commonContainerSpec.getLocalResources()); lResources.putAll(localResources); // Setup environment by cloning from common env. Map<String, String> env = commonContainerSpec.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env); myEnv.putAll(vertexEnv); String modifiedJavaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(javaOpts, containerResource, conf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION, TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT)); if (LOG.isDebugEnabled()) { if (!modifiedJavaOpts.equals(javaOpts)) { LOG.debug("Modified java opts for container" + ", containerId=" + containerId + ", originalJavaOpts=" + javaOpts + ", modifiedJavaOpts=" + modifiedJavaOpts); } } List<String> commands = TezRuntimeChildJVM.getVMCommand(taskAttemptListenerAddress, containerId.toString(), appContext.getApplicationID().toString(), appContext.getApplicationAttemptId().getAttemptId(), modifiedJavaOpts); // Duplicate the ByteBuffers for access by multiple containers. Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>(); for (Entry<String, ByteBuffer> entry : commonContainerSpec.getServiceData().entrySet()) { myServiceData.put(entry.getKey(), entry.getValue().duplicate()); } // Construct the actual Container ContainerLaunchContext container = ContainerLaunchContext.newInstance(lResources, myEnv, commands, myServiceData, commonContainerSpec.getTokens().duplicate(), acls); return container; }
From source file:org.apache.tez.dag.app.TestMockDAGAppMaster.java
License:Apache License
@Test(timeout = 5000) public void testLocalResourceSetup() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start();//w ww .j a va 2 s . com MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); Map<String, LocalResource> lrDAG = Maps.newHashMap(); String lrName1 = "LR1"; lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Map<String, LocalResource> lrVertex = Maps.newHashMap(); String lrName2 = "LR2"; lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); DAG dag = DAG.create("test").addTaskLocalFiles(lrDAG); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex); dag.addVertex(vA); DAGClient dagClient = tezClient.submitDAG(dag); mockLauncher.waitTillContainersLaunched(); ContainerData cData = mockLauncher.getContainers().values().iterator().next(); ContainerLaunchContext launchContext = cData.launchContext; Map<String, LocalResource> taskLR = launchContext.getLocalResources(); // verify tasks are launched with both DAG and task resources. Assert.assertTrue(taskLR.containsKey(lrName1)); Assert.assertTrue(taskLR.containsKey(lrName2)); mockLauncher.startScheduling(true); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); tezClient.stop(); }
From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ContainerManagerHandler.java
License:Apache License
public StartContainerResponse startContainer(List<String> commands, Map<String, LocalResource> localResources, Map<String, String> env) throws IOException { if (containerManager == null) throw new IllegalStateException("Cannot start a continer before connecting to the container manager!"); ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); ctx.setContainerId(container.getId()); ctx.setResource(container.getResource()); ctx.setLocalResources(localResources); ctx.setCommands(commands);/*w ww. jav a 2s.co m*/ ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); ctx.setEnvironment(env); if (LOG.isDebugEnabled()) { LOG.debug("Using ContainerLaunchContext with" + ", containerId=" + ctx.getContainerId() + ", memory=" + ctx.getResource().getMemory() + ", localResources=" + ctx.getLocalResources().toString() + ", commands=" + ctx.getCommands().toString() + ", env=" + ctx.getEnvironment().toString()); } StartContainerRequest request = Records.newRecord(StartContainerRequest.class); request.setContainerLaunchContext(ctx); LOG.info("Starting container, containerId=" + container.getId().toString() + ", host=" + container.getNodeId().getHost() + ", http=" + container.getNodeHttpAddress()); return containerManager.startContainer(request); }
From source file:tachyon.yarn.ApplicationMasterTest.java
License:Apache License
/** * @param expectedContext the context to test for matching * @return an argument matcher which tests for matching the given container launch context *//* ww w. j a v a 2s .c om*/ private ArgumentMatcher<ContainerLaunchContext> getContextMatcher( final ContainerLaunchContext expectedContext) { return new ArgumentMatcher<ContainerLaunchContext>() { public boolean matches(Object arg) { if (!(arg instanceof ContainerLaunchContext)) { return false; } ContainerLaunchContext ctx = (ContainerLaunchContext) arg; return ctx.getLocalResources().equals(expectedContext.getLocalResources()) && ctx.getCommands().equals(expectedContext.getCommands()) && ctx.getEnvironment().equals(expectedContext.getEnvironment()); } }; }
From source file:yarnkit.container.ContainerLaunchContextFactory.java
License:Apache License
public ContainerLaunchContext duplicate(@CheckForNull ContainerLaunchContext original) { Preconditions.checkNotNull(original, "ContainerLaunchContext should not be null"); ContainerLaunchContext copy = Records.newRecord(ContainerLaunchContext.class); copy.setCommands(original.getCommands()); copy.setEnvironment(original.getEnvironment()); copy.setLocalResources(original.getLocalResources()); ByteBuffer token = original.getTokens(); if (token != null) { copy.setTokens(token.duplicate()); }/*from w ww . j ava 2s .c o m*/ return copy; }