List of usage examples for org.apache.hadoop.yarn.api.records ContainerId newInstance
@Private
@Deprecated
@Unstable
public static ContainerId newInstance(ApplicationAttemptId appAttemptId, int containerId)
From source file:org.apache.flink.yarn.YarnFlinkResourceManagerTest.java
License:Apache License
@Test public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Exception { new JavaTestKit(system) { {// www . ja v a2 s . co m final Deadline deadline = new FiniteDuration(3, TimeUnit.MINUTES).fromNow(); Configuration flinkConfig = new Configuration(); YarnConfiguration yarnConfig = new YarnConfiguration(); SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(null, null); String applicationMasterHostName = "localhost"; String webInterfaceURL = "foobar"; ContaineredTaskManagerParameters taskManagerParameters = new ContaineredTaskManagerParameters(1L, 1L, 1L, 1, new HashMap<String, String>()); ContainerLaunchContext taskManagerLaunchContext = mock(ContainerLaunchContext.class); int yarnHeartbeatIntervalMillis = 1000; int maxFailedContainers = 10; int numInitialTaskManagers = 5; final YarnResourceManagerCallbackHandler callbackHandler = new YarnResourceManagerCallbackHandler(); AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = mock(AMRMClientAsync.class); NMClient nodeManagerClient = mock(NMClient.class); UUID leaderSessionID = UUID.randomUUID(); final List<Container> containerList = new ArrayList<>(); for (int i = 0; i < numInitialTaskManagers; i++) { Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(ContainerId.newInstance(ApplicationAttemptId .newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), i)); when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance("container", 1234)); containerList.add(mockContainer); } doAnswer(new Answer() { int counter = 0; @Override public Object answer(InvocationOnMock invocation) throws Throwable { if (counter < containerList.size()) { callbackHandler .onContainersAllocated(Collections.singletonList(containerList.get(counter++))); } return null; } }).when(resourceManagerClient).addContainerRequest(Matchers.any(AMRMClient.ContainerRequest.class)); final CompletableFuture<AkkaActorGateway> resourceManagerFuture = new CompletableFuture<>(); final CompletableFuture<AkkaActorGateway> leaderGatewayFuture = new CompletableFuture<>(); doAnswer((InvocationOnMock invocation) -> { Container container = (Container) invocation.getArguments()[0]; resourceManagerFuture.thenCombine(leaderGatewayFuture, (resourceManagerGateway, leaderGateway) -> { resourceManagerGateway.tell( new NotifyResourceStarted( YarnFlinkResourceManager.extractResourceID(container)), leaderGateway); return null; }); return null; }).when(nodeManagerClient).startContainer(Matchers.any(Container.class), Matchers.any(ContainerLaunchContext.class)); ActorRef resourceManager = null; ActorRef leader1; try { leader1 = system.actorOf(Props.create(TestingUtils.ForwardingActor.class, getRef(), Option.apply(leaderSessionID))); resourceManager = system.actorOf(Props.create(TestingYarnFlinkResourceManager.class, flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers, callbackHandler, resourceManagerClient, nodeManagerClient)); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); final AkkaActorGateway leader1Gateway = new AkkaActorGateway(leader1, leaderSessionID); final AkkaActorGateway resourceManagerGateway = new AkkaActorGateway(resourceManager, leaderSessionID); leaderGatewayFuture.complete(leader1Gateway); resourceManagerFuture.complete(resourceManagerGateway); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> taskManagerRegisteredFuture = resourceManagerGateway .ask(new NotifyWhenResourcesRegistered(numInitialTaskManagers), deadline.timeLeft()); Await.ready(taskManagerRegisteredFuture, deadline.timeLeft()); leaderRetrievalService.notifyListener(null, null); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); for (Container container : containerList) { resourceManagerGateway.tell( new NotifyResourceStarted(YarnFlinkResourceManager.extractResourceID(container)), leader1Gateway); } for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> numberOfRegisteredResourcesFuture = resourceManagerGateway .ask(RequestNumberOfRegisteredResources.INSTANCE, deadline.timeLeft()); int numberOfRegisteredResources = (Integer) Await.result(numberOfRegisteredResourcesFuture, deadline.timeLeft()); assertEquals(numInitialTaskManagers, numberOfRegisteredResources); } finally { if (resourceManager != null) { resourceManager.tell(PoisonPill.getInstance(), ActorRef.noSender()); } } } }; }
From source file:org.apache.flink.yarn.YarnResourceManagerTest.java
License:Apache License
private static Container mockContainer(String host, int port, int containerId) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); ContainerId mockContainerId = ContainerId.newInstance( ApplicationAttemptId.newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), containerId);// w w w .j a v a 2s .com when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockContainer.getResource()).thenReturn(Resource.newInstance(200, 1)); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; }
From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncherTest.java
License:Apache License
/** * Test that the dynamic config is added to the config specified when the {@link GobblinApplicationMaster} * is instantiated.//from www .j a va 2s. c o m */ @Test public void testDynamicConfig() throws Exception { Config config = this.config.withFallback(ConfigFactory.parseMap(ImmutableMap.of( ConfigurationKeys.DYNAMIC_CONFIG_GENERATOR_CLASS_KEY, TestDynamicConfigGenerator.class.getName()))); ContainerId containerId = ContainerId .newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0), 0); TestApplicationMaster appMaster = new TestApplicationMaster("testApp", containerId, config, new YarnConfiguration()); Assert.assertEquals(appMaster.getConfig().getString("dynamicKey1"), "dynamicValue1"); Assert.assertEquals(appMaster.getConfig().getString(ConfigurationKeys.DYNAMIC_CONFIG_GENERATOR_CLASS_KEY), TestDynamicConfigGenerator.class.getName()); ServiceBasedAppLauncher appLauncher = appMaster.getAppLauncher(); Field servicesField = ServiceBasedAppLauncher.class.getDeclaredField("services"); servicesField.setAccessible(true); List<Service> services = (List<Service>) servicesField.get(appLauncher); Optional<Service> yarnServiceOptional = services.stream().filter(e -> e instanceof YarnService).findFirst(); Assert.assertTrue(yarnServiceOptional.isPresent()); YarnService yarnService = (YarnService) yarnServiceOptional.get(); Field configField = YarnService.class.getDeclaredField("config"); configField.setAccessible(true); Config yarnServiceConfig = (Config) configField.get(yarnService); Assert.assertEquals(yarnServiceConfig.getString("dynamicKey1"), "dynamicValue1"); Assert.assertEquals(yarnServiceConfig.getString(ConfigurationKeys.DYNAMIC_CONFIG_GENERATOR_CLASS_KEY), TestDynamicConfigGenerator.class.getName()); }
From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncherTest.java
License:Apache License
/** * Test that the job cleanup call is called *//*from w w w. j av a2 s . co m*/ @Test public void testJobCleanup() throws Exception { ContainerId containerId = ContainerId .newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0), 0); TestApplicationMaster appMaster = Mockito .spy(new TestApplicationMaster("testApp", containerId, config, new YarnConfiguration())); GobblinHelixMultiManager mockMultiManager = Mockito.mock(GobblinHelixMultiManager.class); appMaster.setMultiManager(mockMultiManager); appMaster.start(); Mockito.verify(mockMultiManager, times(1)).cleanUpJobs(); }
From source file:org.apache.gobblin.yarn.YarnServiceTest.java
License:Apache License
@Test(groups = { "gobblin.yarn", "disabledOnTravis" }, dependsOnMethods = "testScaleDown")
public void testReleasedContainerCache() throws Exception {
Config modifiedConfig = this.config.withValue(
GobblinYarnConfigurationKeys.RELEASED_CONTAINERS_CACHE_EXPIRY_SECS,
ConfigValueFactory.fromAnyRef("2"));
TestYarnService yarnService = new TestYarnService(modifiedConfig, "testApp1", "appId1", this.clusterConf,
FileSystem.getLocal(new Configuration()), this.eventBus);
ContainerId containerId1 = ContainerId
.newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 0), 0), 0);
yarnService.getReleasedContainerCache().put(containerId1, "");
Assert.assertTrue(yarnService.getReleasedContainerCache().getIfPresent(containerId1) != null);
// give some time for element to expire
Thread.sleep(4000);//from ww w . jav a 2 s. co m
Assert.assertTrue(yarnService.getReleasedContainerCache().getIfPresent(containerId1) == null);
}
From source file:org.apache.gobblin.yarn.YarnServiceTest.java
License:Apache License
@Test(groups = { "gobblin.yarn", "disabledOnTravis" }, dependsOnMethods = "testReleasedContainerCache")
public void testBuildContainerCommand() throws Exception {
Config modifiedConfig = this.config
.withValue(GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY,
ConfigValueFactory.fromAnyRef("10"))
.withValue(GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY,
ConfigValueFactory.fromAnyRef("0.8"));
TestYarnService yarnService = new TestYarnService(modifiedConfig, "testApp2", "appId2", this.clusterConf,
FileSystem.getLocal(new Configuration()), this.eventBus);
ContainerId containerId = ContainerId
.newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 0), 0), 0);
Resource resource = Resource.newInstance(2048, 1);
Container container = Container.newInstance(containerId, null, null, resource, null, null);
String command = yarnService.buildContainerCommand(container, "helixInstance1");
// 1628 is from 2048 * 0.8 - 10
Assert.assertTrue(command.contains("-Xmx1628"));
}
From source file:org.apache.tez.client.LocalClient.java
License:Apache License
protected Thread createDAGAppMaster(final ApplicationSubmissionContext appContext) { Thread thread = new Thread(new Runnable() { @Override/*from w w w. ja v a2 s . c o m*/ public void run() { try { ApplicationId appId = appContext.getApplicationId(); // Set up working directory for DAGAppMaster Path staging = TezCommonUtils.getTezSystemStagingPath(conf, appId.toString()); Path userDir = TezCommonUtils.getTezSystemStagingPath(conf, appId.toString() + "_wd"); LOG.info("Using working directory: " + userDir.toUri().getPath()); FileSystem fs = FileSystem.get(conf); // copy data from staging directory to working directory to simulate the resource localizing FileUtil.copy(fs, staging, fs, userDir, false, conf); // Prepare Environment Path logDir = new Path(userDir, "localmode-log-dir"); Path localDir = new Path(userDir, "localmode-local-dir"); fs.mkdirs(logDir); fs.mkdirs(localDir); UserGroupInformation.setConfiguration(conf); // Add session specific credentials to the AM credentials. ByteBuffer tokens = appContext.getAMContainerSpec().getTokens(); Credentials amCredentials; if (tokens != null) { amCredentials = TezCommonUtils.parseCredentialsBytes(tokens.array()); } else { amCredentials = new Credentials(); } // Construct, initialize, and start the DAGAppMaster ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 0); ContainerId cId = ContainerId.newInstance(applicationAttemptId, 1); String currentHost = InetAddress.getLocalHost().getHostName(); int nmPort = YarnConfiguration.DEFAULT_NM_PORT; int nmHttpPort = YarnConfiguration.DEFAULT_NM_WEBAPP_PORT; long appSubmitTime = System.currentTimeMillis(); dagAppMaster = createDAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, new SystemClock(), appSubmitTime, isSession, userDir.toUri().getPath(), new String[] { localDir.toUri().getPath() }, new String[] { logDir.toUri().getPath() }, amCredentials, UserGroupInformation.getCurrentUser().getShortUserName()); clientHandler = new DAGClientHandler(dagAppMaster); DAGAppMaster.initAndStartAppMaster(dagAppMaster, conf); } catch (Throwable t) { LOG.fatal("Error starting DAGAppMaster", t); if (dagAppMaster != null) { dagAppMaster.stop(); } amFailException = t; } } }); thread.setName("DAGAppMaster Thread"); LOG.info("DAGAppMaster thread has been created"); return thread; }
From source file:org.apache.tez.dag.app.dag.app.TestTezTaskCommunicatorManager.java
License:Apache License
@SuppressWarnings("deprecation") private ContainerId createContainerId(ApplicationId applicationId, int containerIdx) { ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1); return ContainerId.newInstance(appAttemptId, containerIdx); }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskAttempt.java
License:Apache License
@Test(timeout = 5000) // Ensure the dag does not go into an error state if a attempt kill is // received while STARTING public void testLaunchFailedWhileKilling() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0); TezDAGID dagID = TezDAGID.getInstance(appId, 1); TezVertexID vertexID = TezVertexID.getInstance(dagID, 1); TezTaskID taskID = TezTaskID.getInstance(vertexID, 1); TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 0); MockEventHandler eventHandler = new MockEventHandler(); TaskAttemptListener taListener = mock(TaskAttemptListener.class); when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0)); Configuration taskConf = new Configuration(); taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); taskConf.setBoolean("fs.file.impl.disable.cache", true); TaskLocationHint locationHint = TaskLocationHint .createTaskLocationHint(new HashSet<String>(Arrays.asList(new String[] { "127.0.0.1" })), null); Resource resource = Resource.newInstance(1024, 1); AppContext mockAppContext = mock(AppContext.class); doReturn(new ClusterInfo()).when(mockAppContext).getClusterInfo(); TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, taListener, taskConf, new SystemClock(), mock(TaskHeartbeatHandler.class), mockAppContext, locationHint, false, resource, createFakeContainerContext(), false); NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0)); // At state STARTING. taImpl.handle(new TaskAttemptEventKillRequest(taskAttemptID, null, TaskAttemptTerminationCause.TERMINATED_BY_CLIENT)); // At some KILLING state. taImpl.handle(new TaskAttemptEventKillRequest(taskAttemptID, null, TaskAttemptTerminationCause.TERMINATED_BY_CLIENT)); // taImpl.handle(new TaskAttemptEventContainerTerminating(taskAttemptID, // null));/* www.j a v a 2s .com*/ assertFalse(eventHandler.internalError); }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskAttempt.java
License:Apache License
@Test(timeout = 5000) // Ensure ContainerTerminating and ContainerTerminated is handled correctly by // the TaskAttempt public void testContainerTerminationWhileRunning() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0); TezDAGID dagID = TezDAGID.getInstance(appId, 1); TezVertexID vertexID = TezVertexID.getInstance(dagID, 1); TezTaskID taskID = TezTaskID.getInstance(vertexID, 1); MockEventHandler eventHandler = spy(new MockEventHandler()); TaskAttemptListener taListener = mock(TaskAttemptListener.class); when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0)); Configuration taskConf = new Configuration(); taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); taskConf.setBoolean("fs.file.impl.disable.cache", true); TaskLocationHint locationHint = TaskLocationHint .createTaskLocationHint(new HashSet<String>(Arrays.asList(new String[] { "127.0.0.1" })), null); Resource resource = Resource.newInstance(1024, 1); NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); when(container.getNodeHttpAddress()).thenReturn("localhost:0"); AppContext appCtx = mock(AppContext.class); AMContainerMap containers = new AMContainerMap(mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class), new ContainerContextMatcher(), appCtx); containers.addContainerIfNew(container); doReturn(new ClusterInfo()).when(appCtx).getClusterInfo(); doReturn(containers).when(appCtx).getAllContainers(); TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class); TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, taListener, taskConf, new SystemClock(), mockHeartbeatHandler, appCtx, locationHint, false, resource, createFakeContainerContext(), false); TezTaskAttemptID taskAttemptID = taImpl.getID(); ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class); taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0)); // At state STARTING. taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, contId, null)); assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(), TaskAttemptState.RUNNING); verify(mockHeartbeatHandler).register(taskAttemptID); int expectedEventsAtRunning = 3; verify(eventHandler, times(expectedEventsAtRunning)).handle(arg.capture()); taImpl.handle(new TaskAttemptEventContainerTerminating(taskAttemptID, "Terminating", TaskAttemptTerminationCause.APPLICATION_ERROR)); assertFalse("InternalError occurred trying to handle TA_CONTAINER_TERMINATING", eventHandler.internalError); verify(mockHeartbeatHandler).unregister(taskAttemptID); assertEquals("Task attempt is not in the FAILED state", taImpl.getState(), TaskAttemptState.FAILED); assertEquals(1, taImpl.getDiagnostics().size()); assertEquals("Terminating", taImpl.getDiagnostics().get(0)); assertEquals(TaskAttemptTerminationCause.APPLICATION_ERROR, taImpl.getTerminationCause()); int expectedEvenstAfterTerminating = expectedEventsAtRunning + 3; arg = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture()); verifyEventType(arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), TaskEventTAUpdate.class, 1); verifyEventType(arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1); verifyEventType(arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), DAGEventCounterUpdate.class, 1); taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID, "Terminated", TaskAttemptTerminationCause.CONTAINER_EXITED)); // verify unregister is not invoked again verify(mockHeartbeatHandler, times(1)).unregister(taskAttemptID); int expectedEventAfterTerminated = expectedEvenstAfterTerminating + 0; arg = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(expectedEventAfterTerminated)).handle(arg.capture()); assertEquals(2, taImpl.getDiagnostics().size()); assertEquals("Terminated", taImpl.getDiagnostics().get(1)); // check that original error cause is retained assertEquals(TaskAttemptTerminationCause.APPLICATION_ERROR, taImpl.getTerminationCause()); }