List of usage examples for org.apache.hadoop.yarn.api.records ApplicationAttemptId newInstance
@Public
@Unstable
public static ApplicationAttemptId newInstance(ApplicationId appId, int attemptId)
From source file:io.hops.metadata.util.RMUtilities.java
License:Apache License
/** * Retrieve applications (RMApp, RMAppAttempt) from NDB. MUST be used only * by ResourceTrackerService as some fields of the objects are not set. * * @param rmContext//from www . j a va2s . c om * @param conf * @param applicationId * @return * @throws java.io.IOException */ public static RMApp getRMApp(RMContext rmContext, Configuration conf, String applicationId) throws IOException { //Retrieve all applicationIds from NDB ApplicationState hopAppState = RMUtilities.getApplicationState(applicationId); if (hopAppState != null) { //Create ApplicationState for every application ApplicationId appId = ConverterUtils.toApplicationId(hopAppState.getApplicationid()); ApplicationStateDataPBImpl appStateData = new ApplicationStateDataPBImpl( ApplicationStateDataProto.parseFrom(hopAppState.getAppstate())); RMStateStore.ApplicationState appState = new RMStateStore.ApplicationState(appStateData.getSubmitTime(), appStateData.getStartTime(), appStateData.getApplicationSubmissionContext(), appStateData.getUser(), appStateData.getState(), appStateData.getDiagnostics(), appStateData.getFinishTime(), appStateData.getStateBeforeKilling(), appStateData.getUpdatedNodesId()); LOG.debug("loadRMAppState for app " + appState.getAppId() + " state " + appState.getState()); //Create RMApp //Null fields are not required by ResourceTrackerService RMAppImpl application = new RMAppImpl(appId, rmContext, conf, appState.getApplicationSubmissionContext().getApplicationName(), appState.getUser(), appState.getApplicationSubmissionContext().getQueue(), appState.getApplicationSubmissionContext(), null, null, appState.getSubmitTime(), appState.getApplicationSubmissionContext().getApplicationType(), appState.getApplicationSubmissionContext().getApplicationTags(), null); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, appState.getAttemptCount() + 1); ApplicationAttemptState hopAppAttemptState = RMUtilities.getApplicationAttemptState(applicationId, appAttemptId.toString()); if (hopAppAttemptState != null) { ApplicationAttemptStateDataPBImpl attemptStateData = new ApplicationAttemptStateDataPBImpl( YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProto .parseFrom(hopAppAttemptState.getApplicationattemptstate())); RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId, rmContext, null, null, appState.getApplicationSubmissionContext(), conf, application.getMaxAppAttempts() == application.getAppAttempts().size()); ((RMAppAttemptImpl) attempt).setMasterContainer(attemptStateData.getMasterContainer()); application.addRMAppAttempt(appAttemptId, attempt); return application; } } return null; }
From source file:org.apache.flink.yarn.AbstractYarnClusterTest.java
License:Apache License
private ApplicationReport createApplicationReport(ApplicationId applicationId, YarnApplicationState yarnApplicationState, FinalApplicationStatus finalApplicationStatus) { return ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 0), "user", "queue", "name", "localhost", 42, null, yarnApplicationState, null, null, 1L, 2L, finalApplicationStatus, null, null, 1.0f, null, null); }
From source file:org.apache.flink.yarn.YarnFlinkResourceManagerTest.java
License:Apache License
@Test public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Exception { new JavaTestKit(system) { {/*from w ww . j a va 2s. com*/ final Deadline deadline = new FiniteDuration(3, TimeUnit.MINUTES).fromNow(); Configuration flinkConfig = new Configuration(); YarnConfiguration yarnConfig = new YarnConfiguration(); SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(null, null); String applicationMasterHostName = "localhost"; String webInterfaceURL = "foobar"; ContaineredTaskManagerParameters taskManagerParameters = new ContaineredTaskManagerParameters(1L, 1L, 1L, 1, new HashMap<String, String>()); ContainerLaunchContext taskManagerLaunchContext = mock(ContainerLaunchContext.class); int yarnHeartbeatIntervalMillis = 1000; int maxFailedContainers = 10; int numInitialTaskManagers = 5; final YarnResourceManagerCallbackHandler callbackHandler = new YarnResourceManagerCallbackHandler(); AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = mock(AMRMClientAsync.class); NMClient nodeManagerClient = mock(NMClient.class); UUID leaderSessionID = UUID.randomUUID(); final List<Container> containerList = new ArrayList<>(); for (int i = 0; i < numInitialTaskManagers; i++) { Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(ContainerId.newInstance(ApplicationAttemptId .newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), i)); when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance("container", 1234)); containerList.add(mockContainer); } doAnswer(new Answer() { int counter = 0; @Override public Object answer(InvocationOnMock invocation) throws Throwable { if (counter < containerList.size()) { callbackHandler .onContainersAllocated(Collections.singletonList(containerList.get(counter++))); } return null; } }).when(resourceManagerClient).addContainerRequest(Matchers.any(AMRMClient.ContainerRequest.class)); final CompletableFuture<AkkaActorGateway> resourceManagerFuture = new CompletableFuture<>(); final CompletableFuture<AkkaActorGateway> leaderGatewayFuture = new CompletableFuture<>(); doAnswer((InvocationOnMock invocation) -> { Container container = (Container) invocation.getArguments()[0]; resourceManagerFuture.thenCombine(leaderGatewayFuture, (resourceManagerGateway, leaderGateway) -> { resourceManagerGateway.tell( new NotifyResourceStarted( YarnFlinkResourceManager.extractResourceID(container)), leaderGateway); return null; }); return null; }).when(nodeManagerClient).startContainer(Matchers.any(Container.class), Matchers.any(ContainerLaunchContext.class)); ActorRef resourceManager = null; ActorRef leader1; try { leader1 = system.actorOf(Props.create(TestingUtils.ForwardingActor.class, getRef(), Option.apply(leaderSessionID))); resourceManager = system.actorOf(Props.create(TestingYarnFlinkResourceManager.class, flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers, callbackHandler, resourceManagerClient, nodeManagerClient)); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); final AkkaActorGateway leader1Gateway = new AkkaActorGateway(leader1, leaderSessionID); final AkkaActorGateway resourceManagerGateway = new AkkaActorGateway(resourceManager, leaderSessionID); leaderGatewayFuture.complete(leader1Gateway); resourceManagerFuture.complete(resourceManagerGateway); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> taskManagerRegisteredFuture = resourceManagerGateway .ask(new NotifyWhenResourcesRegistered(numInitialTaskManagers), deadline.timeLeft()); Await.ready(taskManagerRegisteredFuture, deadline.timeLeft()); leaderRetrievalService.notifyListener(null, null); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); for (Container container : containerList) { resourceManagerGateway.tell( new NotifyResourceStarted(YarnFlinkResourceManager.extractResourceID(container)), leader1Gateway); } for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> numberOfRegisteredResourcesFuture = resourceManagerGateway .ask(RequestNumberOfRegisteredResources.INSTANCE, deadline.timeLeft()); int numberOfRegisteredResources = (Integer) Await.result(numberOfRegisteredResourcesFuture, deadline.timeLeft()); assertEquals(numInitialTaskManagers, numberOfRegisteredResources); } finally { if (resourceManager != null) { resourceManager.tell(PoisonPill.getInstance(), ActorRef.noSender()); } } } }; }
From source file:org.apache.flink.yarn.YarnResourceManagerTest.java
License:Apache License
private static Container mockContainer(String host, int port, int containerId) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); ContainerId mockContainerId = ContainerId.newInstance( ApplicationAttemptId.newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), containerId);//from w ww.jav a 2 s .c om when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockContainer.getResource()).thenReturn(Resource.newInstance(200, 1)); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; }
From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncherTest.java
License:Apache License
/** * Test that the dynamic config is added to the config specified when the {@link GobblinApplicationMaster} * is instantiated./* w w w .j a va 2 s . c o m*/ */ @Test public void testDynamicConfig() throws Exception { Config config = this.config.withFallback(ConfigFactory.parseMap(ImmutableMap.of( ConfigurationKeys.DYNAMIC_CONFIG_GENERATOR_CLASS_KEY, TestDynamicConfigGenerator.class.getName()))); ContainerId containerId = ContainerId .newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0), 0); TestApplicationMaster appMaster = new TestApplicationMaster("testApp", containerId, config, new YarnConfiguration()); Assert.assertEquals(appMaster.getConfig().getString("dynamicKey1"), "dynamicValue1"); Assert.assertEquals(appMaster.getConfig().getString(ConfigurationKeys.DYNAMIC_CONFIG_GENERATOR_CLASS_KEY), TestDynamicConfigGenerator.class.getName()); ServiceBasedAppLauncher appLauncher = appMaster.getAppLauncher(); Field servicesField = ServiceBasedAppLauncher.class.getDeclaredField("services"); servicesField.setAccessible(true); List<Service> services = (List<Service>) servicesField.get(appLauncher); Optional<Service> yarnServiceOptional = services.stream().filter(e -> e instanceof YarnService).findFirst(); Assert.assertTrue(yarnServiceOptional.isPresent()); YarnService yarnService = (YarnService) yarnServiceOptional.get(); Field configField = YarnService.class.getDeclaredField("config"); configField.setAccessible(true); Config yarnServiceConfig = (Config) configField.get(yarnService); Assert.assertEquals(yarnServiceConfig.getString("dynamicKey1"), "dynamicValue1"); Assert.assertEquals(yarnServiceConfig.getString(ConfigurationKeys.DYNAMIC_CONFIG_GENERATOR_CLASS_KEY), TestDynamicConfigGenerator.class.getName()); }
From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncherTest.java
License:Apache License
/** * Test that the job cleanup call is called *///from w w w. ja v a 2s .c o m @Test public void testJobCleanup() throws Exception { ContainerId containerId = ContainerId .newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0), 0); TestApplicationMaster appMaster = Mockito .spy(new TestApplicationMaster("testApp", containerId, config, new YarnConfiguration())); GobblinHelixMultiManager mockMultiManager = Mockito.mock(GobblinHelixMultiManager.class); appMaster.setMultiManager(mockMultiManager); appMaster.start(); Mockito.verify(mockMultiManager, times(1)).cleanUpJobs(); }
From source file:org.apache.gobblin.yarn.YarnServiceTest.java
License:Apache License
@Test(groups = { "gobblin.yarn", "disabledOnTravis" }, dependsOnMethods = "testScaleDown")
public void testReleasedContainerCache() throws Exception {
Config modifiedConfig = this.config.withValue(
GobblinYarnConfigurationKeys.RELEASED_CONTAINERS_CACHE_EXPIRY_SECS,
ConfigValueFactory.fromAnyRef("2"));
TestYarnService yarnService = new TestYarnService(modifiedConfig, "testApp1", "appId1", this.clusterConf,
FileSystem.getLocal(new Configuration()), this.eventBus);
ContainerId containerId1 = ContainerId
.newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 0), 0), 0);
yarnService.getReleasedContainerCache().put(containerId1, "");
Assert.assertTrue(yarnService.getReleasedContainerCache().getIfPresent(containerId1) != null);
// give some time for element to expire
Thread.sleep(4000);/* w w w .j a va2s .c o m*/
Assert.assertTrue(yarnService.getReleasedContainerCache().getIfPresent(containerId1) == null);
}
From source file:org.apache.gobblin.yarn.YarnServiceTest.java
License:Apache License
@Test(groups = { "gobblin.yarn", "disabledOnTravis" }, dependsOnMethods = "testReleasedContainerCache")
public void testBuildContainerCommand() throws Exception {
Config modifiedConfig = this.config
.withValue(GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY,
ConfigValueFactory.fromAnyRef("10"))
.withValue(GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY,
ConfigValueFactory.fromAnyRef("0.8"));
TestYarnService yarnService = new TestYarnService(modifiedConfig, "testApp2", "appId2", this.clusterConf,
FileSystem.getLocal(new Configuration()), this.eventBus);
ContainerId containerId = ContainerId
.newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 0), 0), 0);
Resource resource = Resource.newInstance(2048, 1);
Container container = Container.newInstance(containerId, null, null, resource, null, null);
String command = yarnService.buildContainerCommand(container, "helixInstance1");
// 1628 is from 2048 * 0.8 - 10
Assert.assertTrue(command.contains("-Xmx1628"));
}
From source file:org.apache.ignite.yarn.IgniteApplicationMasterSelfTest.java
License:Apache License
/** * @param host Host.//from www. ja v a2 s . c o m * @param cpu Cpu count. * @param mem Memory. * @return Container. */ private Container createContainer(String host, int cpu, int mem) { return Container.newInstance( ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0), ThreadLocalRandom.current().nextLong()), NodeId.newInstance(host, 0), "example.com", new MockResource(mem, cpu), Priority.newInstance(0), null); }
From source file:org.apache.myriad.TestObjectFactory.java
License:Apache License
/** * Returns a new RMContainer corresponding to the RMNode and RMContext. The RMContainer is the * ResourceManager's view of an application container per the Hadoop docs * /*from w w w . j a v a 2 s . co m*/ * @param node * @param context * @param appId * @param cores * @param memory * @return RMContainer */ public static RMContainer getRMContainer(RMNode node, RMContext context, int appId, int cores, int memory) { ContainerId containerId = ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(123456789, 1), 1), appId); Container container = Container.newInstance(containerId, node.getNodeID(), node.getHttpAddress(), Resources.createResource(memory, cores), null, null); return new RMContainerImpl(container, containerId.getApplicationAttemptId(), node.getNodeID(), "user1", context); }