List of usage examples for org.apache.hadoop.yarn.api.records NodeId newInstance
@Public
@Stable
public static NodeId newInstance(String host, int port)
From source file:alluxio.yarn.ApplicationMasterTest.java
License:Apache License
/** * Tests that the correct type and number of containers are requested. *///from ww w. jav a2s .c o m @Test(timeout = 10000) public void requestContainersOnceTest() throws Exception { // Mock the Yarn client to give a NodeReport with NUM_WORKERS nodes List<NodeReport> nodeReports = Lists.newArrayList(); final List<String> nodeHosts = Lists.newArrayList(); for (int i = 0; i < NUM_WORKERS; i++) { String host = "host" + i; NodeReport report = Mockito.mock(NodeReport.class); Mockito.when(report.getNodeId()).thenReturn(NodeId.newInstance(host, 0)); nodeReports.add(report); nodeHosts.add(host); } // We need to use anyVararg because Mockito is dumb and assumes that an array argument must be // vararg. Using regular any() will only match an array if it has length 1. Mockito.when(mYarnClient.getNodeReports(Matchers.<NodeState[]>anyVararg())).thenReturn(nodeReports); // Mock the Resource Manager to "allocate" containers when they are requested and update // ApplicationMaster internal state Mockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { mPrivateAccess.getMasterAllocated().countDown(); return null; } }).when(mRMClient).addContainerRequest(Mockito.argThat(getMasterContainerMatcher())); Mockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { Multiset<String> workerNodes = mPrivateAccess.getWorkerHosts(); synchronized (workerNodes) { workerNodes.add("host-" + UUID.randomUUID()); mPrivateAccess.getOutstandingWorkerContainerReqeustsLatch().countDown(); workerNodes.notify(); if (workerNodes.size() == NUM_WORKERS) { // Once all workers are allocated, we shut down the master so that // requestContainers() doesn't run forever mMaster.onShutdownRequest(); } } return null; } }).when(mRMClient).addContainerRequest(Mockito.argThat(getWorkerContainerMatcher(nodeHosts))); // This will hang if incorrect worker container requests are made mMaster.requestContainers(); // Verify that the right types and numbers of containers were requested Mockito.verify(mRMClient).addContainerRequest(Mockito.argThat(getMasterContainerMatcher())); Mockito.verify(mRMClient, Mockito.times(NUM_WORKERS)) .addContainerRequest(Mockito.argThat(getWorkerContainerMatcher(nodeHosts))); }
From source file:alluxio.yarn.ApplicationMasterTest.java
License:Apache License
/** * Mocks mRMClient to randomly allocated one of the requested hosts. * * This involves/*from w w w.j a va 2 s . c o m*/ * 1) Creating NUM_WORKERS mock containers, each with a different mock host * 2) Mocking mYarnClient to return the mock hosts of the mock containers * 3) Mocking mRMClient.addContainerRequest to asynchronously call mMaster.onContainersAllocated * with a random container on a requested host * * @param numContainers the number of mock container hosts */ private void mockResourceManager(int numContainers) throws Exception { final Random random = new Random(); final List<Container> mockContainers = Lists.newArrayList(); List<NodeReport> nodeReports = Lists.newArrayList(); List<String> hosts = Lists.newArrayList(MASTER_ADDRESS); for (int i = 0; i < numContainers - 1; i++) { String host = "host" + i; hosts.add(host); } for (String host : hosts) { Container mockContainer = Mockito.mock(Container.class); Mockito.when(mockContainer.getNodeHttpAddress()).thenReturn(host + ":8042"); Mockito.when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance(host, 0)); mockContainers.add(mockContainer); NodeReport report = Mockito.mock(NodeReport.class); Mockito.when(report.getNodeId()).thenReturn(NodeId.newInstance(host, 0)); nodeReports.add(report); } // We need to use anyVararg because Mockito is dumb and assumes that an array argument must be // vararg. Using regular any() will only match an array if it has length 1. Mockito.when(mYarnClient.getNodeReports(Matchers.<NodeState[]>anyVararg())).thenReturn(nodeReports); // Pretend to be the Resource Manager, allocating containers when they are requested. Mockito.doAnswer(new Answer<Void>() { @Override public Void answer(final InvocationOnMock invocation) { new Thread(new Runnable() { @Override public void run() { // Allow the requests to interleave randomly CommonUtils.sleepMs(50 + random.nextInt(200)); // Allocate a randomly chosen container from among the requested hosts ContainerRequest request = invocation.getArgumentAt(0, ContainerRequest.class); Set<String> requestedHosts = Sets.newHashSet(request.getNodes()); List<Container> requestedContainers = Lists.newArrayList(); for (Container container : mockContainers) { if (requestedHosts.contains(container.getNodeId().getHost())) { requestedContainers.add(container); } } mMaster.onContainersAllocated(Lists .newArrayList(requestedContainers.get(random.nextInt(requestedContainers.size())))); } }).start(); return null; } }).when(mRMClient).addContainerRequest(Mockito.<ContainerRequest>any()); }
From source file:alluxio.yarn.ApplicationMasterTest.java
License:Apache License
/** * Tests that the Alluxio worker containers are launched properly. *///from w w w .ja v a 2 s . c o m @Test public void launchAlluxioWorkerContainersTest() throws Exception { Container mockContainer1 = Mockito.mock(Container.class); Container mockContainer2 = Mockito.mock(Container.class); // The containers must be from different hosts because we don't support multiple clients on the // same host. Mockito.when(mockContainer1.getNodeId()).thenReturn(NodeId.newInstance("host1", 0)); Mockito.when(mockContainer2.getNodeId()).thenReturn(NodeId.newInstance("host2", 0)); // Say that the master is allocated so that container offers are assumed to be worker offers mPrivateAccess.getMasterAllocated().countDown(); mPrivateAccess.setMasterContainerAddress("masterAddress"); mPrivateAccess.setOutstandingWorkerContainerRequestsLatch(new CountDownLatch(2)); List<Container> containers = Lists.newArrayList(mockContainer1, mockContainer2); mMaster.onContainersAllocated(containers); // Generate the context that we expect Yarn to launch workers with. Map<String, String> expectedWorkerEnvironment = ImmutableMap.<String, String>builder() .put("ALLUXIO_HOME", ApplicationConstants.Environment.PWD.$()) .put("ALLUXIO_MASTER_HOSTNAME", "masterAddress") .put("ALLUXIO_WORKER_MEMORY_SIZE", Integer.toString(RAMDISK_MEM_MB) + ".00MB").build(); String expectedWorkerCommand = "./alluxio-yarn-setup.sh alluxio-worker 1><LOG_DIR>/stdout 2><LOG_DIR>/stderr "; ContainerLaunchContext expectedWorkerContext = ContainerLaunchContext.newInstance( getExpectedLocalResources(), expectedWorkerEnvironment, Lists.newArrayList(expectedWorkerCommand), null, null, null); Mockito.verify(mNMClient).startContainer(Mockito.same(mockContainer1), Mockito.argThat(getContextMatcher(expectedWorkerContext))); Mockito.verify(mNMClient).startContainer(Mockito.same(mockContainer2), Mockito.argThat(getContextMatcher(expectedWorkerContext))); Assert.assertEquals(containers.size(), mPrivateAccess.getWorkerHosts().size()); }
From source file:alluxio.yarn.ContainerAllocatorTest.java
License:Apache License
private ContainerAllocator setup(int numHosts, int maxContainersPerHost, int numContainers) throws Exception { ContainerAllocator containerAllocator = new ContainerAllocator(CONTAINER_NAME, numContainers, maxContainersPerHost, mResource, mYarnClient, mRMClient); List<NodeReport> nodeReports = new ArrayList<>(); for (int i = 0; i < numHosts; i++) { NodeReport nodeReport = Records.newRecord(NodeReport.class); nodeReport.setNodeId(NodeId.newInstance("host" + i, 0)); nodeReports.add(nodeReport);//from w w w . ja v a 2 s . com } when(mYarnClient.getNodeReports(Matchers.<NodeState[]>anyVararg())).thenReturn(nodeReports); doAnswer(allocateFirstHostAnswer(containerAllocator)).when(mRMClient) .addContainerRequest(any(ContainerRequest.class)); return containerAllocator; }
From source file:alluxio.yarn.ContainerAllocatorTest.java
License:Apache License
private Answer<Void> allocateFirstHostAnswer(final ContainerAllocator containerAllocator) { return new Answer<Void>() { @Override/*from w w w . j av a2 s . com*/ public Void answer(InvocationOnMock invocation) throws Throwable { ContainerRequest containerRequest = invocation.getArgumentAt(0, ContainerRequest.class); Container container = Records.newRecord(Container.class); container.setNodeId(NodeId.newInstance(containerRequest.getNodes().get(0), 0)); containerAllocator.allocateContainer(container); return null; } }; }
From source file:de.huberlin.wbi.hiway.scheduler.TestScheduler.java
License:Apache License
private static void run(Scheduler scheduler, List<String> nodeNames, List<String> taskNames, int[][] runtimes) { Queue<NodeId> availableNodes = new LinkedList<>(); for (String nodeName : nodeNames) { NodeId node = NodeId.newInstance(nodeName, 0); availableNodes.add(node);//from www. jav a2s . co m } Map<Container, TaskInstance> runningTasks = new HashMap<>(); Map<Container, Long> finishTimes = new HashMap<>(); int runningId = 0; long clock = 0; while (!scheduler.nothingToSchedule() || !runningTasks.isEmpty()) { if (!scheduler.nothingToSchedule() && !availableNodes.isEmpty()) { NodeId nodeId = availableNodes.remove(); ContainerId containerId = ContainerId.newContainerId(null, runningId++); Container container = Container.newInstance(containerId, nodeId, "", null, null, null); TaskInstance task = scheduler.getNextTask(container); runningTasks.put(container, task); long runtime = (runtimes == null) ? 1 : runtimes[nodeNames.indexOf(nodeId.getHost())][taskNames.indexOf(task.getTaskName())]; finishTimes.put(container, clock + runtime); } for (Container container : finishTimes.keySet()) { if (clock == finishTimes.get(container)) { NodeId nodeId = container.getNodeId(); ContainerStatus containerStatus = ContainerStatus.newInstance(container.getId(), null, "", 0); TaskInstance task = runningTasks.get(container); task.setCompleted(); long runtime = (runtimes == null) ? 1 : runtimes[nodeNames.indexOf(nodeId.getHost())][taskNames.indexOf(task.getTaskName())]; scheduler.taskCompleted(task, containerStatus, runtime); runningTasks.remove(container); availableNodes.add(nodeId); } } clock++; } }
From source file:io.hops.metadata.util.DistributedRTClientEvaluation.java
License:Apache License
public DistributedRTClientEvaluation(String rtAddress, int nbSimulatedNM, int hbPeriod, long duration, String output, int startingPort, int nbNMTotal) throws IOException, YarnException, InterruptedException { this.nbNM = nbSimulatedNM; this.hbPeriod = hbPeriod; this.duration = duration; this.output = output; this.nbNMTotal = nbNMTotal; conf = new YarnConfiguration(); conf.setStrings(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, rtAddress); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); //Create NMs/*from w w w. ja va 2 s.c o m*/ for (int i = 0; i < nbSimulatedNM; i++) { nmMap.put(i, NodeId.newInstance(InetAddress.getLocalHost().getHostName(), startingPort + i)); } start(); }
From source file:org.apache.flink.yarn.YarnFlinkResourceManagerTest.java
License:Apache License
@Test public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Exception { new JavaTestKit(system) { {/* w ww . j a v a2s.c o m*/ final Deadline deadline = new FiniteDuration(3, TimeUnit.MINUTES).fromNow(); Configuration flinkConfig = new Configuration(); YarnConfiguration yarnConfig = new YarnConfiguration(); SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(null, null); String applicationMasterHostName = "localhost"; String webInterfaceURL = "foobar"; ContaineredTaskManagerParameters taskManagerParameters = new ContaineredTaskManagerParameters(1L, 1L, 1L, 1, new HashMap<String, String>()); ContainerLaunchContext taskManagerLaunchContext = mock(ContainerLaunchContext.class); int yarnHeartbeatIntervalMillis = 1000; int maxFailedContainers = 10; int numInitialTaskManagers = 5; final YarnResourceManagerCallbackHandler callbackHandler = new YarnResourceManagerCallbackHandler(); AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = mock(AMRMClientAsync.class); NMClient nodeManagerClient = mock(NMClient.class); UUID leaderSessionID = UUID.randomUUID(); final List<Container> containerList = new ArrayList<>(); for (int i = 0; i < numInitialTaskManagers; i++) { Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(ContainerId.newInstance(ApplicationAttemptId .newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), i)); when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance("container", 1234)); containerList.add(mockContainer); } doAnswer(new Answer() { int counter = 0; @Override public Object answer(InvocationOnMock invocation) throws Throwable { if (counter < containerList.size()) { callbackHandler .onContainersAllocated(Collections.singletonList(containerList.get(counter++))); } return null; } }).when(resourceManagerClient).addContainerRequest(Matchers.any(AMRMClient.ContainerRequest.class)); final CompletableFuture<AkkaActorGateway> resourceManagerFuture = new CompletableFuture<>(); final CompletableFuture<AkkaActorGateway> leaderGatewayFuture = new CompletableFuture<>(); doAnswer((InvocationOnMock invocation) -> { Container container = (Container) invocation.getArguments()[0]; resourceManagerFuture.thenCombine(leaderGatewayFuture, (resourceManagerGateway, leaderGateway) -> { resourceManagerGateway.tell( new NotifyResourceStarted( YarnFlinkResourceManager.extractResourceID(container)), leaderGateway); return null; }); return null; }).when(nodeManagerClient).startContainer(Matchers.any(Container.class), Matchers.any(ContainerLaunchContext.class)); ActorRef resourceManager = null; ActorRef leader1; try { leader1 = system.actorOf(Props.create(TestingUtils.ForwardingActor.class, getRef(), Option.apply(leaderSessionID))); resourceManager = system.actorOf(Props.create(TestingYarnFlinkResourceManager.class, flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers, callbackHandler, resourceManagerClient, nodeManagerClient)); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); final AkkaActorGateway leader1Gateway = new AkkaActorGateway(leader1, leaderSessionID); final AkkaActorGateway resourceManagerGateway = new AkkaActorGateway(resourceManager, leaderSessionID); leaderGatewayFuture.complete(leader1Gateway); resourceManagerFuture.complete(resourceManagerGateway); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> taskManagerRegisteredFuture = resourceManagerGateway .ask(new NotifyWhenResourcesRegistered(numInitialTaskManagers), deadline.timeLeft()); Await.ready(taskManagerRegisteredFuture, deadline.timeLeft()); leaderRetrievalService.notifyListener(null, null); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); for (Container container : containerList) { resourceManagerGateway.tell( new NotifyResourceStarted(YarnFlinkResourceManager.extractResourceID(container)), leader1Gateway); } for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> numberOfRegisteredResourcesFuture = resourceManagerGateway .ask(RequestNumberOfRegisteredResources.INSTANCE, deadline.timeLeft()); int numberOfRegisteredResources = (Integer) Await.result(numberOfRegisteredResourcesFuture, deadline.timeLeft()); assertEquals(numInitialTaskManagers, numberOfRegisteredResources); } finally { if (resourceManager != null) { resourceManager.tell(PoisonPill.getInstance(), ActorRef.noSender()); } } } }; }
From source file:org.apache.flink.yarn.YarnResourceManagerTest.java
License:Apache License
private static Container mockContainer(String host, int port, int containerId) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); ContainerId mockContainerId = ContainerId.newInstance( ApplicationAttemptId.newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), containerId);// w ww . ja va 2 s . co m when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockContainer.getResource()).thenReturn(Resource.newInstance(200, 1)); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; }
From source file:org.apache.hoya.yarn.appmaster.state.AppState.java
License:Apache License
/** * build up the special master node, which lives * in the live node set but has a lifecycle bonded to the AM * @param containerId the AM master// ww w. java 2 s . com * @param host * @param nodeHttpAddress */ public void buildAppMasterNode(ContainerId containerId, String host, int amPort, String nodeHttpAddress) { Container container = new ContainerPBImpl(); container.setId(containerId); NodeId nodeId = NodeId.newInstance(host, amPort); container.setNodeId(nodeId); container.setNodeHttpAddress(nodeHttpAddress); RoleInstance am = new RoleInstance(container); am.role = HoyaKeys.COMPONENT_AM; appMasterNode = am; //it is also added to the set of live nodes getLiveNodes().put(containerId, am); }