List of usage examples for org.apache.hadoop.yarn.api.records NodeId newInstance
@Public
@Stable
public static NodeId newInstance(String host, int port)
From source file:org.apache.tez.dag.history.logging.ats.TestATSV15HistoryLoggingService.java
License:Apache License
private List<DAGHistoryEvent> makeHistoryEvents(TezDAGID dagId, ATSV15HistoryLoggingService service) { List<DAGHistoryEvent> historyEvents = new ArrayList<>(); long time = System.currentTimeMillis(); Configuration conf = new Configuration(service.getConfig()); historyEvents.add(new DAGHistoryEvent(null, new AMStartedEvent(attemptId, time, user))); historyEvents.add(new DAGHistoryEvent(dagId, new DAGSubmittedEvent(dagId, time, DAGPlan.getDefaultInstance(), attemptId, null, user, conf, null))); TezVertexID vertexID = TezVertexID.getInstance(dagId, 1); historyEvents.add(new DAGHistoryEvent(dagId, new VertexStartedEvent(vertexID, time, time))); TezTaskID tezTaskID = TezTaskID.getInstance(vertexID, 1); historyEvents.add(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskID, "test", time, time))); historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(tezTaskID, 1), "test", time, ContainerId.newContainerId(attemptId, 1), NodeId.newInstance("localhost", 8765), null, null, null)));/*w w w . j av a 2s. co m*/ return historyEvents; }
From source file:org.apache.tez.dag.history.logging.ats.TestHistoryEventTimelineConversion.java
License:Apache License
@Before public void setup() { applicationId = ApplicationId.newInstance(9999l, 1); applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 1); tezDAGID = TezDAGID.getInstance(applicationId, random.nextInt()); tezVertexID = TezVertexID.getInstance(tezDAGID, random.nextInt()); tezTaskID = TezTaskID.getInstance(tezVertexID, random.nextInt()); tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, random.nextInt()); dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build(); containerId = ContainerId.newInstance(applicationAttemptId, 111); nodeId = NodeId.newInstance("node", 13435); }
From source file:org.apache.tez.dag.history.logging.proto.TestHistoryEventProtoConverter.java
License:Apache License
@Before public void setup() { applicationId = ApplicationId.newInstance(9999l, 1); applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 1); tezDAGID = TezDAGID.getInstance(applicationId, random.nextInt()); tezVertexID = TezVertexID.getInstance(tezDAGID, random.nextInt()); tezTaskID = TezTaskID.getInstance(tezVertexID, random.nextInt()); tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, random.nextInt()); CallerContextProto.Builder callerContextProto = CallerContextProto.newBuilder(); callerContextProto.setContext("ctxt"); callerContextProto.setCallerId("Caller_ID"); callerContextProto.setCallerType("Caller_Type"); callerContextProto.setBlob("Desc_1"); dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").setCallerContext(callerContextProto).build(); containerId = ContainerId.newContainerId(applicationAttemptId, 111); nodeId = NodeId.newInstance("node", 13435); }
From source file:org.apache.tez.dag.history.logging.proto.TestProtoHistoryLoggingService.java
License:Apache License
private List<DAGHistoryEvent> makeHistoryEvents(TezDAGID dagId, ProtoHistoryLoggingService service) { List<DAGHistoryEvent> historyEvents = new ArrayList<>(); DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build(); long time = System.currentTimeMillis(); Configuration conf = new Configuration(service.getConfig()); historyEvents.add(new DAGHistoryEvent(null, new AppLaunchedEvent(appId, time, time, user, conf, new VersionInfo("component", "1.1.0", "rev1", "20120101", "git.apache.org") { })));/*from w w w .ja v a 2 s . co m*/ historyEvents.add(new DAGHistoryEvent(dagId, new DAGSubmittedEvent(dagId, time, DAGPlan.getDefaultInstance(), attemptId, null, user, conf, null, "default"))); TezVertexID vertexID = TezVertexID.getInstance(dagId, 1); historyEvents.add(new DAGHistoryEvent(dagId, new VertexStartedEvent(vertexID, time, time))); TezTaskID tezTaskID = TezTaskID.getInstance(vertexID, 1); historyEvents.add(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskID, "test", time, time))); historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(tezTaskID, 1), "test", time, ContainerId.newContainerId(attemptId, 1), NodeId.newInstance("localhost", 8765), null, null, null))); historyEvents.add(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, time, time, DAGState.ERROR, "diagnostics", null, user, dagPlan.getName(), new HashMap<String, Integer>(), attemptId, dagPlan))); return historyEvents; }
From source file:org.apache.tez.dag.history.TestHistoryEventHandler.java
License:Apache License
private List<DAGHistoryEvent> makeHistoryEvents(TezDAGID dagId, Configuration inConf) { List<DAGHistoryEvent> historyEvents = new ArrayList<>(); long time = System.currentTimeMillis(); Configuration conf = new Configuration(inConf); historyEvents.add(new DAGHistoryEvent(null, new AMStartedEvent(attemptId, time, user))); historyEvents.add(new DAGHistoryEvent(dagId, new DAGSubmittedEvent(dagId, time, DAGPlan.getDefaultInstance(), attemptId, null, user, conf, null))); TezVertexID vertexID = TezVertexID.getInstance(dagId, 1); historyEvents.add(new DAGHistoryEvent(dagId, new VertexStartedEvent(vertexID, time, time))); ContainerId containerId = ContainerId.newContainerId(attemptId, dagId.getId()); TezTaskID tezTaskID = TezTaskID.getInstance(vertexID, 1); historyEvents.add(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskID, "test", time, time))); historyEvents.add(new DAGHistoryEvent(new ContainerLaunchedEvent(containerId, time, attemptId))); historyEvents.add(/* www . j a v a 2 s. c o m*/ new DAGHistoryEvent(dagId, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(tezTaskID, 1), "test", time, containerId, NodeId.newInstance("localhost", 8765), null, null, null))); historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptFinishedEvent(TezTaskAttemptID.getInstance(tezTaskID, 1), "test", time, time + 1, TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.EXTERNAL_PREEMPTION, "", null, null, null, time, null, time, containerId, NodeId.newInstance("localhost", 8765), null, null, null))); historyEvents.add( new DAGHistoryEvent(dagId, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(tezTaskID, 2), "test", time, containerId, NodeId.newInstance("localhost", 8765), null, null, null))); historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptFinishedEvent(TezTaskAttemptID.getInstance(tezTaskID, 2), "test", time + 2, time + 3, TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.INTERRUPTED_BY_USER, "", null, null, null, time, null, time + 2, containerId, NodeId.newInstance("localhost", 8765), null, null, null))); historyEvents.add(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, time, time, DAGState.SUCCEEDED, null, null, user, "test", null, attemptId, DAGPlan.getDefaultInstance()))); historyEvents.add(new DAGHistoryEvent(new ContainerStoppedEvent(containerId, time + 4, 0, attemptId))); return historyEvents; }
From source file:org.apache.tez.test.TestRecovery.java
License:Apache License
@Test(timeout = 1800000) public void testRecovery_OrderedWordCount() throws Exception { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vertexId0 = TezVertexID.getInstance(dagId, 0); TezVertexID vertexId1 = TezVertexID.getInstance(dagId, 1); TezVertexID vertexId2 = TezVertexID.getInstance(dagId, 2); ContainerId containerId = ContainerId.newInstance(ApplicationAttemptId.newInstance(appId, 1), 1); NodeId nodeId = NodeId.newInstance("localhost", 10); List<TezEvent> initGeneratedEvents = Lists.newArrayList( new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null)); List<SimpleShutdownCondition> shutdownConditions = Lists.newArrayList( new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent(dagId, 0L, "username", "dagName", null)), new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId, 0L, "username", "dagName")), new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent(dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(), "username", "dagName", new HashMap<String, Integer>(), ApplicationAttemptId.newInstance(appId, 1), null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId0, "Tokenizer", 0L, 0L, 0, "", null, initGeneratedEvents, null)),/*w w w . j a va 2s. co m*/ new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId1, "Summation", 0L, 0L, 0, "", null, null, null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId2, "Sorter", 0L, 0L, 0, "", null, null, null)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId1, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId2, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId0, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId1, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId2, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId1, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId2, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId1, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId2, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent( TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent( TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId1, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent( TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId2, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")) ); Random rand = new Random(); for (int i = 0; i < shutdownConditions.size(); i++) { // randomly choose half of the test scenario to avoid // timeout. if (rand.nextDouble() < 0.5) { // generate split in client side when HistoryEvent type is VERTEX_STARTED (TEZ-2976) testOrderedWordCount(shutdownConditions.get(i), true, shutdownConditions.get(i).getHistoryEvent() .getEventType() == HistoryEventType.VERTEX_STARTED); } } }
From source file:org.apache.tez.test.TestRecovery.java
License:Apache License
@Test(timeout = 1800000) public void testRecovery_HashJoin() throws Exception { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vertexId0 = TezVertexID.getInstance(dagId, 0); TezVertexID vertexId1 = TezVertexID.getInstance(dagId, 1); TezVertexID vertexId2 = TezVertexID.getInstance(dagId, 2); ContainerId containerId = ContainerId.newInstance(ApplicationAttemptId.newInstance(appId, 1), 1); NodeId nodeId = NodeId.newInstance("localhost", 10); List<TezEvent> initGeneratedEvents = Lists.newArrayList( new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null)); List<SimpleShutdownCondition> shutdownConditions = Lists.newArrayList( new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent(dagId, 0L, "username", "dagName", null)), new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId, 0L, "username", "dagName")), new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent(dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(), "username", "dagName", new HashMap<String, Integer>(), ApplicationAttemptId.newInstance(appId, 1), null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId0, "hashSide", 0L, 0L, 0, "", null, initGeneratedEvents, null)),// w w w . ja v a 2s .c o m new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId1, "streamingSide", 0L, 0L, 0, "", null, null, null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId2, "joiner", 0L, 0L, 0, "", null, null, null)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId0, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId1, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId2, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId1, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId2, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId1, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId2, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId1, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId2, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent( TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent( TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId1, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent( TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId2, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")) ); Random rand = new Random(); for (int i = 0; i < shutdownConditions.size(); i++) { // randomly choose half of the test scenario to avoid // timeout. if (rand.nextDouble() < 0.5) { // generate split in client side when HistoryEvent type is VERTEX_STARTED (TEZ-2976) testHashJoinExample(shutdownConditions.get(i), true, shutdownConditions.get(i).getHistoryEvent() .getEventType() == HistoryEventType.VERTEX_STARTED); } } }
From source file:org.apache.tez.test.TestRecovery.java
License:Apache License
@Test(timeout = 1800000) public void testTwoRoundsRecoverying() throws Exception { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vertexId0 = TezVertexID.getInstance(dagId, 0); TezVertexID vertexId1 = TezVertexID.getInstance(dagId, 1); TezVertexID vertexId2 = TezVertexID.getInstance(dagId, 2); ContainerId containerId = ContainerId.newInstance(ApplicationAttemptId.newInstance(appId, 1), 1); NodeId nodeId = NodeId.newInstance("localhost", 10); List<TezEvent> initGeneratedEvents = Lists.newArrayList( new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null)); List<SimpleShutdownCondition> shutdownConditions = Lists.newArrayList( new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent(dagId, 0L, "username", "dagName", null)), new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId, 0L, "username", "dagName")), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent(vertexId0, "Tokenizer", 0L, 0L, 0, "", null, initGeneratedEvents, null)),//from ww w.j av a 2 s . c om new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(vertexId0, 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)), new SimpleShutdownCondition(TIMING.POST, new TaskAttemptStartedEvent( TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L, containerId, nodeId, "", "", "")), new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L, null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), new HashMap<String, Integer>(), null)), new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent(dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(), "username", "dagName", new HashMap<String, Integer>(), ApplicationAttemptId.newInstance(appId, 1), null)) ); Random rand = new Random(); for (int i = 0; i < shutdownConditions.size() - 1; i++) { // randomly choose half of the test scenario to avoid // timeout. if (rand.nextDouble() < 0.5) { int nextSimpleConditionIndex = i + 1 + rand.nextInt(shutdownConditions.size() - i - 1); if (nextSimpleConditionIndex == shutdownConditions.size() - 1) { testOrderedWordCountMultipleRoundRecoverying( new RecoveryServiceWithEventHandlingHook.MultipleRoundShutdownCondition( Lists.newArrayList(shutdownConditions.get(i), shutdownConditions.get(nextSimpleConditionIndex))), true, shutdownConditions.get(i).getHistoryEvent() .getEventType() == HistoryEventType.VERTEX_STARTED); } } } }
From source file:tachyon.yarn.ApplicationMasterTest.java
License:Apache License
/** * Tests that the correct type and number of containers are requested. *//* w w w . j a va2 s .co m*/ @Test(timeout = 10000) public void requestContainersOnceTest() throws Exception { // Mock the Yarn client to give a NodeReport with NUM_WORKERS nodes List<NodeReport> nodeReports = Lists.newArrayList(); final List<String> nodeHosts = Lists.newArrayList(); for (int i = 0; i < NUM_WORKERS; i++) { String host = "host" + i; NodeReport report = Mockito.mock(NodeReport.class); Mockito.when(report.getNodeId()).thenReturn(NodeId.newInstance(host, 0)); nodeReports.add(report); nodeHosts.add(host); } Mockito.when(YarnUtils.getNodeHosts(mYarnClient)).thenReturn(Sets.newHashSet(nodeHosts)); // Mock the Resource Manager to "allocate" containers when they are requested and update // ApplicationMaster internal state Mockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { mPrivateAccess.getMasterAllocated().countDown(); return null; } }).when(mRMClient).addContainerRequest(Mockito.argThat(getMasterContainerMatcher())); Mockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { Multiset<String> workerNodes = mPrivateAccess.getWorkerHosts(); synchronized (workerNodes) { workerNodes.add("host-" + UUID.randomUUID()); mPrivateAccess.getOutstandingWorkerContainerReqeustsLatch().countDown(); workerNodes.notify(); if (workerNodes.size() == NUM_WORKERS) { // Once all workers are allocated, we shut down the master so that // requestContainers() doesn't run forever mMaster.onShutdownRequest(); } } return null; } }).when(mRMClient).addContainerRequest(Mockito.argThat(getWorkerContainerMatcher(nodeHosts))); // This will hang if incorrect worker container requests are made mMaster.requestContainers(); // Verify that the right types and numbers of containers were requested Mockito.verify(mRMClient).addContainerRequest(Mockito.argThat(getMasterContainerMatcher())); Mockito.verify(mRMClient, Mockito.times(NUM_WORKERS)) .addContainerRequest(Mockito.argThat(getWorkerContainerMatcher(nodeHosts))); }
From source file:tachyon.yarn.ApplicationMasterTest.java
License:Apache License
/** * Mocks mRMClient to randomly allocated one of the requested hosts. * * This involves/*from ww w . jav a 2 s .c o m*/ * 1) Creating NUM_WORKERS mock containers, each with a different mock host * 2) Mocking mYarnClient to return the mock hosts of the mock containers * 3) Mocking mRMClient.addContainerRequest to asynchronously call mMaster.onContainersAllocated * with a random container on a requested host * * @param numContainers the number of mock container hosts */ private void mockResourceManager(int numContainers) throws Exception { final Random random = new Random(); final List<Container> mockContainers = Lists.newArrayList(); List<NodeReport> nodeReports = Lists.newArrayList(); List<String> hosts = Lists.newArrayList(MASTER_ADDRESS); for (int i = 0; i < numContainers - 1; i++) { String host = "host" + i; hosts.add(host); } for (String host : hosts) { Container mockContainer = Mockito.mock(Container.class); Mockito.when(mockContainer.getNodeHttpAddress()).thenReturn(host + ":8042"); Mockito.when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance(host, 0)); mockContainers.add(mockContainer); NodeReport report = Mockito.mock(NodeReport.class); Mockito.when(report.getNodeId()).thenReturn(NodeId.newInstance(host, 0)); nodeReports.add(report); } Mockito.when(YarnUtils.getNodeHosts(mYarnClient)).thenReturn(Sets.newHashSet(hosts)); // Pretend to be the Resource Manager, allocating containers when they are requested. Mockito.doAnswer(new Answer<Void>() { @Override public Void answer(final InvocationOnMock invocation) { new Thread(new Runnable() { @Override public void run() { // Allow the requests to interleave randomly CommonUtils.sleepMs(50 + random.nextInt(200)); // Allocate a randomly chosen container from among the requested hosts ContainerRequest request = invocation.getArgumentAt(0, ContainerRequest.class); Set<String> requestedHosts = Sets.newHashSet(request.getNodes()); List<Container> requestedContainers = Lists.newArrayList(); for (Container container : mockContainers) { if (requestedHosts.contains(container.getNodeId().getHost())) { requestedContainers.add(container); } } mMaster.onContainersAllocated(Lists .newArrayList(requestedContainers.get(random.nextInt(requestedContainers.size())))); } }).start(); return null; } }).when(mRMClient).addContainerRequest(Mockito.<ContainerRequest>any()); }