List of usage examples for org.apache.hadoop.yarn.api.records ContainerId newInstance
@Private
@Deprecated
@Unstable
public static ContainerId newInstance(ApplicationAttemptId appAttemptId, int containerId)
From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)//w ww . j a v a 2 s . c om public void testVertexTaskAttemptOutputFailure() throws Exception { initAllVertices(VertexState.INITED); VertexImpl v = vertices.get("vertex1"); startVertex(v); dispatcher.await(); TaskAttemptImpl ta = (TaskAttemptImpl) v.getTask(0).getAttempts().values().iterator().next(); ta.handle(new TaskAttemptEventSchedule(ta.getID(), 2, 2)); NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); when(container.getNodeHttpAddress()).thenReturn("localhost:0"); AMContainerMap containers = new AMContainerMap(mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class), new ContainerContextMatcher(), appContext); containers.addContainerIfNew(container); doReturn(containers).when(appContext).getAllContainers(); ta.handle(new TaskAttemptEventStartedRemotely(ta.getID(), contId, null)); Assert.assertEquals(TaskAttemptStateInternal.RUNNING, ta.getInternalState()); dispatcher.getEventHandler() .handle(new VertexEventRouteEvent(v.getVertexId(), Collections.singletonList(new TezEvent( new TaskAttemptFailedEvent("Failed"), new EventMetaData(EventProducerConsumerType.OUTPUT, v.getName(), null, ta.getID()))))); dispatcher.await(); Assert.assertEquals(VertexState.RUNNING, v.getState()); Assert.assertEquals(TaskAttemptTerminationCause.OUTPUT_WRITE_ERROR, ta.getTerminationCause()); }
From source file:org.apache.tez.dag.app.rm.container.TestAMContainerMap.java
License:Apache License
private ContainerId mockContainerId(int cId) { ApplicationId appId = ApplicationId.newInstance(1000, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId = ContainerId.newInstance(appAttemptId, cId); return containerId; }
From source file:org.apache.tez.dag.app.rm.TestContainerReuse.java
License:Apache License
private Container createContainer(int id, String host, Resource resource, Priority priority) { ContainerId containerID = ContainerId .newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), id); NodeId nodeID = NodeId.newInstance(host, 0); Container container = Container.newInstance(containerID, nodeID, host + ":0", resource, priority, null); return container; }
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)/*from w ww . j a v a2 s .com*/ public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { RackResolver.init(new YarnConfiguration()); TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class); AppContext mockAppContext = mock(AppContext.class); when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING); TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class); String appHost = "host"; int appPort = 0; String appUrl = "url"; TaskSchedulerWithDrainableAppCallback scheduler = new TaskSchedulerWithDrainableAppCallback(mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext); Configuration conf = new Configuration(); scheduler.init(conf); RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class); Resource mockMaxResource = mock(Resource.class); Map<ApplicationAccessType, String> mockAcls = mock(Map.class); when(mockRegResponse.getMaximumResourceCapability()).thenReturn(mockMaxResource); when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls); when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString())) .thenReturn(mockRegResponse); Resource mockClusterResource = mock(Resource.class); when(mockRMClient.getAvailableResources()).thenReturn(mockClusterResource); scheduler.start(); String rack1 = "r1"; String rack2 = "r2"; String rack3 = "r3"; String node1Rack1 = "n1r1"; String node2Rack1 = "n2r1"; String node1Rack2 = "n1r2"; String node2Rack2 = "n2r2"; String node1Rack3 = "n1r3"; ApplicationAttemptId appId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0); ContainerId mockCId1 = ContainerId.newInstance(appId, 0); HeldContainer hc1 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc1.getNode()).thenReturn(node1Rack1); when(hc1.getRack()).thenReturn(rack1); when(hc1.getContainer().getId()).thenReturn(mockCId1); ContainerId mockCId2 = ContainerId.newInstance(appId, 1); HeldContainer hc2 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc2.getNode()).thenReturn(node2Rack1); when(hc2.getRack()).thenReturn(rack1); when(hc2.getContainer().getId()).thenReturn(mockCId2); ContainerId mockCId3 = ContainerId.newInstance(appId, 2); HeldContainer hc3 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc3.getNode()).thenReturn(node1Rack1); when(hc3.getRack()).thenReturn(rack1); when(hc3.getContainer().getId()).thenReturn(mockCId3); ContainerId mockCId4 = ContainerId.newInstance(appId, 3); HeldContainer hc4 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc4.getNode()).thenReturn(node2Rack1); when(hc4.getRack()).thenReturn(rack1); when(hc4.getContainer().getId()).thenReturn(mockCId4); ContainerId mockCId5 = ContainerId.newInstance(appId, 4); HeldContainer hc5 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc5.getNode()).thenReturn(node1Rack2); when(hc5.getRack()).thenReturn(rack2); when(hc5.getContainer().getId()).thenReturn(mockCId5); ContainerId mockCId6 = ContainerId.newInstance(appId, 5); HeldContainer hc6 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc6.getNode()).thenReturn(node2Rack2); when(hc6.getRack()).thenReturn(rack2); when(hc6.getContainer().getId()).thenReturn(mockCId6); ContainerId mockCId7 = ContainerId.newInstance(appId, 6); HeldContainer hc7 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc7.getNode()).thenReturn(node1Rack3); when(hc7.getRack()).thenReturn(rack3); when(hc7.getContainer().getId()).thenReturn(mockCId7); scheduler.heldContainers.put(mockCId1, hc1); scheduler.heldContainers.put(mockCId2, hc2); scheduler.heldContainers.put(mockCId3, hc3); scheduler.heldContainers.put(mockCId4, hc4); scheduler.heldContainers.put(mockCId5, hc5); scheduler.heldContainers.put(mockCId6, hc6); scheduler.heldContainers.put(mockCId7, hc7); // test empty case scheduler.sessionNumMinHeldContainers = 0; scheduler.determineMinHeldContainers(); Assert.assertEquals(0, scheduler.sessionMinHeldContainers.size()); // test min >= held scheduler.sessionNumMinHeldContainers = 7; scheduler.determineMinHeldContainers(); Assert.assertEquals(7, scheduler.sessionMinHeldContainers.size()); // test min < held scheduler.sessionNumMinHeldContainers = 5; scheduler.determineMinHeldContainers(); Assert.assertEquals(5, scheduler.sessionMinHeldContainers.size()); Set<HeldContainer> heldContainers = Sets.newHashSet(); for (ContainerId cId : scheduler.sessionMinHeldContainers) { heldContainers.add(scheduler.heldContainers.get(cId)); } Set<String> racks = Sets.newHashSet(); Set<String> nodes = Sets.newHashSet(); for (HeldContainer hc : heldContainers) { nodes.add(hc.getNode()); racks.add(hc.getRack()); } // 1 container from each node in rack1 and rack2. 1 container from rack3. // covers not enough containers in rack (rack 3) // covers just enough containers in rack (rack 2) // covers more than enough containers in rack (rack 1) Assert.assertEquals(5, nodes.size()); Assert.assertTrue(nodes.contains(node1Rack1) && nodes.contains(node2Rack1) && nodes.contains(node1Rack2) && nodes.contains(node2Rack2) && nodes.contains(node1Rack3)); Assert.assertEquals(3, racks.size()); Assert.assertTrue(racks.contains(rack1) && racks.contains(rack2) && racks.contains(rack3)); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.stop(); scheduler.close(); }
From source file:org.apache.tez.dag.app.TestDAGAppMaster.java
License:Apache License
@SuppressWarnings("deprecation") private void testDagCredentials(boolean doMerge) throws IOException { TezConfiguration conf = new TezConfiguration(); conf.setBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, doMerge); conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true); conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.toString()); ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); // create some sample AM credentials Credentials amCreds = new Credentials(); JobTokenSecretManager jtsm = new JobTokenSecretManager(); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appId.toString())); Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, jtsm); sessionToken.setService(identifier.getJobId()); TokenCache.setSessionToken(sessionToken, amCreds); TestTokenSecretManager ttsm = new TestTokenSecretManager(); Text tokenAlias1 = new Text("alias1"); Token<TestTokenIdentifier> amToken1 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("amtoken1")), ttsm); amCreds.addToken(tokenAlias1, amToken1); Text tokenAlias2 = new Text("alias2"); Token<TestTokenIdentifier> amToken2 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("amtoken2")), ttsm); amCreds.addToken(tokenAlias2, amToken2); FileSystem fs = FileSystem.getLocal(conf); FSDataOutputStream sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, new Path(TEST_DIR.toString(), TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); DAGProtos.PlanLocalResourcesProto.getDefaultInstance().writeDelimitedTo(sessionJarsPBOutStream); sessionJarsPBOutStream.close();/* www . j a v a 2 s.c o m*/ DAGAppMaster am = new DAGAppMaster(attemptId, ContainerId.newInstance(attemptId, 1), "127.0.0.1", 0, 0, new SystemClock(), 1, true, TEST_DIR.toString(), new String[] { TEST_DIR.toString() }, new String[] { TEST_DIR.toString() }, new TezApiVersionInfo().getVersion(), 1, amCreds, "someuser", null); am.init(conf); am.start(); // create some sample DAG credentials Credentials dagCreds = new Credentials(); Token<TestTokenIdentifier> dagToken1 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("dagtoken1")), ttsm); dagCreds.addToken(tokenAlias2, dagToken1); Text tokenAlias3 = new Text("alias3"); Token<TestTokenIdentifier> dagToken2 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("dagtoken2")), ttsm); dagCreds.addToken(tokenAlias3, dagToken2); TezDAGID dagId = TezDAGID.getInstance(appId, 1); DAGPlan dagPlan = DAGPlan.newBuilder().setName("somedag") .setCredentialsBinary(DagTypeConverters.convertCredentialsToProto(dagCreds)).build(); DAGImpl dag = am.createDAG(dagPlan, dagId); Credentials fetchedDagCreds = dag.getCredentials(); am.stop(); Token<? extends TokenIdentifier> fetchedToken1 = fetchedDagCreds.getToken(tokenAlias1); if (doMerge) { assertNotNull("AM creds missing from DAG creds", fetchedToken1); compareTestTokens(amToken1, fetchedDagCreds.getToken(tokenAlias1)); } else { assertNull("AM creds leaked to DAG creds", fetchedToken1); } compareTestTokens(dagToken1, fetchedDagCreds.getToken(tokenAlias2)); compareTestTokens(dagToken2, fetchedDagCreds.getToken(tokenAlias3)); }
From source file:org.apache.tez.dag.history.events.TestHistoryEventsProtoConversion.java
License:Apache License
private void testTaskAttemptStartedEvent() throws Exception { TaskAttemptStartedEvent event = new TaskAttemptStartedEvent( TezTaskAttemptID.getInstance(TezTaskID.getInstance( TezVertexID.getInstance(TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111), 1), 1),// www . ja v a 2s .com "vertex1", 10009l, ContainerId.newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1001), NodeId.newInstance("host1", 19999), "inProgress", "Completed", "nodeHttpAddress"); TaskAttemptStartedEvent deserializedEvent = (TaskAttemptStartedEvent) testProtoConversion(event); Assert.assertEquals(event.getTaskAttemptID(), deserializedEvent.getTaskAttemptID()); Assert.assertEquals(event.getContainerId(), deserializedEvent.getContainerId()); Assert.assertEquals(event.getNodeId(), deserializedEvent.getNodeId()); Assert.assertEquals(event.getStartTime(), deserializedEvent.getStartTime()); logEvents(event, deserializedEvent); }
From source file:org.apache.tez.dag.history.events.TestHistoryEventsProtoConversion.java
License:Apache License
private void testContainerLaunchedEvent() throws Exception { ContainerLaunchedEvent event = new ContainerLaunchedEvent( ContainerId.newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1001), 100034566, ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1)); ContainerLaunchedEvent deserializedEvent = (ContainerLaunchedEvent) testProtoConversion(event); Assert.assertEquals(event.getContainerId(), deserializedEvent.getContainerId()); Assert.assertEquals(event.getLaunchTime(), deserializedEvent.getLaunchTime()); Assert.assertEquals(event.getApplicationAttemptId(), deserializedEvent.getApplicationAttemptId()); logEvents(event, deserializedEvent); }
From source file:org.apache.tez.dag.history.events.TestHistoryEventsProtoConversion.java
License:Apache License
private void testContainerStoppedEvent() throws Exception { ContainerStoppedEvent event = new ContainerStoppedEvent( ContainerId.newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1001), 100034566, ContainerExitStatus.SUCCESS, ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1)); ContainerStoppedEvent deserializedEvent = (ContainerStoppedEvent) testProtoConversion(event); Assert.assertEquals(event.getContainerId(), deserializedEvent.getContainerId()); Assert.assertEquals(event.getStoppedTime(), deserializedEvent.getStoppedTime()); Assert.assertEquals(event.getApplicationAttemptId(), deserializedEvent.getApplicationAttemptId()); logEvents(event, deserializedEvent); }
From source file:org.apache.tez.dag.history.logging.ats.TestHistoryEventTimelineConversion.java
License:Apache License
@Before public void setup() { applicationId = ApplicationId.newInstance(9999l, 1); applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 1); tezDAGID = TezDAGID.getInstance(applicationId, random.nextInt()); tezVertexID = TezVertexID.getInstance(tezDAGID, random.nextInt()); tezTaskID = TezTaskID.getInstance(tezVertexID, random.nextInt()); tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, random.nextInt()); dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build(); containerId = ContainerId.newInstance(applicationAttemptId, 111); nodeId = NodeId.newInstance("node", 13435); }
From source file:org.apache.tez.records.TezContainerId.java
License:Apache License
@Override public void readFields(DataInput in) throws IOException { long timestamp = in.readLong(); int appId = in.readInt(); int appAttemptId = in.readInt(); int id = in.readInt(); ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, appAttemptId); this.containerId = ContainerId.newInstance(applicationAttemptId, id); }