List of usage examples for org.apache.hadoop.yarn.api.records Container getResource
@Public @Stable public abstract Resource getResource();
Resource allocated to the container. From source file:org.apache.tez.dag.app.rm.TaskScheduler.java
License:Apache License
private Object unAssignContainer(ContainerId containerId, boolean releaseIfFound) { Object task = containerAssigments.remove(containerId); if (task == null) { return null; }/*www. jav a 2 s . c o m*/ Container container = taskAllocations.remove(task); assert container != null; Resources.subtractFrom(allocatedResources, container.getResource()); assert allocatedResources.getMemory() >= 0; if (releaseIfFound) { releaseContainer(containerId, task); } return task; }
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test(timeout = 5000)/*from w ww . j a va 2 s . c o m*/
public void testTaskSchedulerPreemption() throws Exception {
RackResolver.init(new YarnConfiguration());
TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class);
AppContext mockAppContext = mock(AppContext.class);
when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING);
TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class);
String appHost = "host";
int appPort = 0;
String appUrl = "url";
final TaskSchedulerWithDrainableAppCallback scheduler = new TaskSchedulerWithDrainableAppCallback(mockApp,
new PreemptionMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext);
TaskSchedulerAppCallbackDrainable drainableAppCallback = scheduler.getDrainableAppCallback();
Configuration conf = new Configuration();
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
scheduler.init(conf);
RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class);
when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString()))
.thenReturn(mockRegResponse);
scheduler.start();
Resource totalResource = Resource.newInstance(4000, 4);
when(mockRMClient.getAvailableResources()).thenReturn(totalResource);
// no preemption
scheduler.getProgress();
drainableAppCallback.drain();
Assert.assertEquals(totalResource, scheduler.getTotalResources());
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
// allocate task
Object mockTask1 = mock(Object.class);
Object mockTask2 = mock(Object.class);
Object mockTask3 = mock(Object.class);
Object mockTask3Wait = mock(Object.class);
Object mockTask3Retry = mock(Object.class);
Object mockTask3KillA = mock(Object.class);
Object mockTask3KillB = mock(Object.class);
Object obj3 = new Object();
Priority pri2 = Priority.newInstance(2);
Priority pri4 = Priority.newInstance(4);
Priority pri5 = Priority.newInstance(5);
Priority pri6 = Priority.newInstance(6);
ArgumentCaptor<CookieContainerRequest> requestCaptor = ArgumentCaptor
.forClass(CookieContainerRequest.class);
final ArrayList<CookieContainerRequest> anyContainers = new ArrayList<CookieContainerRequest>();
Resource taskAsk = Resource.newInstance(1024, 1);
scheduler.allocateTask(mockTask1, taskAsk, null, null, pri2, null, null);
drainableAppCallback.drain();
verify(mockRMClient, times(1)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
scheduler.allocateTask(mockTask3, taskAsk, null, null, pri6, obj3, null);
drainableAppCallback.drain();
verify(mockRMClient, times(2)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
// later one in the allocation gets killed between the two task3's
scheduler.allocateTask(mockTask3KillA, taskAsk, null, null, pri6, obj3, null);
drainableAppCallback.drain();
verify(mockRMClient, times(3)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
// later one in the allocation gets killed between the two task3's
scheduler.allocateTask(mockTask3KillB, taskAsk, null, null, pri6, obj3, null);
drainableAppCallback.drain();
verify(mockRMClient, times(4)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
Resource freeResource = Resource.newInstance(500, 0);
when(mockRMClient.getAvailableResources()).thenReturn(freeResource);
scheduler.getProgress();
drainableAppCallback.drain();
Assert.assertEquals(totalResource, scheduler.getTotalResources());
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
final List<ArrayList<CookieContainerRequest>> anyList = new LinkedList<ArrayList<CookieContainerRequest>>();
final List<ArrayList<CookieContainerRequest>> emptyList = new LinkedList<ArrayList<CookieContainerRequest>>();
anyList.add(anyContainers);
List<Container> containers = new ArrayList<Container>();
Container mockContainer1 = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer1.getNodeId().getHost()).thenReturn("host1");
when(mockContainer1.getResource()).thenReturn(taskAsk);
when(mockContainer1.getPriority()).thenReturn(pri2);
ContainerId mockCId1 = mock(ContainerId.class);
when(mockContainer1.getId()).thenReturn(mockCId1);
containers.add(mockContainer1);
Container mockContainer2 = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer2.getNodeId().getHost()).thenReturn("host1");
when(mockContainer2.getResource()).thenReturn(taskAsk);
when(mockContainer2.getPriority()).thenReturn(pri6);
ContainerId mockCId2 = mock(ContainerId.class);
when(mockContainer2.getId()).thenReturn(mockCId2);
containers.add(mockContainer2);
Container mockContainer3A = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer3A.getNodeId().getHost()).thenReturn("host1");
when(mockContainer3A.getResource()).thenReturn(taskAsk);
when(mockContainer3A.getPriority()).thenReturn(pri6);
ContainerId mockCId3A = mock(ContainerId.class);
when(mockContainer3A.getId()).thenReturn(mockCId3A);
containers.add(mockContainer3A);
Container mockContainer3B = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer3B.getNodeId().getHost()).thenReturn("host1");
when(mockContainer3B.getResource()).thenReturn(taskAsk);
when(mockContainer3B.getPriority()).thenReturn(pri2); // high priority container
ContainerId mockCId3B = mock(ContainerId.class);
when(mockContainer3B.getId()).thenReturn(mockCId3B);
containers.add(mockContainer3B);
when(mockRMClient.getMatchingRequests((Priority) any(), eq("host1"), (Resource) any()))
.thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() {
@Override
public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation)
throws Throwable {
return emptyList;
}
});
// RackResolver by default puts hosts in default-rack
when(mockRMClient.getMatchingRequests((Priority) any(), eq("/default-rack"), (Resource) any()))
.thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() {
@Override
public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation)
throws Throwable {
return emptyList;
}
});
when(mockRMClient.getMatchingRequests((Priority) any(), eq(ResourceRequest.ANY), (Resource) any()))
.thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() {
int calls = 0;
@Override
public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation)
throws Throwable {
if (calls > 0) {
anyContainers.remove(0);
}
calls++;
return anyList;
}
});
Mockito.doAnswer(new Answer() {
public Object answer(InvocationOnMock invocation) {
Object[] args = invocation.getArguments();
ContainerId cId = (ContainerId) args[0];
scheduler.deallocateContainer(cId);
return null;
}
}).when(mockApp).preemptContainer((ContainerId) any());
scheduler.onContainersAllocated(containers);
drainableAppCallback.drain();
Assert.assertEquals(4, scheduler.taskAllocations.size());
Assert.assertEquals(4096, scheduler.allocatedResources.getMemory());
Assert.assertEquals(mockCId1, scheduler.taskAllocations.get(mockTask1).getId());
Assert.assertEquals(mockCId2, scheduler.taskAllocations.get(mockTask3).getId());
Assert.assertEquals(mockCId3A, scheduler.taskAllocations.get(mockTask3KillA).getId());
// high priority container assigned to lower pri task. This task should still be preempted
// because the task priority is relevant for preemption and not the container priority
Assert.assertEquals(mockCId3B, scheduler.taskAllocations.get(mockTask3KillB).getId());
// no preemption
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
Object mockTask3WaitCookie = new Object();
scheduler.allocateTask(mockTask3Wait, taskAsk, null, null, pri6, obj3, mockTask3WaitCookie);
// no preemption - same pri
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
Priority pri8 = Priority.newInstance(8);
Container mockContainer4 = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer4.getNodeId().getHost()).thenReturn("host1");
when(mockContainer4.getResource()).thenReturn(taskAsk);
when(mockContainer4.getPriority()).thenReturn(pri8);
ContainerId mockCId4 = mock(ContainerId.class);
when(mockContainer4.getId()).thenReturn(mockCId4);
containers.clear();
containers.add(mockContainer4);
// Fudge new container being present in delayed allocation list due to race
HeldContainer heldContainer = new HeldContainer(mockContainer4, -1, -1, null);
scheduler.delayedContainerManager.delayedContainers.add(heldContainer);
// no preemption - container assignment attempts < 3
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
heldContainer.incrementAssignmentAttempts();
// no preemption - container assignment attempts < 3
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
heldContainer.incrementAssignmentAttempts();
heldContainer.incrementAssignmentAttempts();
// preemption - container released and resource asked again
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any());
verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId4);
verify(mockRMClient, times(5)).addContainerRequest(requestCaptor.capture());
CookieContainerRequest reAdded = requestCaptor.getValue();
Assert.assertEquals(pri6, reAdded.getPriority());
Assert.assertEquals(taskAsk, reAdded.getCapability());
Assert.assertEquals(mockTask3WaitCookie, reAdded.getCookie().getAppCookie());
// remove fudging.
scheduler.delayedContainerManager.delayedContainers.clear();
scheduler.allocateTask(mockTask3Retry, taskAsk, null, null, pri5, obj3, null);
// no preemption - higher pri. exact match
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any());
for (int i = 0; i < 11; ++i) {
scheduler.allocateTask(mockTask2, taskAsk, null, null, pri4, null, null);
}
drainableAppCallback.drain();
// mockTaskPri3KillB gets preempted to clear 10% of outstanding running preemptable tasks
// this is also a higher priority container than the pending task priority but was running a
// lower priority task. Task priority is relevant for preemption and not container priority as
// containers can run tasks of different priorities
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any());
verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3B);
// next 3 heartbeats do nothing, waiting for the RM to act on the last released resources
scheduler.getProgress();
scheduler.getProgress();
scheduler.getProgress();
verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any());
scheduler.getProgress();
drainableAppCallback.drain();
// Next oldest mockTaskPri3KillA gets preempted to clear 10% of outstanding running preemptable tasks
verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any());
verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3A);
AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, "", appUrl);
when(mockApp.getFinalAppStatus()).thenReturn(finalStatus);
scheduler.stop();
drainableAppCallback.drain();
scheduler.close();
}
From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java
License:Apache License
@Override public synchronized void allocateTask(Object task, Resource capability, ContainerId containerId, Priority priority, Object containerSignature, Object clientCookie) { HeldContainer heldContainer = heldContainers.get(containerId); String[] hosts = null;/*w w w . jav a 2 s . c o m*/ String[] racks = null; if (heldContainer != null) { Container container = heldContainer.getContainer(); if (canFit(capability, container.getResource())) { // just specify node and use YARN's soft locality constraint for the rest hosts = new String[1]; hosts[0] = container.getNodeId().getHost(); priorityHasAffinity.add(priority); } else { LOG.warn("Matching requested to container: " + containerId + " but requested capability: " + capability + " does not fit in container resource: " + container.getResource()); } } else { LOG.warn("Matching requested to unknown container: " + containerId); } CRCookie cookie = new CRCookie(task, clientCookie, containerSignature); CookieContainerRequest request = new CookieContainerRequest(capability, containerId, hosts, racks, priority, cookie); addRequestAndTrigger(task, request, hosts, racks); }
From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java
License:Apache License
private CookieContainerRequest getMatchingRequestWithPriority(Container container, String location) { Priority priority = container.getPriority(); Resource capability = container.getResource(); List<? extends Collection<CookieContainerRequest>> requestsList = amRmClient.getMatchingRequests(priority, location, capability);//from w w w . j a v a2s .co m if (!requestsList.isEmpty()) { // pick first one for (Collection<CookieContainerRequest> requests : requestsList) { for (CookieContainerRequest cookieContainerRequest : requests) { if (canAssignTaskToContainer(cookieContainerRequest, container)) { return cookieContainerRequest; } } } } return null; }
From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java
License:Apache License
private CookieContainerRequest getMatchingRequestWithoutPriority(Container container, String location, boolean considerContainerAffinity) { Resource capability = container.getResource(); List<? extends Collection<CookieContainerRequest>> pRequestsList = amRmClient .getMatchingRequestsForTopPriority(location, capability); if (considerContainerAffinity && !priorityHasAffinity.contains(amRmClient.getTopPriority())) { considerContainerAffinity = false; }/*from w w w .j ava 2 s .c om*/ if (pRequestsList == null || pRequestsList.isEmpty()) { return null; } CookieContainerRequest firstMatch = null; for (Collection<CookieContainerRequest> requests : pRequestsList) { for (CookieContainerRequest cookieContainerRequest : requests) { if (firstMatch == null || // we dont have a match. So look for one // we have a match but are looking for a better container level match. // skip the expensive canAssignTaskToContainer() if the request is // not affinitized to the container container.getId().equals(cookieContainerRequest.getAffinitizedContainer())) { if (canAssignTaskToContainer(cookieContainerRequest, container)) { // request matched to container if (!considerContainerAffinity) { return cookieContainerRequest; } ContainerId affCId = cookieContainerRequest.getAffinitizedContainer(); boolean canMatchTaskWithAffinity = true; if (affCId == null || !heldContainers.containsKey(affCId) || inUseContainers.contains(affCId)) { // affinity not specified // affinitized container is no longer held // affinitized container is in use canMatchTaskWithAffinity = false; } if (canMatchTaskWithAffinity) { if (container.getId().equals(cookieContainerRequest.getAffinitizedContainer())) { // container level match if (LOG.isDebugEnabled()) { LOG.debug("Matching with affinity for request: " + cookieContainerRequest + " container: " + affCId); } return cookieContainerRequest; } if (LOG.isDebugEnabled()) { LOG.debug("Skipping request for container " + container.getId() + " due to affinity. Request: " + cookieContainerRequest + " affContainer: " + affCId); } } else { firstMatch = cookieContainerRequest; } } } } } return firstMatch; }
From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java
License:Apache License
private void assignContainer(Object task, Container container, CookieContainerRequest assigned) { CookieContainerRequest request = removeTaskRequest(task); assert request != null; //assert assigned.equals(request); Container result = taskAllocations.put(task, container); assert result == null; inUseContainers.add(container.getId()); containerAssignments.put(container.getId(), task); HeldContainer heldContainer = heldContainers.get(container.getId()); if (!shouldReuseContainers && heldContainer == null) { heldContainers.put(container.getId(), new HeldContainer(container, -1, -1, assigned)); Resources.addTo(allocatedResources, container.getResource()); } else {//from ww w . j ava 2s. co m if (heldContainer.isNew()) { // check for existence before adding since the first container potentially // has the broadest signature as subsequent uses dont expand any dimension. // This will need to be enhanced to track other signatures too when we // think about preferring within vertex matching etc. heldContainers.put(container.getId(), new HeldContainer(container, heldContainer.getNextScheduleTime(), heldContainer.getContainerExpiryTime(), assigned)); } heldContainer.setLastTaskInfo(assigned); } }
From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java
License:Apache License
private void pushNewContainerToDelayed(List<Container> containers) { long expireTime = -1; if (idleContainerTimeoutMin > 0) { long currentTime = System.currentTimeMillis(); expireTime = currentTime + idleContainerTimeoutMin; }//w w w.j a v a2s. com synchronized (delayedContainerManager) { for (Container container : containers) { if (heldContainers.put(container.getId(), new HeldContainer(container, -1, expireTime, null)) != null) { throw new TezUncheckedException("New container " + container.getId() + " is already held."); } long nextScheduleTime = delayedContainerManager.maxScheduleTimeSeen; if (delayedContainerManager.maxScheduleTimeSeen == -1) { nextScheduleTime = System.currentTimeMillis(); } Resources.addTo(allocatedResources, container.getResource()); delayedContainerManager.addDelayedContainer(container, nextScheduleTime + 1); } } delayedContainerManager.triggerScheduling(false); }
From source file:org.hdl.caffe.yarn.app.ApplicationMaster.java
License:Apache License
public boolean startAllContainers() throws Exception { if (numAllocatedContainers.get() == numTotalContainers) { int numCaffeContainers = 0; if (this.allocatedContainers.size() < numTotalProcessorContainers) { LOG.error("not enough caffe containers allocated!"); return false; }/*from w w w . j av a2 s .co m*/ for (Container allocatedContainer : this.allocatedContainers) { if (numCaffeContainers < numTotalProcessorContainers) { LOG.info("work cid: " + allocatedContainer.getId().toString()); clusterSpec.addServerSpec(allocatedContainer.getId().toString(), allocatedContainer.getNodeId().getHost()); numCaffeContainers++; } else { break; } } for (Container allocatedContainer : this.allocatedContainers) { LOG.info("Launching a new container." + ", containerId=" + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory" + allocatedContainer.getResource().getMemorySize() + ", containerResourceVirtualCores" + allocatedContainer.getResource().getVirtualCores()); LOG.info("server cid: " + allocatedContainer.getId().toString()); LaunchContainerThread launchDelegator = new LaunchContainerThread(allocatedContainer, train, solver, feature, label, model, output, connection, this, clusterSpec.getServerAddress(allocatedContainer.getId().toString())); LOG.info("ServerAdress: " + clusterSpec.getServerAddress(allocatedContainer.getId().toString())); launchDelegator.setCaffeProcessorJar(caffeProcessorJar); launchDelegator.setContainerMemory(containerMemory); launchDelegator.setContainerRetryPolicy(containerRetryPolicy); launchDelegator.setContainerRetryErrorCodes(containerRetryErrorCodes); launchDelegator.setContainerMaxRetries(containerMaxRetries); launchDelegator.setContainrRetryInterval(containrRetryInterval); Thread launchThread = new Thread(launchDelegator); // launch and start the container on a separate thread to keep // the main thread unblocked // as all containers may not be allocated at one go. launchThreads.add(launchThread); launchedContainers.add(allocatedContainer.getId()); launchThread.start(); } } else { throw new Exception("containers are not allocated!"); } return true; }
From source file:org.springframework.xd.yarn.XdAppmaster.java
License:Apache License
@Override protected List<String> onContainerLaunchCommands(Container container, ContainerCluster cluster, List<String> commands) { // only modify container so assume presence of -Dspring.application.name=container if (findPosition(commands, "-Dspring.application.name=container") < 0) { return commands; }//from w w w . j a v a 2 s .com ArrayList<String> list = new ArrayList<String>(commands); Map<String, Object> extraProperties = cluster.getExtraProperties(); if (extraProperties != null && extraProperties.containsKey("containerGroups")) { int position = findPosition(commands, "-Dxd.container.groups="); String value = "-Dxd.container.groups=" + cluster.getExtraProperties().get("containerGroups"); if (position < 0) { list.add(Math.max(list.size() - 3, 0), value); } else { list.set(position, value); } } int memory = container.getResource().getMemory(); int virtualCores = container.getResource().getVirtualCores(); list.add(Math.max(list.size() - 3, 0), "-Dxd.container.memory=" + memory); list.add(Math.max(list.size() - 3, 0), "-Dxd.container.virtualCores=" + virtualCores); return list; }
From source file:org.springframework.yarn.am.monitor.DefaultContainerMonitorTests.java
License:Apache License
/** * Mock {@link Container}//from w w w . j a va2 s. co m * * @param containerId the {@link ContainerId} * @param nodeId the {@link NodeId} * @param resource the {@link Resource} * @param priority the {@link Priority} * @param state the {@link ContainerState} * @return mocked {@link Container} */ public static Container getMockContainer(ContainerId containerId, NodeId nodeId, Resource resource, Priority priority) { Container container = mock(Container.class); when(container.getId()).thenReturn(containerId); when(container.getNodeId()).thenReturn(nodeId); when(container.getResource()).thenReturn(resource); when(container.getPriority()).thenReturn(priority); return container; }