List of usage examples for org.apache.hadoop.yarn.api.records Resource newInstance
@Public
@Stable
public static Resource newInstance(long memory, int vCores)
From source file:org.apache.tez.dag.app.dag.impl.TestTaskAttempt.java
License:Apache License
@Test(timeout = 5000) // Verifies that multiple TooManyFetchFailures are handled correctly by the // TaskAttempt.// w w w . ja v a 2 s . c o m public void testMultipleOutputFailed() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0); TezDAGID dagID = TezDAGID.getInstance(appId, 1); TezVertexID vertexID = TezVertexID.getInstance(dagID, 1); TezTaskID taskID = TezTaskID.getInstance(vertexID, 1); MockEventHandler mockEh = new MockEventHandler(); MockEventHandler eventHandler = spy(mockEh); TaskAttemptListener taListener = mock(TaskAttemptListener.class); when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0)); Configuration taskConf = new Configuration(); taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); taskConf.setBoolean("fs.file.impl.disable.cache", true); TaskLocationHint locationHint = TaskLocationHint .createTaskLocationHint(new HashSet<String>(Arrays.asList(new String[] { "127.0.0.1" })), null); Resource resource = Resource.newInstance(1024, 1); NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); when(container.getNodeHttpAddress()).thenReturn("localhost:0"); AppContext appCtx = mock(AppContext.class); AMContainerMap containers = new AMContainerMap(mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class), new ContainerContextMatcher(), appCtx); containers.addContainerIfNew(container); doReturn(new ClusterInfo()).when(appCtx).getClusterInfo(); doReturn(containers).when(appCtx).getAllContainers(); TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class); MockTaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, taListener, taskConf, new SystemClock(), mockHeartbeatHandler, appCtx, locationHint, false, resource, createFakeContainerContext(), false); TezTaskAttemptID taskAttemptID = taImpl.getID(); taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0)); // At state STARTING. taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, contId, null)); verify(mockHeartbeatHandler).register(taskAttemptID); taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE)); assertEquals("Task attempt is not in succeeded state", taImpl.getState(), TaskAttemptState.SUCCEEDED); verify(mockHeartbeatHandler).unregister(taskAttemptID); int expectedEventsTillSucceeded = 6; ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(expectedEventsTillSucceeded)).handle(arg.capture()); verifyEventType(arg.getAllValues(), TaskEventTAUpdate.class, 2); InputReadErrorEvent mockReEvent = InputReadErrorEvent.create("", 0, 1); EventMetaData mockMeta = mock(EventMetaData.class); TezTaskAttemptID mockDestId1 = mock(TezTaskAttemptID.class); when(mockMeta.getTaskAttemptID()).thenReturn(mockDestId1); TezEvent tzEvent = new TezEvent(mockReEvent, mockMeta); taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 4)); // failure threshold not met. state is SUCCEEDED assertEquals("Task attempt is not in succeeded state", taImpl.getState(), TaskAttemptState.SUCCEEDED); // sending same error again doesnt change anything taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 4)); assertEquals("Task attempt is not in succeeded state", taImpl.getState(), TaskAttemptState.SUCCEEDED); // default value of error cause assertEquals(TaskAttemptTerminationCause.UNKNOWN_ERROR, taImpl.getTerminationCause()); // different destination attempt reports error. now threshold crossed TezTaskAttemptID mockDestId2 = mock(TezTaskAttemptID.class); when(mockMeta.getTaskAttemptID()).thenReturn(mockDestId2); taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 4)); assertEquals("Task attempt is not in FAILED state", taImpl.getState(), TaskAttemptState.FAILED); assertEquals(TaskAttemptTerminationCause.OUTPUT_LOST, taImpl.getTerminationCause()); // verify unregister is not invoked again verify(mockHeartbeatHandler, times(1)).unregister(taskAttemptID); assertEquals(true, taImpl.inputFailedReported); int expectedEventsAfterFetchFailure = expectedEventsTillSucceeded + 2; arg.getAllValues().clear(); verify(eventHandler, times(expectedEventsAfterFetchFailure)).handle(arg.capture()); verifyEventType(arg.getAllValues().subList(expectedEventsTillSucceeded, expectedEventsAfterFetchFailure), TaskEventTAUpdate.class, 1); taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 1)); assertEquals("Task attempt is not in FAILED state, still", taImpl.getState(), TaskAttemptState.FAILED); assertFalse("InternalError occurred trying to handle TA_TOO_MANY_FETCH_FAILURES", eventHandler.internalError); // No new events. verify(eventHandler, times(expectedEventsAfterFetchFailure)).handle(arg.capture()); }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskAttemptRecovery.java
License:Apache License
@Before public void setUp() { mockEventHandler = mock(EventHandler.class); TezTaskID taskId = TezTaskID.fromString("task_1407371892933_0001_1_00_000000"); ta = new TaskAttemptImpl(taskId, 0, mockEventHandler, mock(TaskAttemptListener.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), mock(AppContext.class), false, Resource.newInstance(1, 1), mock(ContainerContext.class), false); }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskImpl.java
License:Apache License
@Before public void setup() { conf = new Configuration(); taskAttemptListener = mock(TaskAttemptListener.class); taskHeartbeatHandler = mock(TaskHeartbeatHandler.class); credentials = new Credentials(); clock = new SystemClock(); locationHint = TaskLocationHint.createTaskLocationHint(null, null); appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); dagId = TezDAGID.getInstance(appId, 1); vertexId = TezVertexID.getInstance(dagId, 1); appContext = mock(AppContext.class, RETURNS_DEEP_STUBS); mockContainerId = mock(ContainerId.class); mockContainer = mock(Container.class); mockAMContainer = mock(AMContainer.class); mockNodeId = mock(NodeId.class); when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockAMContainer.getContainer()).thenReturn(mockContainer); when(appContext.getAllContainers().get(mockContainerId)).thenReturn(mockAMContainer); taskResource = Resource.newInstance(1024, 1); localResources = new HashMap<String, LocalResource>(); environment = new HashMap<String, String>(); javaOpts = ""; leafVertex = false;/*from w w w . ja v a 2s . c om*/ containerContext = new ContainerContext(localResources, credentials, environment, javaOpts); Vertex vertex = mock(Vertex.class); eventHandler = new TestEventHandler(); mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskAttemptListener, clock, taskHeartbeatHandler, appContext, leafVertex, locationHint, taskResource, containerContext, vertex); }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskRecovery.java
License:Apache License
@Before public void setUp() { dispatcher = new DrainDispatcher(); dispatcher.register(DAGEventType.class, mock(EventHandler.class)); dispatcher.register(VertexEventType.class, mock(EventHandler.class)); dispatcher.register(TaskEventType.class, new TaskEventHandler()); dispatcher.register(TaskAttemptEventType.class, taEventHandler); dispatcher.init(new Configuration()); dispatcher.start();//from w w w .j ava 2 s.c om vertex = mock(Vertex.class, RETURNS_DEEP_STUBS); when(vertex.getProcessorDescriptor().getClassName()).thenReturn(""); mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS); when(mockAppContext.getCurrentDAG().getVertex(any(TezVertexID.class))).thenReturn(vertex); task = new TaskImpl(vertexId, 0, dispatcher.getEventHandler(), new Configuration(), mock(TaskAttemptListener.class), new SystemClock(), mock(TaskHeartbeatHandler.class), mockAppContext, false, Resource.newInstance(1, 1), mock(ContainerContext.class), mock(StateChangeNotifier.class)); Map<String, OutputCommitter> committers = new HashMap<String, OutputCommitter>(); committers.put("out1", new TestOutputCommitter(mock(OutputCommitterContext.class), true, false)); when(task.getVertex().getOutputCommitters()).thenReturn(committers); }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl.java
License:Apache License
private DAGPlan createDAGPlanWithMixedEdges() { LOG.info("Setting up mixed edge dag plan"); org.apache.tez.dag.api.DAG dag = org.apache.tez.dag.api.DAG.create("MixedEdges"); org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create("vertex1", ProcessorDescriptor.create("v1.class"), 1, Resource.newInstance(0, 0)); org.apache.tez.dag.api.Vertex v2 = org.apache.tez.dag.api.Vertex.create("vertex2", ProcessorDescriptor.create("v2.class"), 1, Resource.newInstance(0, 0)); org.apache.tez.dag.api.Vertex v3 = org.apache.tez.dag.api.Vertex.create("vertex3", ProcessorDescriptor.create("v3.class"), 1, Resource.newInstance(0, 0)); org.apache.tez.dag.api.Vertex v4 = org.apache.tez.dag.api.Vertex.create("vertex4", ProcessorDescriptor.create("v4.class"), 1, Resource.newInstance(0, 0)); org.apache.tez.dag.api.Vertex v5 = org.apache.tez.dag.api.Vertex.create("vertex5", ProcessorDescriptor.create("v5.class"), 1, Resource.newInstance(0, 0)); org.apache.tez.dag.api.Vertex v6 = org.apache.tez.dag.api.Vertex.create("vertex6", ProcessorDescriptor.create("v6.class"), 1, Resource.newInstance(0, 0)); dag.addVertex(v1).addVertex(v2).addVertex(v3).addVertex(v4).addVertex(v5).addVertex(v6); dag.addEdge(org.apache.tez.dag.api.Edge.create(v1, v2, EdgeProperty.create(DataMovementType.BROADCAST, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"), InputDescriptor.create("out.class")))); dag.addEdge(org.apache.tez.dag.api.Edge.create(v1, v3, EdgeProperty.create(DataMovementType.BROADCAST, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"), InputDescriptor.create("out.class")))); dag.addEdge(org.apache.tez.dag.api.Edge.create(v4, v2, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"), InputDescriptor.create("out.class")))); dag.addEdge(org.apache.tez.dag.api.Edge.create(v5, v3, EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"), InputDescriptor.create("out.class")))); dag.addEdge(org.apache.tez.dag.api.Edge.create(v4, v6, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"), InputDescriptor.create("out.class")))); dag.addEdge(org.apache.tez.dag.api.Edge.create(v5, v6, EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"), InputDescriptor.create("out.class")))); return dag.createDag(conf, null, null, null, true); }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl.java
License:Apache License
@SuppressWarnings({ "unchecked", "rawtypes" })
public void setupPostDagCreation() throws AMUserCodeException {
String dagName = "dag0";
dispatcher = new DrainDispatcher();
appContext = mock(AppContext.class);
thh = mock(TaskHeartbeatHandler.class);
historyEventHandler = mock(HistoryEventHandler.class);
TaskSchedulerEventHandler taskScheduler = mock(TaskSchedulerEventHandler.class);
UserGroupInformation ugi;/* w ww . jav a2 s . c o m*/
try {
ugi = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
throw new RuntimeException(e);
}
DAG dag = mock(DAG.class);
doReturn(ugi).when(dag).getDagUGI();
doReturn(dagName).when(dag).getName();
doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
doReturn(dag).when(appContext).getCurrentDAG();
execService = mock(ListeningExecutorService.class);
final ListenableFuture<Void> mockFuture = mock(ListenableFuture.class);
Mockito.doAnswer(new Answer() {
public ListenableFuture<Void> answer(InvocationOnMock invocation) {
Object[] args = invocation.getArguments();
CallableEvent e = (CallableEvent) args[0];
dispatcher.getEventHandler().handle(e);
return mockFuture;
}
}).when(execService).submit((Callable<Void>) any());
doReturn(execService).when(appContext).getExecService();
doReturn(conf).when(appContext).getAMConf();
doReturn(new Credentials()).when(dag).getCredentials();
doReturn(DAGPlan.getDefaultInstance()).when(dag).getJobPlan();
doReturn(dagId).when(appContext).getCurrentDAGID();
doReturn(dagId).when(dag).getID();
doReturn(taskScheduler).when(appContext).getTaskScheduler();
doReturn(Resource.newInstance(102400, 60)).when(taskScheduler).getTotalResources();
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
vertexGroups = Maps.newHashMap();
for (PlanVertexGroupInfo groupInfo : dagPlan.getVertexGroupsList()) {
vertexGroups.put(groupInfo.getGroupName(), new VertexGroupInfo(groupInfo));
}
updateTracker = new StateChangeNotifier(appContext.getCurrentDAG());
setupVertices();
when(dag.getVertex(any(TezVertexID.class))).thenAnswer(new Answer<Vertex>() {
@Override
public Vertex answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
if (args.length != 1) {
return null;
}
TezVertexID vId = (TezVertexID) args[0];
return vertexIdMap.get(vId);
}
});
when(dag.getVertex(any(String.class))).thenAnswer(new Answer<Vertex>() {
@Override
public Vertex answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
if (args.length != 1) {
return null;
}
String vId = (String) args[0];
return vertices.get(vId);
}
});
// TODO - this test logic is tightly linked to impl DAGImpl code.
edges = new HashMap<String, Edge>();
for (EdgePlan edgePlan : dagPlan.getEdgeList()) {
EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(edgePlan);
edges.put(edgePlan.getId(), new Edge(edgeProperty, dispatcher.getEventHandler()));
}
parseVertexEdges();
for (Edge edge : edges.values()) {
edge.initialize();
}
dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
taskEventDispatcher = new TaskEventDispatcher();
dispatcher.register(TaskEventType.class, taskEventDispatcher);
vertexEventDispatcher = new VertexEventDispatcher();
dispatcher.register(VertexEventType.class, vertexEventDispatcher);
dagEventDispatcher = new DagEventDispatcher();
dispatcher.register(DAGEventType.class, dagEventDispatcher);
dispatcher.init(conf);
dispatcher.start();
}
From source file:org.apache.tez.dag.app.rm.container.TestAMContainerMap.java
License:Apache License
private Container mockContainer(ContainerId containerId) { NodeId nodeId = NodeId.newInstance("localhost", 43255); Container container = Container.newInstance(containerId, nodeId, "localhost:33333", Resource.newInstance(1024, 1), Priority.newInstance(1), mock(Token.class)); return container; }
From source file:org.apache.tez.dag.app.rm.LlapTaskSchedulerService.java
License:Apache License
@VisibleForTesting public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, Clock clock) { super(taskSchedulerContext); this.clock = clock; try {/*from ww w. ja v a2 s .c o m*/ this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload()); } catch (IOException e) { throw new TezUncheckedException( "Failed to parse user payload for " + LlapTaskSchedulerService.class.getSimpleName(), e); } this.containerFactory = new ContainerFactory(taskSchedulerContext.getApplicationAttemptId(), taskSchedulerContext.getCustomClusterIdentifier()); this.memoryPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB); this.coresPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_VCPUS_PER_INSTANCE); this.executorsPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS); this.nodeBlacklistConf = new NodeBlacklistConf( HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS, TimeUnit.MILLISECONDS), HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MAX_TIMEOUT_MS, TimeUnit.MILLISECONDS), HiveConf.getFloatVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_DISABLE_BACK_OFF_FACTOR)); this.numSchedulableTasksPerNode = HiveConf.getIntVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NUM_SCHEDULABLE_TASKS_PER_NODE); long localityDelayMs = HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY, TimeUnit.MILLISECONDS); if (localityDelayMs == -1) { this.forceLocation = true; } else { this.forceLocation = false; } int memoryPerExecutor = (int) (memoryPerInstance / (float) executorsPerInstance); int coresPerExecutor = (int) (coresPerInstance / (float) executorsPerInstance); this.resourcePerExecutor = Resource.newInstance(memoryPerExecutor, coresPerExecutor); String instanceId = HiveConf.getTrimmedVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS); Preconditions.checkNotNull(instanceId, ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname + " must be defined"); ExecutorService executorServiceRaw = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapSchedulerNodeEnabler").build()); nodeEnabledExecutor = MoreExecutors.listeningDecorator(executorServiceRaw); ExecutorService schedulerExecutorServiceRaw = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapScheduler").build()); schedulerExecutor = MoreExecutors.listeningDecorator(schedulerExecutorServiceRaw); LOG.info("Running with configuration: " + "memoryPerInstance=" + memoryPerInstance + ", vCoresPerInstance=" + coresPerInstance + ", executorsPerInstance=" + executorsPerInstance + ", resourcePerInstanceInferred=" + resourcePerExecutor + ", nodeBlacklistConf=" + nodeBlacklistConf + ", forceLocation=" + forceLocation); }
From source file:org.apache.tez.dag.app.rm.LlapTaskSchedulerService.java
License:Apache License
@Override public Resource getTotalResources() { int memory = 0; int vcores = 0; readLock.lock();// w w w .j a v a2s . com try { for (ServiceInstance inst : activeInstances.getAll().values()) { if (inst.isAlive()) { Resource r = inst.getResource(); LOG.info("Found instance " + inst); memory += r.getMemory(); vcores += r.getVirtualCores(); } else { LOG.info("Ignoring dead instance " + inst); } } } finally { readLock.unlock(); } return Resource.newInstance(memory, vcores); }
From source file:org.apache.tez.dag.app.rm.LlapTaskSchedulerService.java
License:Apache License
/** * The difference between this and getTotalResources() is that this only gives currently free * resource instances, while the other lists all the instances that may become available in a * while./*from ww w . j a va 2s . com*/ */ @Override public Resource getAvailableResources() { // need a state store eventually for current state & measure backoffs int memory = 0; int vcores = 0; readLock.lock(); try { for (Entry<ServiceInstance, NodeInfo> entry : instanceToNodeMap.entrySet()) { if (entry.getKey().isAlive() && !entry.getValue().isDisabled()) { Resource r = entry.getKey().getResource(); memory += r.getMemory(); vcores += r.getVirtualCores(); } } } finally { readLock.unlock(); } return Resource.newInstance(memory, vcores); }