Example usage for org.apache.hadoop.yarn.event DrainDispatcher DrainDispatcher

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.event DrainDispatcher DrainDispatcher.

Prototype

public DrainDispatcher() 

Source Link

Usage

From source file:org.apache.tez.dag.app.dag.impl.TestCommit.java

License:Apache License

@SuppressWarnings({ "unchecked", "rawtypes" })
public void setupDAG(DAGPlan dagPlan) {
    conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
    appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(100, 1), 1);
    dagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 1);
    Assert.assertNotNull(dagId);//  ww  w. ja  v a  2 s.  co  m
    dispatcher = new DrainDispatcher();
    fsTokens = new Credentials();
    appContext = mock(AppContext.class);
    when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
    rawExecutor = Executors.newCachedThreadPool(
            new ThreadFactoryBuilder().setDaemon(true).setNameFormat("App Shared Pool - " + "#%d").build());
    execService = MoreExecutors.listeningDecorator(rawExecutor);

    doReturn(execService).when(appContext).getExecService();
    historyEventHandler = new MockHistoryEventHandler(appContext);
    aclManager = new ACLManager("amUser");
    doReturn(conf).when(appContext).getAMConf();
    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
    doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
    doReturn(dagId).when(appContext).getCurrentDAGID();
    doReturn(historyEventHandler).when(appContext).getHistoryHandler();
    doReturn(aclManager).when(appContext).getAMACLManager();
    dag = new DAGImpl(dagId, conf, dagPlan, dispatcher.getEventHandler(), taskCommunicatorManagerInterface,
            fsTokens, clock, "user", thh, appContext);
    doReturn(dag).when(appContext).getCurrentDAG();
    doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
    ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(8192, 10));
    doReturn(clusterInfo).when(appContext).getClusterInfo();
    dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
    taskEventDispatcher = new TaskEventDispatcher();
    dispatcher.register(TaskEventType.class, taskEventDispatcher);
    taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
    dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
    vertexEventDispatcher = new VertexEventDispatcher();
    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
    dagEventDispatcher = new DagEventDispatcher();
    dispatcher.register(DAGEventType.class, dagEventDispatcher);
    dagFinishEventHandler = new DAGFinishEventHandler();
    dispatcher.register(DAGAppMasterEventType.class, dagFinishEventHandler);
    dispatcher.init(conf);
    dispatcher.start();
}

---

From source file:org.apache.tez.dag.app.dag.impl.TestDAGImpl.java

License:Apache License

@SuppressWarnings({ "unchecked", "rawtypes" })
@Before/*  w ww .ja  va 2s .com*/
public void setup() {
    conf = new Configuration();
    conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
    appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(100, 1), 1);
    dagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 1);
    Assert.assertNotNull(dagId);
    dagPlan = createTestDAGPlan();
    dispatcher = new DrainDispatcher();
    fsTokens = new Credentials();
    appContext = mock(AppContext.class);
    execService = mock(ListeningExecutorService.class);
    final ListenableFuture<Void> mockFuture = mock(ListenableFuture.class);

    Mockito.doAnswer(new Answer() {
        public ListenableFuture<Void> answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            CallableEvent e = (CallableEvent) args[0];
            dispatcher.getEventHandler().handle(e);
            return mockFuture;
        }
    }).when(execService).submit((Callable<Void>) any());

    doReturn(execService).when(appContext).getExecService();
    historyEventHandler = mock(HistoryEventHandler.class);
    aclManager = new ACLManager("amUser");
    doReturn(conf).when(appContext).getAMConf();
    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
    doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
    doReturn(dagId).when(appContext).getCurrentDAGID();
    doReturn(historyEventHandler).when(appContext).getHistoryHandler();
    doReturn(aclManager).when(appContext).getAMACLManager();
    dag = new DAGImpl(dagId, conf, dagPlan, dispatcher.getEventHandler(), taskAttemptListener, fsTokens, clock,
            "user", thh, appContext);
    doReturn(dag).when(appContext).getCurrentDAG();
    mrrAppContext = mock(AppContext.class);
    doReturn(aclManager).when(mrrAppContext).getAMACLManager();
    doReturn(execService).when(mrrAppContext).getExecService();
    mrrDagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 2);
    mrrDagPlan = createTestMRRDAGPlan();
    mrrDag = new DAGImpl(mrrDagId, conf, mrrDagPlan, dispatcher.getEventHandler(), taskAttemptListener,
            fsTokens, clock, "user", thh, mrrAppContext);
    doReturn(conf).when(mrrAppContext).getAMConf();
    doReturn(mrrDag).when(mrrAppContext).getCurrentDAG();
    doReturn(appAttemptId).when(mrrAppContext).getApplicationAttemptId();
    doReturn(appAttemptId.getApplicationId()).when(mrrAppContext).getApplicationID();
    doReturn(historyEventHandler).when(mrrAppContext).getHistoryHandler();
    groupAppContext = mock(AppContext.class);
    doReturn(aclManager).when(groupAppContext).getAMACLManager();
    doReturn(execService).when(groupAppContext).getExecService();
    groupDagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 3);
    groupDagPlan = createGroupDAGPlan();
    groupDag = new DAGImpl(groupDagId, conf, groupDagPlan, dispatcher.getEventHandler(), taskAttemptListener,
            fsTokens, clock, "user", thh, groupAppContext);
    doReturn(conf).when(groupAppContext).getAMConf();
    doReturn(groupDag).when(groupAppContext).getCurrentDAG();
    doReturn(appAttemptId).when(groupAppContext).getApplicationAttemptId();
    doReturn(appAttemptId.getApplicationId()).when(groupAppContext).getApplicationID();
    doReturn(historyEventHandler).when(groupAppContext).getHistoryHandler();

    // reset totalCommitCounter to 0
    TotalCountingOutputCommitter.totalCommitCounter = 0;
    dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
    taskEventDispatcher = new TaskEventDispatcher();
    dispatcher.register(TaskEventType.class, taskEventDispatcher);
    taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
    dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
    vertexEventDispatcher = new VertexEventDispatcher();
    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
    dagEventDispatcher = new DagEventDispatcher();
    dispatcher.register(DAGEventType.class, dagEventDispatcher);
    dagFinishEventHandler = new DAGFinishEventHandler();
    dispatcher.register(DAGAppMasterEventType.class, dagFinishEventHandler);
    dispatcher.register(TaskEventType.class, new TaskEventHandler());
    dispatcher.init(conf);
    dispatcher.start();
}

---

From source file:org.apache.tez.dag.app.dag.impl.TestTaskRecovery.java

License:Apache License

@Before
public void setUp() {
    dispatcher = new DrainDispatcher();
    dispatcher.register(DAGEventType.class, mock(EventHandler.class));
    dispatcher.register(VertexEventType.class, mock(EventHandler.class));
    dispatcher.register(TaskEventType.class, new TaskEventHandler());
    dispatcher.register(TaskAttemptEventType.class, taEventHandler);
    dispatcher.init(new Configuration());
    dispatcher.start();/* www  .  j a v  a  2s  .co m*/

    vertex = mock(Vertex.class, RETURNS_DEEP_STUBS);
    when(vertex.getProcessorDescriptor().getClassName()).thenReturn("");

    mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    when(mockAppContext.getCurrentDAG().getVertex(any(TezVertexID.class))).thenReturn(vertex);

    task = new TaskImpl(vertexId, 0, dispatcher.getEventHandler(), new Configuration(),
            mock(TaskAttemptListener.class), new SystemClock(), mock(TaskHeartbeatHandler.class),
            mockAppContext, false, Resource.newInstance(1, 1), mock(ContainerContext.class),
            mock(StateChangeNotifier.class));

    Map<String, OutputCommitter> committers = new HashMap<String, OutputCommitter>();
    committers.put("out1", new TestOutputCommitter(mock(OutputCommitterContext.class), true, false));
    when(task.getVertex().getOutputCommitters()).thenReturn(committers);
}

---

From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl.java

License:Apache License

@SuppressWarnings({ "unchecked", "rawtypes" })
public void setupPostDagCreation() throws AMUserCodeException {
    String dagName = "dag0";
    dispatcher = new DrainDispatcher();
    appContext = mock(AppContext.class);
    thh = mock(TaskHeartbeatHandler.class);
    historyEventHandler = mock(HistoryEventHandler.class);
    TaskSchedulerEventHandler taskScheduler = mock(TaskSchedulerEventHandler.class);
    UserGroupInformation ugi;//from   ww w  .j av  a  2  s.  c o m
    try {
        ugi = UserGroupInformation.getCurrentUser();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    DAG dag = mock(DAG.class);
    doReturn(ugi).when(dag).getDagUGI();
    doReturn(dagName).when(dag).getName();
    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
    doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
    doReturn(dag).when(appContext).getCurrentDAG();
    execService = mock(ListeningExecutorService.class);
    final ListenableFuture<Void> mockFuture = mock(ListenableFuture.class);

    Mockito.doAnswer(new Answer() {
        public ListenableFuture<Void> answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            CallableEvent e = (CallableEvent) args[0];
            dispatcher.getEventHandler().handle(e);
            return mockFuture;
        }
    }).when(execService).submit((Callable<Void>) any());

    doReturn(execService).when(appContext).getExecService();
    doReturn(conf).when(appContext).getAMConf();
    doReturn(new Credentials()).when(dag).getCredentials();
    doReturn(DAGPlan.getDefaultInstance()).when(dag).getJobPlan();
    doReturn(dagId).when(appContext).getCurrentDAGID();
    doReturn(dagId).when(dag).getID();
    doReturn(taskScheduler).when(appContext).getTaskScheduler();
    doReturn(Resource.newInstance(102400, 60)).when(taskScheduler).getTotalResources();
    doReturn(historyEventHandler).when(appContext).getHistoryHandler();
    doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();

    vertexGroups = Maps.newHashMap();
    for (PlanVertexGroupInfo groupInfo : dagPlan.getVertexGroupsList()) {
        vertexGroups.put(groupInfo.getGroupName(), new VertexGroupInfo(groupInfo));
    }
    updateTracker = new StateChangeNotifier(appContext.getCurrentDAG());
    setupVertices();
    when(dag.getVertex(any(TezVertexID.class))).thenAnswer(new Answer<Vertex>() {
        @Override
        public Vertex answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            if (args.length != 1) {
                return null;
            }
            TezVertexID vId = (TezVertexID) args[0];
            return vertexIdMap.get(vId);
        }
    });
    when(dag.getVertex(any(String.class))).thenAnswer(new Answer<Vertex>() {
        @Override
        public Vertex answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            if (args.length != 1) {
                return null;
            }
            String vId = (String) args[0];
            return vertices.get(vId);
        }
    });

    // TODO - this test logic is tightly linked to impl DAGImpl code.
    edges = new HashMap<String, Edge>();
    for (EdgePlan edgePlan : dagPlan.getEdgeList()) {
        EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(edgePlan);
        edges.put(edgePlan.getId(), new Edge(edgeProperty, dispatcher.getEventHandler()));
    }

    parseVertexEdges();

    for (Edge edge : edges.values()) {
        edge.initialize();
    }

    dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
    taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
    dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
    taskEventDispatcher = new TaskEventDispatcher();
    dispatcher.register(TaskEventType.class, taskEventDispatcher);
    vertexEventDispatcher = new VertexEventDispatcher();
    dispatcher.register(VertexEventType.class, vertexEventDispatcher);
    dagEventDispatcher = new DagEventDispatcher();
    dispatcher.register(DAGEventType.class, dagEventDispatcher);
    dispatcher.init(conf);
    dispatcher.start();
}

---

From source file:org.apache.tez.dag.app.dag.impl.TestVertexRecovery.java

License:Apache License

@Before
public void setUp() throws IOException {

    dispatcher = new DrainDispatcher();
    dispatcher.register(DAGEventType.class, mock(EventHandler.class));
    vertexEventHandler = new VertexEventHanlder();
    dispatcher.register(VertexEventType.class, vertexEventHandler);
    taskEventHandler = new TaskEventHandler();
    dispatcher.register(TaskEventType.class, taskEventHandler);
    dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventHandler());
    dispatcher.init(new Configuration());
    dispatcher.start();//from   ww  w. j av  a  2s  .co  m

    mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);

    DAGPlan dagPlan = createDAGPlan();
    dag = new DAGImpl(dagId, new Configuration(), dagPlan, dispatcher.getEventHandler(),
            mock(TaskAttemptListener.class), new Credentials(), new SystemClock(), user,
            mock(TaskHeartbeatHandler.class), mockAppContext);
    when(mockAppContext.getCurrentDAG()).thenReturn(dag);

    dag.handle(new DAGEvent(dagId, DAGEventType.DAG_INIT));
    LOG.info("finish setUp");
}

---

From source file:org.apache.tez.dag.app.rm.node.TestAMNodeMap.java

License:Apache License

@Test
@SuppressWarnings({ "resource", "rawtypes" })
public void testHealthUpdateKnownNode() {

    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(new Configuration());
    dispatcher.start();//from   w ww .j ava 2s.  co  m
    EventHandler eventHandler = dispatcher.getEventHandler();

    AppContext appContext = mock(AppContext.class);

    AMNodeMap amNodeMap = new AMNodeMap(eventHandler, appContext);

    NodeId nodeId = NodeId.newInstance("host1", 2342);
    amNodeMap.nodeSeen(nodeId);

    NodeReport nodeReport = generateNodeReport(nodeId, NodeState.UNHEALTHY);
    amNodeMap.handle(new AMNodeEventStateChanged(nodeReport));
    dispatcher.await();
    assertEquals(AMNodeState.UNHEALTHY, amNodeMap.get(nodeId).getState());
    dispatcher.stop();
}

---

From source file:org.apache.tez.dag.app.rm.node.TestAMNodeMap.java

License:Apache License

@Test
@SuppressWarnings({ "resource", "rawtypes" })
public void testHealthUpdateUnknownNode() {
    DrainDispatcher dispatcher = new DrainDispatcher();
    EventHandler eventHandler = dispatcher.getEventHandler();

    AppContext appContext = mock(AppContext.class);

    AMNodeMap amNodeMap = new AMNodeMap(eventHandler, appContext);

    NodeId nodeId = NodeId.newInstance("unknownhost", 2342);

    NodeReport nodeReport = generateNodeReport(nodeId, NodeState.UNHEALTHY);
    amNodeMap.handle(new AMNodeEventStateChanged(nodeReport));

    // No exceptions - the status update was ignored. Not bothering to capture
    // the log message for verification.
    dispatcher.stop();/*w  w  w  .ja va 2s .c o m*/
}

---

From source file:org.apache.tez.dag.app.rm.node.TestAMNodeTracker.java

License:Apache License

@Before
public void setup() {
    dispatcher = new DrainDispatcher();
    dispatcher.init(new Configuration());
    dispatcher.start();/*from w ww.j a  v a2  s .c om*/
    eventHandler = dispatcher.getEventHandler();
}

---