Example usage for org.apache.hadoop.yarn.api.records ContainerId toString

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ContainerId toString.

Prototype

@Override
public String toString() 

Source Link

Usage

From source file:org.apache.tez.dag.app.TestTaskAttemptListenerImplTezDag.java

License:Apache License

@Test(timeout = 5000)
public void testGetTask() throws IOException {
    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    AppContext appContext = mock(AppContext.class);
    EventHandler eventHandler = mock(EventHandler.class);
    DAG dag = mock(DAG.class);
    AMContainerMap amContainerMap = mock(AMContainerMap.class);
    Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(dag).when(appContext).getCurrentDAG();
    doReturn(appAcls).when(appContext).getApplicationACLs();
    doReturn(amContainerMap).when(appContext).getAllContainers();

    TaskAttemptListenerImpTezDag taskAttemptListener = new TaskAttemptListenerImplForTest(appContext,
            mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), null);

    TaskSpec taskSpec = mock(TaskSpec.class);
    TezTaskAttemptID taskAttemptId = mock(TezTaskAttemptID.class);
    doReturn(taskAttemptId).when(taskSpec).getTaskAttemptID();
    AMContainerTask amContainerTask = new AMContainerTask(taskSpec, null, null, false);
    ContainerTask containerTask = null;/*ww w.j a v  a2  s  . c o  m*/

    ContainerId containerId1 = createContainerId(appId, 1);
    doReturn(mock(AMContainer.class)).when(amContainerMap).get(containerId1);
    ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
    containerTask = taskAttemptListener.getTask(containerContext1);
    assertTrue(containerTask.shouldDie());

    ContainerId containerId2 = createContainerId(appId, 2);
    doReturn(mock(AMContainer.class)).when(amContainerMap).get(containerId2);
    ContainerContext containerContext2 = new ContainerContext(containerId2.toString());
    taskAttemptListener.registerRunningContainer(containerId2);
    containerTask = taskAttemptListener.getTask(containerContext2);
    assertNull(containerTask);

    // Valid task registered
    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2);
    containerTask = taskAttemptListener.getTask(containerContext2);
    assertFalse(containerTask.shouldDie());
    assertEquals(taskSpec, containerTask.getTaskSpec());

    // Task unregistered. Should respond to heartbeats
    taskAttemptListener.unregisterTaskAttempt(taskAttemptId);
    containerTask = taskAttemptListener.getTask(containerContext2);
    assertNull(containerTask);

    // Container unregistered. Should send a shouldDie = true
    taskAttemptListener.unregisterRunningContainer(containerId2);
    containerTask = taskAttemptListener.getTask(containerContext2);
    assertTrue(containerTask.shouldDie());

    ContainerId containerId3 = createContainerId(appId, 3);
    ContainerContext containerContext3 = new ContainerContext(containerId3.toString());
    taskAttemptListener.registerRunningContainer(containerId3);

    // Register task to container3, followed by unregistering container 3 all together
    TaskSpec taskSpec2 = mock(TaskSpec.class);
    TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
    doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
    AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec, null, null, false);
    taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3);
    taskAttemptListener.unregisterRunningContainer(containerId3);
    containerTask = taskAttemptListener.getTask(containerContext3);
    assertTrue(containerTask.shouldDie());
}

---

From source file:org.apache.tez.dag.app.TestTaskAttemptListenerImplTezDag.java

License:Apache License

@Test(timeout = 5000)
public void testGetTaskMultiplePulls() throws IOException {
    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    AppContext appContext = mock(AppContext.class);
    EventHandler eventHandler = mock(EventHandler.class);
    DAG dag = mock(DAG.class);
    AMContainerMap amContainerMap = mock(AMContainerMap.class);
    Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(dag).when(appContext).getCurrentDAG();
    doReturn(appAcls).when(appContext).getApplicationACLs();
    doReturn(amContainerMap).when(appContext).getAllContainers();

    TaskAttemptListenerImpTezDag taskAttemptListener = new TaskAttemptListenerImplForTest(appContext,
            mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), null);

    TaskSpec taskSpec = mock(TaskSpec.class);
    TezTaskAttemptID taskAttemptId = mock(TezTaskAttemptID.class);
    doReturn(taskAttemptId).when(taskSpec).getTaskAttemptID();
    AMContainerTask amContainerTask = new AMContainerTask(taskSpec, null, null, false);
    ContainerTask containerTask = null;/*ww w . ja va 2 s.c  o  m*/

    ContainerId containerId1 = createContainerId(appId, 1);
    doReturn(mock(AMContainer.class)).when(amContainerMap).get(containerId1);
    ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
    taskAttemptListener.registerRunningContainer(containerId1);
    containerTask = taskAttemptListener.getTask(containerContext1);
    assertNull(containerTask);

    // Register task
    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1);
    containerTask = taskAttemptListener.getTask(containerContext1);
    assertFalse(containerTask.shouldDie());
    assertEquals(taskSpec, containerTask.getTaskSpec());

    // Try pulling again - simulates re-use pull
    containerTask = taskAttemptListener.getTask(containerContext1);
    assertNull(containerTask);
}

---

From source file:org.apache.tez.dag.app.TestTaskCommunicatorManager1.java

License:Apache License

@Test(timeout = 5000)
public void testGetTask() throws IOException {

    TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl) taskAttemptListener
            .getTaskCommunicator(0).getTaskCommunicator();
    TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();

    ContainerId containerId1 = createContainerId(appId, 1);
    ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
    containerTask = tezUmbilical.getTask(containerContext1);
    assertTrue(containerTask.shouldDie());

    ContainerId containerId2 = createContainerId(appId, 2);
    ContainerContext containerContext2 = new ContainerContext(containerId2.toString());
    taskAttemptListener.registerRunningContainer(containerId2, 0);
    containerTask = tezUmbilical.getTask(containerContext2);
    assertNull(containerTask);// www .j  av  a 2s  . co  m

    // Valid task registered
    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2, 0);
    containerTask = tezUmbilical.getTask(containerContext2);
    assertFalse(containerTask.shouldDie());
    assertEquals(taskSpec, containerTask.getTaskSpec());

    // Task unregistered. Should respond to heartbeats
    taskAttemptListener.unregisterTaskAttempt(taskAttemptID, 0, TaskAttemptEndReason.OTHER, null);
    containerTask = tezUmbilical.getTask(containerContext2);
    assertNull(containerTask);

    // Container unregistered. Should send a shouldDie = true
    taskAttemptListener.unregisterRunningContainer(containerId2, 0, ContainerEndReason.OTHER, null);
    containerTask = tezUmbilical.getTask(containerContext2);
    assertTrue(containerTask.shouldDie());

    ContainerId containerId3 = createContainerId(appId, 3);
    ContainerContext containerContext3 = new ContainerContext(containerId3.toString());
    taskAttemptListener.registerRunningContainer(containerId3, 0);

    // Register task to container3, followed by unregistering container 3 all together
    TaskSpec taskSpec2 = mock(TaskSpec.class);
    TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
    doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
    AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec, null, null, false, 0);
    taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3, 0);
    taskAttemptListener.unregisterRunningContainer(containerId3, 0, ContainerEndReason.OTHER, null);
    containerTask = tezUmbilical.getTask(containerContext3);
    assertTrue(containerTask.shouldDie());
}

---

From source file:org.apache.tez.dag.app.TestTaskCommunicatorManager1.java

License:Apache License

@Test(timeout = 5000)
public void testGetTaskMultiplePulls() throws IOException {
    TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl) taskAttemptListener
            .getTaskCommunicator(0).getTaskCommunicator();
    TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();

    ContainerId containerId1 = createContainerId(appId, 1);

    ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
    taskAttemptListener.registerRunningContainer(containerId1, 0);
    containerTask = tezUmbilical.getTask(containerContext1);
    assertNull(containerTask);/*  www .java2s  . c o m*/

    // Register task
    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1, 0);
    containerTask = tezUmbilical.getTask(containerContext1);
    assertFalse(containerTask.shouldDie());
    assertEquals(taskSpec, containerTask.getTaskSpec());

    // Try pulling again - simulates re-use pull
    containerTask = tezUmbilical.getTask(containerContext1);
    assertNull(containerTask);
}

---

From source file:org.apache.tez.dag.app.TestTaskCommunicatorManager1.java

License:Apache License

private TaskHeartbeatResponse generateHeartbeat(List<TezEvent> events, int fromEventId, int maxEvents,
        int nextFromEventId, List<TezEvent> sendEvents) throws IOException, TezException {
    ContainerId containerId = createContainerId(appId, 1);
    Vertex vertex = mock(Vertex.class);

    doReturn(vertex).when(dag).getVertex(vertexID);
    doReturn("test_vertex").when(vertex).getName();
    TaskAttemptEventInfo eventInfo = new TaskAttemptEventInfo(nextFromEventId, sendEvents, 0);
    doReturn(eventInfo).when(vertex).getTaskAttemptTezEvents(taskAttemptID, fromEventId, 0, maxEvents);

    taskAttemptListener.registerRunningContainer(containerId, 0);
    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId, 0);

    TaskHeartbeatRequest request = mock(TaskHeartbeatRequest.class);
    doReturn(containerId.toString()).when(request).getContainerIdentifier();
    doReturn(containerId.toString()).when(request).getContainerIdentifier();
    doReturn(taskAttemptID).when(request).getTaskAttemptId();
    doReturn(events).when(request).getEvents();
    doReturn(maxEvents).when(request).getMaxEvents();
    doReturn(fromEventId).when(request).getStartIndex();

    return taskAttemptListener.heartbeat(request);
}

---

From source file:org.apache.tez.dag.app.TestTaskCommunicatorManager2.java

License:Apache License

@SuppressWarnings("unchecked")
@Test(timeout = 5000)/*from ww  w  .j a v  a2s.c o m*/
public void testTaskAttemptFailureViaHeartbeat() throws IOException, TezException {

    TaskCommunicatorManagerWrapperForTest wrapper = new TaskCommunicatorManagerWrapperForTest();

    TaskSpec taskSpec1 = wrapper.createTaskSpec();
    AMContainerTask amContainerTask1 = new AMContainerTask(taskSpec1, null, null, false, 10);

    TaskSpec taskSpec2 = wrapper.createTaskSpec();
    AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec2, null, null, false, 10);

    ContainerId containerId1 = wrapper.createContainerId(1);
    wrapper.registerRunningContainer(containerId1);
    wrapper.registerTaskAttempt(containerId1, amContainerTask1);

    ContainerId containerId2 = wrapper.createContainerId(2);
    wrapper.registerRunningContainer(containerId2);
    wrapper.registerTaskAttempt(containerId2, amContainerTask2);

    List<TezEvent> events = new LinkedList<>();

    EventMetaData sourceInfo1 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR,
            "testVertex", null, taskSpec1.getTaskAttemptID());
    TaskAttemptFailedEvent failedEvent1 = new TaskAttemptFailedEvent("non-fatal test error",
            TaskFailureType.NON_FATAL);
    TezEvent failedEventT1 = new TezEvent(failedEvent1, sourceInfo1);
    events.add(failedEventT1);
    TaskHeartbeatRequest taskHeartbeatRequest1 = new TaskHeartbeatRequest(containerId1.toString(),
            taskSpec1.getTaskAttemptID(), events, 0, 0, 0);
    wrapper.getTaskCommunicatorManager().heartbeat(taskHeartbeatRequest1);

    ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
    verify(wrapper.getEventHandler(), times(1)).handle(argumentCaptor.capture());
    assertTrue(argumentCaptor.getAllValues().get(0) instanceof TaskAttemptEventAttemptFailed);
    TaskAttemptEventAttemptFailed failedEvent = (TaskAttemptEventAttemptFailed) argumentCaptor.getAllValues()
            .get(0);
    assertEquals(TaskFailureType.NON_FATAL, failedEvent.getTaskFailureType());
    assertTrue(failedEvent.getDiagnosticInfo().contains("non-fatal"));

    events.clear();
    reset(wrapper.getEventHandler());

    EventMetaData sourceInfo2 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR,
            "testVertex", null, taskSpec2.getTaskAttemptID());
    TaskAttemptFailedEvent failedEvent2 = new TaskAttemptFailedEvent("-fatal- test error",
            TaskFailureType.FATAL);
    TezEvent failedEventT2 = new TezEvent(failedEvent2, sourceInfo2);
    events.add(failedEventT2);
    TaskHeartbeatRequest taskHeartbeatRequest2 = new TaskHeartbeatRequest(containerId2.toString(),
            taskSpec2.getTaskAttemptID(), events, 0, 0, 0);
    wrapper.getTaskCommunicatorManager().heartbeat(taskHeartbeatRequest2);

    argumentCaptor = ArgumentCaptor.forClass(Event.class);
    verify(wrapper.getEventHandler(), times(1)).handle(argumentCaptor.capture());
    assertTrue(argumentCaptor.getAllValues().get(0) instanceof TaskAttemptEventAttemptFailed);
    failedEvent = (TaskAttemptEventAttemptFailed) argumentCaptor.getAllValues().get(0);
    assertEquals(TaskFailureType.FATAL, failedEvent.getTaskFailureType());
    assertTrue(failedEvent.getDiagnosticInfo().contains("-fatal-"));
}

---

From source file:org.apache.tez.runtime.task.TestContainerExecution.java

License:Apache License

@Test(timeout = 5000)
public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
    ListeningExecutorService executor = null;
    try {//from  w ww . ja  v a 2  s  . com
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        @SuppressWarnings("deprecation")
        ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);

        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        ContainerContext containerContext = new ContainerContext(containerId.toString());

        ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
        ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);

        getTaskFuture.get();
        assertEquals(1, umbilical.getTaskInvocations);

    } finally {
        executor.shutdownNow();
    }
}

---

From source file:org.apache.tez.runtime.task.TestTaskExecution.java

License:Apache License

@Test(timeout = 5000)
public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
    ListeningExecutorService executor = null;
    try {/*w w w . ja  v a  2s  .c  om*/
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);

        TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
        ContainerContext containerContext = new ContainerContext(containerId.toString());

        ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
        ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);

        getTaskFuture.get();
        assertEquals(1, umbilical.getTaskInvocations);

    } finally {
        executor.shutdownNow();
    }
}

---

From source file:org.apache.twill.internal.appmaster.ApplicationMasterService.java

License:Apache License

private RunningContainers initRunningContainers(ContainerId appMasterContainerId, String appMasterHost)
        throws Exception {
    TwillRunResources appMasterResources = new DefaultTwillRunResources(0, appMasterContainerId.toString(),
            Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES)),
            Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_MEMORY_MB)), appMasterHost, null, null);
    String appId = appMasterContainerId.getApplicationAttemptId().getApplicationId().toString();
    return new RunningContainers(appId, appMasterResources, zkClient);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• »