Example usage for org.apache.hadoop.yarn.api.records.timeline TimelineEntityGroupId newInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records.timeline TimelineEntityGroupId newInstance.

Prototype

public static TimelineEntityGroupId newInstance(ApplicationId applicationId, String id) 

Source Link

Usage

From source file:org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin.java

License:Apache License

private Set<TimelineEntityGroupId> toGroupId(ApplicationId applicationId) {
    TimelineEntityGroupId groupId = TimelineEntityGroupId.newInstance(applicationId, applicationId.toString());
    LOG.debug("mapped {} to {}, ", applicationId, groupId);
    Set<TimelineEntityGroupId> result = new HashSet<>();
    result.add(groupId);/*from   w ww .  j  ava 2  s  .  c om*/
    return result;
}

---

From source file:org.apache.tez.dag.history.logging.ats.ATSV15HistoryLoggingService.java

License:Apache License

@VisibleForTesting
public TimelineEntityGroupId getGroupId(DAGHistoryEvent event) {
    // Changing this function will impact TimelineCachePluginImpl and should be done very
    // carefully to account for handling different versions of Tez
    switch (event.getHistoryEvent().getEventType()) {
    case DAG_SUBMITTED:
    case DAG_INITIALIZED:
    case DAG_STARTED:
    case DAG_FINISHED:
    case DAG_KILL_REQUEST:
    case VERTEX_INITIALIZED:
    case VERTEX_STARTED:
    case VERTEX_CONFIGURE_DONE:
    case VERTEX_FINISHED:
    case TASK_STARTED:
    case TASK_FINISHED:
    case TASK_ATTEMPT_STARTED:
    case TASK_ATTEMPT_FINISHED:
    case DAG_COMMIT_STARTED:
    case VERTEX_COMMIT_STARTED:
    case VERTEX_GROUP_COMMIT_STARTED:
    case VERTEX_GROUP_COMMIT_FINISHED:
    case DAG_RECOVERED:
        return TimelineEntityGroupId.newInstance(event.getDagID().getApplicationId(),
                event.getDagID().toString());
    case APP_LAUNCHED:
    case AM_LAUNCHED:
    case AM_STARTED:
    case CONTAINER_LAUNCHED:
    case CONTAINER_STOPPED:
        return TimelineEntityGroupId.newInstance(appContext.getApplicationID(),
                appContext.getApplicationID().toString());
    }/*from   w ww .  j  a va2  s.c  o m*/
    return null;
}

---

From source file:org.apache.tez.dag.history.logging.ats.TestATSV15HistoryLoggingService.java

License:Apache License

@Test(timeout = 2000)
public void testDAGGroupingDefault() throws Exception {
    ATSV15HistoryLoggingService service = createService(-1);

    service.start();/*from  ww w  . ja  v a 2  s  .c o m*/

    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }

    assertEquals(2, timelineClient.entityLog.size());

    List<TimelineEntity> amEvents = timelineClient.entityLog
            .get(TimelineEntityGroupId.newInstance(appId, appId.toString()));
    assertNotNull(amEvents);
    assertEquals(1, amEvents.size());

    List<TimelineEntity> nonGroupedDagEvents = timelineClient.entityLog
            .get(TimelineEntityGroupId.newInstance(appId, dagId1.toString()));
    assertNotNull(nonGroupedDagEvents);
    assertEquals(4, nonGroupedDagEvents.size());

    service.stop();
}

---

From source file:org.apache.tez.dag.history.logging.ats.TestATSV15HistoryLoggingService.java

License:Apache License

@Test(timeout = 2000)
public void testDAGGroupingDisabled() throws Exception {
    ATSV15HistoryLoggingService service = createService(1);
    service.start();//  w  ww .ja v  a 2 s  .  co m

    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }

    assertEquals(2, timelineClient.entityLog.size());

    List<TimelineEntity> amEvents = timelineClient.entityLog
            .get(TimelineEntityGroupId.newInstance(appId, appId.toString()));
    assertNotNull(amEvents);
    assertEquals(1, amEvents.size());

    List<TimelineEntity> nonGroupedDagEvents = timelineClient.entityLog
            .get(TimelineEntityGroupId.newInstance(appId, dagId1.toString()));
    assertNotNull(nonGroupedDagEvents);
    assertEquals(4, nonGroupedDagEvents.size());

    service.stop();
}

---

From source file:org.apache.tez.dag.history.logging.ats.TestATSV15HistoryLoggingService.java

License:Apache License

@Test(timeout = 2000)
public void testDAGGroupingGroupingEnabled() throws Exception {
    int numDagsPerGroup = 100;
    ATSV15HistoryLoggingService service = createService(numDagsPerGroup);
    service.start();//from w w  w  .  j av  a  2  s .c  o  m

    TezDAGID dagId1 = TezDAGID.getInstance(appId, 1);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    TezDAGID dagId2 = TezDAGID.getInstance(appId, numDagsPerGroup);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId2, service)) {
        service.handle(event);
    }

    TezDAGID dagId3 = TezDAGID.getInstance(appId, numDagsPerGroup + 1);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId3, service)) {
        service.handle(event);
    }

    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }

    assertEquals(dagId1.getGroupId(numDagsPerGroup), dagId2.getGroupId(numDagsPerGroup));
    assertNotEquals(dagId2.getGroupId(numDagsPerGroup), dagId3.getGroupId(numDagsPerGroup));

    assertEquals(3, timelineClient.entityLog.size());

    List<TimelineEntity> amEvents = timelineClient.entityLog
            .get(TimelineEntityGroupId.newInstance(appId, appId.toString()));
    assertNotNull(amEvents);
    assertEquals(3, amEvents.size());

    List<TimelineEntity> nonGroupedDagEvents = timelineClient.entityLog
            .get(TimelineEntityGroupId.newInstance(appId, dagId1.toString()));
    assertNull(nonGroupedDagEvents);

    List<TimelineEntity> groupedDagEvents = timelineClient.entityLog
            .get(TimelineEntityGroupId.newInstance(appId, dagId1.getGroupId(numDagsPerGroup)));
    assertNotNull(groupedDagEvents);
    assertEquals(8, groupedDagEvents.size());

    nonGroupedDagEvents = timelineClient.entityLog
            .get(TimelineEntityGroupId.newInstance(appId, dagId3.toString()));
    assertNull(nonGroupedDagEvents);

    groupedDagEvents = timelineClient.entityLog
            .get(TimelineEntityGroupId.newInstance(appId, dagId3.getGroupId(numDagsPerGroup)));
    assertNotNull(groupedDagEvents);
    assertEquals(4, groupedDagEvents.size());

    service.stop();
}

---

From source file:org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl.java

License:Apache License

private TimelineEntityGroupId convertToTimelineEntityGroupId(String entityType, String entityId) {
    if (entityType == null || entityType.isEmpty() || entityId == null || entityId.isEmpty()) {
        return null;
    }//from  w  w w . ja va 2 s.com
    if (entityType.equals(EntityTypes.TEZ_DAG_ID.name())) {
        TezDAGID dagId = TezDAGID.fromString(entityId);
        if (dagId != null) {
            return TimelineEntityGroupId.newInstance(dagId.getApplicationId(), dagId.toString());
        }
    } else if (entityType.equals(EntityTypes.TEZ_VERTEX_ID.name())) {
        TezVertexID vertexID = TezVertexID.fromString(entityId);
        if (vertexID != null) {
            return TimelineEntityGroupId.newInstance(vertexID.getDAGId().getApplicationId(),
                    vertexID.getDAGId().toString());
        }

    } else if (entityType.equals(EntityTypes.TEZ_TASK_ID.name())) {
        TezTaskID taskID = TezTaskID.fromString(entityId);
        if (taskID != null) {
            return TimelineEntityGroupId.newInstance(taskID.getVertexID().getDAGId().getApplicationId(),
                    taskID.getVertexID().getDAGId().toString());
        }
    } else if (entityType.equals(EntityTypes.TEZ_TASK_ATTEMPT_ID.name())) {
        TezTaskAttemptID taskAttemptID = TezTaskAttemptID.fromString(entityId);
        if (taskAttemptID != null) {
            return TimelineEntityGroupId.newInstance(
                    taskAttemptID.getTaskID().getVertexID().getDAGId().getApplicationId(),
                    taskAttemptID.getTaskID().getVertexID().getDAGId().toString());
        }
    } else if (entityType.equals(EntityTypes.TEZ_CONTAINER_ID.name())) {
        String cId = entityId;
        if (cId.startsWith("tez_")) {
            cId = cId.substring(4);
        }
        ContainerId containerId = ContainerId.fromString(cId);
        if (containerId != null) {
            return TimelineEntityGroupId.newInstance(containerId.getApplicationAttemptId().getApplicationId(),
                    containerId.getApplicationAttemptId().getApplicationId().toString());
        }
    }
    return null;
}

---

From source file:org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl.java

License:Apache License

@Override
public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityType, NameValuePair primaryFilter,
        Collection<NameValuePair> secondaryFilters) {
    if (!knownEntityTypes.contains(entityType) || primaryFilter == null
            || !knownEntityTypes.contains(primaryFilter.getName()) || summaryEntityTypes.contains(entityType)) {
        return null;
    }/*from w  w  w  .  ja v a 2s .com*/
    TimelineEntityGroupId groupId = convertToTimelineEntityGroupId(primaryFilter.getName(),
            primaryFilter.getValue().toString());
    if (groupId != null) {
        TimelineEntityGroupId appGroupId = TimelineEntityGroupId.newInstance(groupId.getApplicationId(),
                groupId.getApplicationId().toString());
        return Sets.newHashSet(groupId, appGroupId);
    }
    return null;
}

---

From source file:org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl.java

License:Apache License

@Override
public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityId, String entityType) {
    if (!knownEntityTypes.contains(entityType) || summaryEntityTypes.contains(entityType)) {
        return null;
    }//from ww w  .jav a  2 s .  co m
    TimelineEntityGroupId groupId = convertToTimelineEntityGroupId(entityType, entityId);
    if (groupId != null) {
        TimelineEntityGroupId appGroupId = TimelineEntityGroupId.newInstance(groupId.getApplicationId(),
                groupId.getApplicationId().toString());
        return Sets.newHashSet(groupId, appGroupId);
    }
    return null;
}

---

From source file:org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl.java

License:Apache License

@Override
public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityType, SortedSet<String> entityIds,
        Set<String> eventTypes) {
    if (!knownEntityTypes.contains(entityType) || summaryEntityTypes.contains(entityType) || entityIds == null
            || entityIds.isEmpty()) {/*from  ww w.java  2 s .c  om*/
        return null;
    }
    Set<TimelineEntityGroupId> groupIds = new HashSet<TimelineEntityGroupId>();
    Set<ApplicationId> appIdSet = new HashSet<ApplicationId>();

    for (String entityId : entityIds) {
        TimelineEntityGroupId groupId = convertToTimelineEntityGroupId(entityType, entityId);
        if (groupId != null) {
            groupIds.add(groupId);
            appIdSet.add(groupId.getApplicationId());
        }
    }
    for (ApplicationId appId : appIdSet) {
        groupIds.add(TimelineEntityGroupId.newInstance(appId, appId.toString()));
    }
    return groupIds;
}

---