Example usage for org.apache.hadoop.yarn.api.records ApplicationId toString

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationId toString.

Prototype

@Override
    public String toString() 

Source Link

Usage

From source file:org.apache.tez.dag.history.logging.ats.TestATSHistoryWithMiniCluster.java

License:Apache License

private void verifyEntityExistence(ApplicationId applicationId) {
    Assert.assertNotNull(timelineAddress);

    String appUrl = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_APPLICATION/" + "tez_"
            + applicationId.toString() + "?fields=otherinfo";
    LOG.info("Getting timeline entity for tez application: " + appUrl);
    TimelineEntity appEntity = getTimelineData(appUrl, TimelineEntity.class);
    Assert.assertNotNull(appEntity);//from  w  w  w .j  a va  2s  .  c o  m

    TezDAGID tezDAGID = TezDAGID.getInstance(applicationId, 1);
    String dagUrl = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + tezDAGID.toString()
            + "?fields=otherinfo";
    LOG.info("Getting timeline entity for tez dag: " + dagUrl);
    TimelineEntity dagEntity = getTimelineData(dagUrl, TimelineEntity.class);
    Assert.assertNotNull(dagEntity);
}

---

From source file:org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl.java

License:Apache License

@Override
public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityType, SortedSet<String> entityIds,
        Set<String> eventTypes) {
    if (!knownEntityTypes.contains(entityType) || summaryEntityTypes.contains(entityType) || entityIds == null
            || entityIds.isEmpty()) {/* ww w. j av a  2  s.c o m*/
        return null;
    }
    Set<TimelineEntityGroupId> groupIds = new HashSet<TimelineEntityGroupId>();
    Set<ApplicationId> appIdSet = new HashSet<ApplicationId>();

    for (String entityId : entityIds) {
        TimelineEntityGroupId groupId = convertToTimelineEntityGroupId(entityType, entityId);
        if (groupId != null) {
            groupIds.add(groupId);
            appIdSet.add(groupId.getApplicationId());
        }
    }
    for (ApplicationId appId : appIdSet) {
        groupIds.add(TimelineEntityGroupId.newInstance(appId, appId.toString()));
    }
    return groupIds;
}

---

From source file:org.apache.tez.dag.history.logging.proto.HistoryEventProtoConverter.java

License:Apache License

private HistoryEventProto.Builder makeBuilderForEvent(HistoryEvent event, long time, TezDAGID dagId,
        ApplicationId appId, ApplicationAttemptId appAttemptId, TezVertexID vertexId, TezTaskID taskId,
        TezTaskAttemptID taskAttemptId, String user) {
    HistoryEventProto.Builder builder = HistoryEventProto.newBuilder();
    builder.setEventType(event.getEventType().name());
    builder.setEventTime(time);//  w  ww . ja  v a  2  s  .  com
    if (taskAttemptId != null) {
        builder.setTaskAttemptId(taskAttemptId.toString());
        taskId = taskAttemptId.getTaskID();
    }
    if (taskId != null) {
        builder.setTaskId(taskId.toString());
        vertexId = taskId.getVertexID();
    }
    if (vertexId != null) {
        builder.setVertexId(vertexId.toString());
        dagId = vertexId.getDAGId();
    }
    if (dagId != null) {
        builder.setDagId(dagId.toString());
        if (appId == null) {
            appId = dagId.getApplicationId();
        }
    }
    if (appAttemptId != null) {
        builder.setAppAttemptId(appAttemptId.toString());
        if (appId == null) {
            appId = appAttemptId.getApplicationId();
        }
    }
    if (appId != null) {
        builder.setAppId(appId.toString());
    }
    if (user != null) {
        builder.setUser(user);
    }
    return builder;
}

---

From source file:org.apache.tez.dag.history.logging.proto.TestDagManifestFileScanner.java

License:Apache License

private void createManifestEvents(long time, int numEvents) throws IOException {
    for (int i = 0; i < numEvents; ++i) {
        ApplicationId appId = ApplicationId.newInstance(1000l, i);
        ManifestEntryProto proto = ManifestEntryProto.newBuilder().setAppId(appId.toString())
                .setDagFilePath("dummy_dag_path_" + i).setDagSubmittedEventOffset(0)
                .setDagFinishedEventOffset(1).setAppFilePath("dummp_app_path_" + i).setAppLaunchedEventOffset(2)
                .setWriteTime(clock.getTime()).build();
        ProtoMessageWriter<ManifestEntryProto> writer = manifestLogger.getWriter(appId.toString());
        writer.writeProto(proto);//w w w  . java2 s.  co m
        writer.close();
    }
}

---

From source file:org.apache.tez.engine.common.shuffle.server.ShuffleHandler.java

License:Apache License

@Override
public void initializeApplication(ApplicationInitializationContext initAppContext) {
    // TODO these bytes should be versioned
    try {/*from w  w w  .jav a  2 s  . com*/
        String user = initAppContext.getUser();
        ApplicationId appId = initAppContext.getApplicationId();
        ByteBuffer secret = initAppContext.getApplicationDataForService();
        Token<JobTokenIdentifier> jt = deserializeServiceData(secret);
        // TODO: Once SHuffle is out of NM, this can use MR APIs
        userRsrc.put(appId.toString(), user);
        LOG.info("Added token for " + appId.toString());
        secretManager.addTokenForJob(appId.toString(), jt);
    } catch (IOException e) {
        LOG.error("Error during initApp", e);
        // TODO add API to AuxiliaryServices to report failures
    }
}

---

From source file:org.apache.tez.engine.common.shuffle.server.ShuffleHandler.java

License:Apache License

@Override
public void stopApplication(ApplicationTerminationContext context) {
    ApplicationId appId = context.getApplicationId();
    secretManager.removeTokenForJob(appId.toString());
    userRsrc.remove(appId.toString());/*from   ww w .  jav  a 2 s .com*/
}

---

From source file:org.apache.tez.history.TestHistoryParser.java

License:Apache License

private String runWordCount(String tokenizerProcessor, String summationProcessor, String dagName,
        boolean withTimeline) throws Exception {
    //HDFS path/*from   w  ww  . j a  v  a  2s  . c  om*/
    Path outputLoc = new Path("/tmp/outPath_" + System.currentTimeMillis());

    DataSourceDescriptor dataSource = MRInput
            .createConfigBuilder(conf, TextInputFormat.class, inputLoc.toString()).build();

    DataSinkDescriptor dataSink = MROutput
            .createConfigBuilder(conf, TextOutputFormat.class, outputLoc.toString()).build();

    Vertex tokenizerVertex = Vertex.create(TOKENIZER, ProcessorDescriptor.create(tokenizerProcessor))
            .addDataSource(INPUT, dataSource);

    OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
            .newBuilder(Text.class.getName(), IntWritable.class.getName(), HashPartitioner.class.getName())
            .build();

    Vertex summationVertex = Vertex.create(SUMMATION, ProcessorDescriptor.create(summationProcessor), 1)
            .addDataSink(OUTPUT, dataSink);

    // Create DAG and add the vertices. Connect the producer and consumer vertices via the edge
    DAG dag = DAG.create(dagName);
    dag.addVertex(tokenizerVertex).addVertex(summationVertex)
            .addEdge(Edge.create(tokenizerVertex, summationVertex, edgeConf.createDefaultEdgeProperty()));

    TezClient tezClient = getTezClient(withTimeline);

    // Update Caller Context
    CallerContext callerContext = CallerContext.create("TezExamples", "Tez WordCount Example Job");
    ApplicationId appId = tezClient.getAppMasterApplicationId();
    if (appId == null) {
        appId = ApplicationId.newInstance(1001l, 1);
    }
    callerContext.setCallerIdAndType(appId.toString(), "TezApplication");
    dag.setCallerContext(callerContext);

    DAGClient client = tezClient.submitDAG(dag);
    client.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezDAGID tezDAGID = TezDAGID.getInstance(tezClient.getAppMasterApplicationId(), 1);

    if (tezClient != null) {
        tezClient.stop();
    }
    return tezDAGID.toString();
}

---

From source file:org.apache.tez.runtime.library.shuffle.common.ShuffleUtils.java

License:Apache License

public static StringBuilder constructBaseURIForShuffleHandler(String host, int port, int partition,
        ApplicationId appId) {
    StringBuilder sb = new StringBuilder("http://");
    sb.append(host);/*from ww w.  ja va 2 s . co  m*/
    sb.append(":");
    sb.append(String.valueOf(port));
    sb.append("/");
    sb.append("mapOutput?job=");
    sb.append(appId.toString().replace("application", "job"));
    sb.append("&reduce=");
    sb.append(String.valueOf(partition));
    sb.append("&map=");
    return sb;
}

---

From source file:org.apache.tez.test.TestAMRecovery.java

License:Apache License

private List<HistoryEvent> readRecoveryLog(int attemptNum) throws IOException {
    ApplicationId appId = tezSession.getAppMasterApplicationId();
    Path tezSystemStagingDir = TezCommonUtils.getTezSystemStagingPath(tezConf, appId.toString());
    Path recoveryDataDir = TezCommonUtils.getRecoveryPath(tezSystemStagingDir, tezConf);
    FileSystem fs = tezSystemStagingDir.getFileSystem(tezConf);
    Path currentAttemptRecoveryDataDir = TezCommonUtils.getAttemptRecoveryPath(recoveryDataDir, attemptNum);
    Path recoveryFilePath = new Path(currentAttemptRecoveryDataDir,
            appId.toString().replace("application", "dag") + "_1"
                    + TezConstants.DAG_RECOVERY_RECOVER_FILE_SUFFIX);
    return RecoveryParser.parseDAGRecoveryFile(fs.open(recoveryFilePath));
}

---

From source file:org.apache.tez.test.TestDAGRecovery.java

License:Apache License

private void verifyRecoveryLog() throws IOException {
    ApplicationId appId = tezSession.getAppMasterApplicationId();
    Path tezSystemStagingDir = TezCommonUtils.getTezSystemStagingPath(tezConf, appId.toString());
    Path recoveryDataDir = TezCommonUtils.getRecoveryPath(tezSystemStagingDir, tezConf);

    FileSystem fs = tezSystemStagingDir.getFileSystem(tezConf);
    for (int i = 1; i <= 3; ++i) {
        Path currentAttemptRecoveryDataDir = TezCommonUtils.getAttemptRecoveryPath(recoveryDataDir, i);
        Path recoveryFilePath = new Path(currentAttemptRecoveryDataDir,
                appId.toString().replace("application", "dag") + "_1"
                        + TezConstants.DAG_RECOVERY_RECOVER_FILE_SUFFIX);
        List<HistoryEvent> historyEvents = RecoveryParser.parseDAGRecoveryFile(fs.open(recoveryFilePath));

        int inputInfoEventIndex = -1;
        int vertexInitedEventIndex = -1;
        for (int j = 0; j < historyEvents.size(); ++j) {
            HistoryEvent historyEvent = historyEvents.get(j);
            LOG.info("Parsed event from recovery stream" + ", eventType=" + historyEvent.getEventType()
                    + ", event=" + historyEvent);
            if (historyEvent.getEventType() == HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED) {
                VertexRecoverableEventsGeneratedEvent dmEvent = (VertexRecoverableEventsGeneratedEvent) historyEvent;
                // TODO do not need to check whether it is -1 after Tez-1521 is resolved
                if (dmEvent.getVertexID().getId() == 0 && inputInfoEventIndex == -1) {
                    inputInfoEventIndex = j;
                }/*from  ww  w .j  a  v a  2s  . co  m*/
            }
            if (historyEvent.getEventType() == HistoryEventType.VERTEX_INITIALIZED) {
                VertexInitializedEvent vInitedEvent = (VertexInitializedEvent) historyEvent;
                if (vInitedEvent.getVertexID().getId() == 0) {
                    vertexInitedEventIndex = j;
                }
            }
        }
        // v1's init events must be logged before its VertexInitializedEvent (Tez-1345)
        Assert.assertTrue("can not find VERTEX_DATA_MOVEMENT_EVENTS_GENERATED for v1",
                inputInfoEventIndex != -1);
        Assert.assertTrue("can not find VERTEX_INITIALIZED for v1", vertexInitedEventIndex != -1);
        Assert.assertTrue("VERTEX_DATA_MOVEMENT_EVENTS_GENERATED is logged before VERTEX_INITIALIZED for v1",
                inputInfoEventIndex < vertexInitedEventIndex);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• »