Example usage for org.apache.hadoop.yarn.api.records ApplicationId newInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationId newInstance.

Prototype

@Public
    @Unstable
    public static ApplicationId newInstance(long clusterTimestamp, int id) 

Source Link

Usage

From source file:org.apache.tez.dag.history.recovery.TestRecoveryService.java

License:Apache License

@Test(timeout = 5000)
public void testMultipleDAGFinishedEvent() throws IOException {
    Configuration conf = new Configuration();
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(TEST_ROOT_DIR));
    when(appContext.getClock()).thenReturn(new SystemClock());
    when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
    when(appContext.getApplicationID()).thenReturn(appId);

    MockRecoveryService recoveryService = new MockRecoveryService(appContext);
    conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
    recoveryService.init(conf);//from w  w  w . ja  v  a  2  s  . c o  m
    recoveryService.start();
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1);
    int randEventCount = new Random().nextInt(100) + 100;
    for (int i = 0; i < randEventCount; ++i) {
        recoveryService.handle(new DAGHistoryEvent(dagId, new TaskStartedEvent(
                TezTaskID.getInstance(TezVertexID.getInstance(dagId, 1), 1), "v1", 0L, 0L)));
    }
    recoveryService.await();
    assertTrue(recoveryService.outputStreamMap.containsKey(dagId));
    // 2 DAGFinishedEvent
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, 1L, 2L, DAGState.FAILED,
            "diag", null, "user", "dag1", null, appAttemptId, null)));
    // outputStream removed
    assertFalse(recoveryService.outputStreamMap.containsKey(dagId));
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, 1L, 2L, DAGState.ERROR,
            "diag", null, "user", "dag1", null, appAttemptId, null)));
    // no new outputStream opened
    assertEquals(recoveryService.outputStreamMap.size(), 0);
    assertFalse(recoveryService.outputStreamMap.containsKey(dagId));
    recoveryService.stop();
}

---

From source file:org.apache.tez.dag.history.recovery.TestRecoveryService.java

License:Apache License

@Test(timeout = 5000)
public void testSummaryPathExisted() throws IOException {
    Configuration conf = new Configuration();
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(TEST_ROOT_DIR));
    when(appContext.getClock()).thenReturn(new SystemClock());
    when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
    when(appContext.getApplicationID()).thenReturn(appId);

    MockRecoveryService recoveryService = new MockRecoveryService(appContext);
    conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
    recoveryService.init(conf);/*  w  w  w.j  av a2  s  .c  om*/
    recoveryService.start();
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1);
    Path dagRecoveryPath = TezCommonUtils.getSummaryRecoveryPath(recoveryService.recoveryPath);
    touchFile(dagRecoveryPath);
    assertFalse(recoveryService.hasRecoveryFailed());
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, 1L, 2L, DAGState.ERROR,
            "diag", null, "user", "dag1", null, appAttemptId, null)));
    assertTrue(recoveryService.hasRecoveryFailed());
    // be able to handle event after fatal error
    recoveryService.handle(new DAGHistoryEvent(dagId, new DAGFinishedEvent(dagId, 1L, 2L, DAGState.ERROR,
            "diag", null, "user", "dag1", null, appAttemptId, null)));
}

---

From source file:org.apache.tez.dag.history.recovery.TestRecoveryService.java

License:Apache License

@Test(timeout = 5000)
public void testRecoveryPathExisted() throws IOException {
    Configuration conf = new Configuration();
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(TEST_ROOT_DIR));
    when(appContext.getClock()).thenReturn(new SystemClock());
    when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
    when(appContext.getApplicationID()).thenReturn(appId);

    MockRecoveryService recoveryService = new MockRecoveryService(appContext);
    conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
    recoveryService.init(conf);/*from   www  . ja  va 2  s .co  m*/
    recoveryService.start();
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1);
    Path dagRecoveryPath = TezCommonUtils.getDAGRecoveryPath(recoveryService.recoveryPath, dagId.toString());
    touchFile(dagRecoveryPath);
    assertFalse(recoveryService.hasRecoveryFailed());
    recoveryService.handle(new DAGHistoryEvent(dagId,
            new TaskStartedEvent(TezTaskID.getInstance(TezVertexID.getInstance(dagId, 1), 1), "v1", 0L, 0L)));
    // wait for recovery event to be handled
    recoveryService.await();
    assertTrue(recoveryService.hasRecoveryFailed());
    // be able to handle recovery event after fatal error
    recoveryService.handle(new DAGHistoryEvent(dagId,
            new TaskStartedEvent(TezTaskID.getInstance(TezVertexID.getInstance(dagId, 1), 1), "v1", 0L, 0L)));
}

---

From source file:org.apache.tez.dag.history.utils.TestDAGUtils.java

License:Apache License

@Test(timeout = 5000)
@SuppressWarnings("unchecked")
public void testConvertDAGPlanToATSMap() throws IOException, JSONException {
    DAGPlan dagPlan = createDAG();/*from  w  w w  . j  ava 2s .  c o  m*/
    Map<String, TezVertexID> idNameMap = new HashMap<String, TezVertexID>();
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vId1 = TezVertexID.getInstance(dagId, 1);
    TezVertexID vId2 = TezVertexID.getInstance(dagId, 2);
    TezVertexID vId3 = TezVertexID.getInstance(dagId, 3);
    idNameMap.put("vertex1", vId1);
    idNameMap.put("vertex2", vId2);
    idNameMap.put("vertex3", vId3);

    Map<String, Object> atsMap = DAGUtils.convertDAGPlanToATSMap(dagPlan);
    Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_NAME_KEY));
    Assert.assertEquals("testDag", atsMap.get(DAGUtils.DAG_NAME_KEY));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_INFO_KEY));
    Assert.assertEquals("dagInfo", atsMap.get(DAGUtils.DAG_INFO_KEY));
    Assert.assertEquals(dagPlan.getName(), atsMap.get(DAGUtils.DAG_NAME_KEY));
    Assert.assertTrue(atsMap.containsKey("version"));
    Assert.assertEquals(1, atsMap.get("version"));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.VERTICES_KEY));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.EDGES_KEY));
    Assert.assertTrue(atsMap.containsKey(DAGUtils.VERTEX_GROUPS_KEY));

    Assert.assertEquals(3, ((Collection<?>) atsMap.get(DAGUtils.VERTICES_KEY)).size());

    Set<String> inEdgeIds = new HashSet<String>();
    Set<String> outEdgeIds = new HashSet<String>();

    int additionalInputCount = 0;
    int additionalOutputCount = 0;

    for (Object o : ((Collection<?>) atsMap.get(DAGUtils.VERTICES_KEY))) {
        Map<String, Object> v = (Map<String, Object>) o;
        Assert.assertTrue(v.containsKey(DAGUtils.VERTEX_NAME_KEY));
        String vName = (String) v.get(DAGUtils.VERTEX_NAME_KEY);
        Assert.assertTrue(v.containsKey(DAGUtils.PROCESSOR_CLASS_KEY));
        Assert.assertTrue(v.containsKey(DAGUtils.USER_PAYLOAD_AS_TEXT));

        if (v.containsKey(DAGUtils.IN_EDGE_IDS_KEY)) {
            inEdgeIds.addAll(((Collection<String>) v.get(DAGUtils.IN_EDGE_IDS_KEY)));
        }
        if (v.containsKey(DAGUtils.OUT_EDGE_IDS_KEY)) {
            outEdgeIds.addAll(((Collection<String>) v.get(DAGUtils.OUT_EDGE_IDS_KEY)));
        }

        Assert.assertTrue(idNameMap.containsKey(vName));
        String procPayload = vName + " Processor HistoryText";
        Assert.assertEquals(procPayload, v.get(DAGUtils.USER_PAYLOAD_AS_TEXT));

        if (v.containsKey(DAGUtils.ADDITIONAL_INPUTS_KEY)) {
            additionalInputCount += ((Collection<?>) v.get(DAGUtils.ADDITIONAL_INPUTS_KEY)).size();
            for (Object input : ((Collection<?>) v.get(DAGUtils.ADDITIONAL_INPUTS_KEY))) {
                Map<String, Object> inputMap = (Map<String, Object>) input;
                Assert.assertTrue(inputMap.containsKey(DAGUtils.NAME_KEY));
                Assert.assertTrue(inputMap.containsKey(DAGUtils.CLASS_KEY));
                Assert.assertFalse(inputMap.containsKey(DAGUtils.INITIALIZER_KEY));
                Assert.assertEquals("input HistoryText", inputMap.get(DAGUtils.USER_PAYLOAD_AS_TEXT));
            }
        }

        if (v.containsKey(DAGUtils.ADDITIONAL_OUTPUTS_KEY)) {
            additionalOutputCount += ((Collection<?>) v.get(DAGUtils.ADDITIONAL_OUTPUTS_KEY)).size();
            for (Object output : ((Collection<?>) v.get(DAGUtils.ADDITIONAL_OUTPUTS_KEY))) {
                Map<String, Object> outputMap = (Map<String, Object>) output;
                Assert.assertTrue(outputMap.containsKey(DAGUtils.NAME_KEY));
                Assert.assertTrue(outputMap.containsKey(DAGUtils.CLASS_KEY));
                Assert.assertTrue(outputMap.containsKey(DAGUtils.INITIALIZER_KEY));
                Assert.assertEquals("uvOut HistoryText", outputMap.get(DAGUtils.USER_PAYLOAD_AS_TEXT));
            }
        }
    }

    // 1 input
    Assert.assertEquals(1, additionalInputCount);
    // 3 outputs due to vertex group
    Assert.assertEquals(3, additionalOutputCount);

    // 1 edge translates to 2 due to vertex group
    Assert.assertEquals(2, inEdgeIds.size());
    Assert.assertEquals(2, outEdgeIds.size());

    for (Object o : ((Collection<?>) atsMap.get(DAGUtils.EDGES_KEY))) {
        Map<String, Object> e = (Map<String, Object>) o;

        Assert.assertTrue(inEdgeIds.contains(e.get(DAGUtils.EDGE_ID_KEY)));
        Assert.assertTrue(outEdgeIds.contains(e.get(DAGUtils.EDGE_ID_KEY)));
        Assert.assertTrue(e.containsKey(DAGUtils.INPUT_VERTEX_NAME_KEY));
        Assert.assertTrue(e.containsKey(DAGUtils.OUTPUT_VERTEX_NAME_KEY));
        Assert.assertEquals(DataMovementType.SCATTER_GATHER.name(), e.get(DAGUtils.DATA_MOVEMENT_TYPE_KEY));
        Assert.assertEquals(DataSourceType.PERSISTED.name(), e.get(DAGUtils.DATA_SOURCE_TYPE_KEY));
        Assert.assertEquals(SchedulingType.SEQUENTIAL.name(), e.get(DAGUtils.SCHEDULING_TYPE_KEY));
        Assert.assertEquals("dummy output class", e.get(DAGUtils.EDGE_SOURCE_CLASS_KEY));
        Assert.assertEquals("dummy input class", e.get(DAGUtils.EDGE_DESTINATION_CLASS_KEY));
        Assert.assertEquals("Dummy History Text", e.get(DAGUtils.OUTPUT_USER_PAYLOAD_AS_TEXT));
        Assert.assertEquals("Dummy History Text", e.get(DAGUtils.INPUT_USER_PAYLOAD_AS_TEXT));
    }

    for (Object o : ((Collection<?>) atsMap.get(DAGUtils.VERTEX_GROUPS_KEY))) {
        Map<String, Object> e = (Map<String, Object>) o;
        Assert.assertEquals("uv12", e.get(DAGUtils.VERTEX_GROUP_NAME_KEY));
        Assert.assertTrue(e.containsKey(DAGUtils.VERTEX_GROUP_MEMBERS_KEY));
        Assert.assertTrue(e.containsKey(DAGUtils.VERTEX_GROUP_OUTPUTS_KEY));
        Assert.assertTrue(e.containsKey(DAGUtils.VERTEX_GROUP_EDGE_MERGED_INPUTS_KEY));
    }
}

---

From source file:org.apache.tez.dag.records.TestTezIds.java

License:Apache License

@Test(timeout = 5000)
public void testIdStringify() {
    ApplicationId appId = ApplicationId.newInstance(9999, 72);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vId = TezVertexID.getInstance(dagId, 35);
    TezTaskID tId = TezTaskID.getInstance(vId, 389);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(tId, 2);

    String dagIdStr = dagId.toString();
    String vIdStr = vId.toString();
    String tIdStr = tId.toString();
    String taIdStr = taId.toString();

    LOG.info("DAG ID:" + dagIdStr);
    LOG.info("Vertex ID:" + vIdStr);
    LOG.info("Task ID:" + tIdStr);
    LOG.info("Attempt ID:" + taIdStr);

    Assert.assertTrue(dagIdStr.startsWith("dag"));
    Assert.assertTrue(vIdStr.startsWith("vertex"));
    Assert.assertTrue(tIdStr.startsWith("task"));
    Assert.assertTrue(taIdStr.startsWith("attempt"));

    verifyDagId(dagIdStr, dagId);/*  w  ww .j a v a2 s.  c  o  m*/
    verifyVertexId(vIdStr, vId);
    verifyTaskId(tIdStr, tId);
    verifyAttemptId(taIdStr, taId);
}

---

From source file:org.apache.tez.dag.records.TezDAGID.java

License:Apache License

private TezDAGID(String yarnRMIdentifier, int appId, int id) {
    this(ApplicationId.newInstance(Long.valueOf(yarnRMIdentifier), appId), id);
}

---

From source file:org.apache.tez.dag.records.TezDAGID.java

License:Apache License

@Override
// Can't do much about this instance if used via the RPC layer. Any downstream
// users can however avoid using this method.
public void readFields(DataInput in) throws IOException {
    // ApplicationId could be cached in this case. All of this will change for Protobuf RPC.
    applicationId = ApplicationId.newInstance(in.readLong(), in.readInt());
    super.readFields(in);
}

---

From source file:org.apache.tez.dag.records.TezDAGID.java

License:Apache License

public static TezDAGID readTezDAGID(DataInput in) throws IOException {
    long clusterId = in.readLong();
    int appId = in.readInt();
    int dagIdInt = TezID.readID(in);
    TezDAGID dagID = getInstance(ApplicationId.newInstance(clusterId, appId), dagIdInt);
    return dagID;
}

---

From source file:org.apache.tez.history.TestHistoryParser.java

License:Apache License

private String runWordCount(String tokenizerProcessor, String summationProcessor, String dagName,
        boolean withTimeline) throws Exception {
    //HDFS path/*  w  ww  .j ava 2 s .  com*/
    Path outputLoc = new Path("/tmp/outPath_" + System.currentTimeMillis());

    DataSourceDescriptor dataSource = MRInput
            .createConfigBuilder(conf, TextInputFormat.class, inputLoc.toString()).build();

    DataSinkDescriptor dataSink = MROutput
            .createConfigBuilder(conf, TextOutputFormat.class, outputLoc.toString()).build();

    Vertex tokenizerVertex = Vertex.create(TOKENIZER, ProcessorDescriptor.create(tokenizerProcessor))
            .addDataSource(INPUT, dataSource);

    OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
            .newBuilder(Text.class.getName(), IntWritable.class.getName(), HashPartitioner.class.getName())
            .build();

    Vertex summationVertex = Vertex.create(SUMMATION, ProcessorDescriptor.create(summationProcessor), 1)
            .addDataSink(OUTPUT, dataSink);

    // Create DAG and add the vertices. Connect the producer and consumer vertices via the edge
    DAG dag = DAG.create(dagName);
    dag.addVertex(tokenizerVertex).addVertex(summationVertex)
            .addEdge(Edge.create(tokenizerVertex, summationVertex, edgeConf.createDefaultEdgeProperty()));

    TezClient tezClient = getTezClient(withTimeline);

    // Update Caller Context
    CallerContext callerContext = CallerContext.create("TezExamples", "Tez WordCount Example Job");
    ApplicationId appId = tezClient.getAppMasterApplicationId();
    if (appId == null) {
        appId = ApplicationId.newInstance(1001l, 1);
    }
    callerContext.setCallerIdAndType(appId.toString(), "TezApplication");
    dag.setCallerContext(callerContext);

    DAGClient client = tezClient.submitDAG(dag);
    client.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezDAGID tezDAGID = TezDAGID.getInstance(tezClient.getAppMasterApplicationId(), 1);

    if (tezClient != null) {
        tezClient.stop();
    }
    return tezDAGID.toString();
}

---

From source file:org.apache.tez.history.Utils.java

License:Apache License

public static ApplicationId getAppIdFromDAGId(String dagId) {
    int appId = 0;
    String[] split = dagId.split("_");
    if (split.length != 4 || !dagId.startsWith(DAG + "_")) {
        throw new IllegalArgumentException("Invalid DAG Id format : " + dagId);
    }/*from   w w  w .  ja  va2  s . co  m*/
    String rmId = split[1];
    try {
        appId = Integer.parseInt(split[2]);
    } catch (NumberFormatException e) {
        throw new IllegalArgumentException(
                "Error while parsing App Id '" + split[2] + "' of DAG Id : " + dagId);
    }
    try {
        int id = Integer.parseInt(split[3]);
    } catch (NumberFormatException e) {
        throw new IllegalArgumentException("Error while parsing Id '" + split[3] + "' of DAG Id : " + dagId);
    }
    return ApplicationId.newInstance(Long.parseLong(rmId), appId);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• »