Example usage for org.apache.hadoop.yarn.api.records Resource newInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Resource newInstance.

Prototype

@Public
    @Stable
    public static Resource newInstance(long memory, int vCores) 

Source Link

Usage

From source file:org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerService.java

License:Apache License

@Override
public Resource getAvailableResources() {
    // TODO This needs information about all running executors, and the amount of memory etc available across the cluster.
    return Resource.newInstance(Ints.checkedCast(serviceHosts.size() * memoryPerInstance),
            serviceHosts.size() * coresPerInstance);
}

---

From source file:org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerService.java

License:Apache License

@Override
public Resource getTotalResources() {
    return Resource.newInstance(Ints.checkedCast(serviceHosts.size() * memoryPerInstance),
            serviceHosts.size() * coresPerInstance);
}

---

From source file:org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerServiceWithErrors.java

License:Apache License

@Override
public Resource getAvailableResources() {
    return Resource.newInstance(2048, 2);
}

---

From source file:org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerServiceWithErrors.java

License:Apache License

@Override
public Resource getTotalResources() {
    return Resource.newInstance(2048, 2);
}

---

From source file:org.apache.tez.dag.history.ats.acls.TestATSHistoryV15.java

License:Apache License

@Test(timeout = 50000)
public void testSimpleDAG() throws Exception {
    TezClient tezSession = null;//from   w ww  .ja va 2 s  .  c o m
    ApplicationId applicationId;
    String viewAcls = "nobody nobody_group";
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);

        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName())
                .setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);

        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());

        tezConf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES, "TEZ_DAG_ID");

        tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
                ATSV15HistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs
                .makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();

        applicationId = tezSession.getAppMasterApplicationId();

        DAGClient dagClient = tezSession.submitDAG(dag);

        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
                    + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());

        // Verify HDFS data
        int count = verifyATSDataOnHDFS(atsActivePath, 0, applicationId);
        Assert.assertTrue("Count is: " + count, count > 0);

    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }

}

---

From source file:org.apache.tez.dag.history.ats.acls.TestATSHistoryWithACLs.java

License:Apache License

@Test(timeout = 50000)
public void testSimpleAMACls() throws Exception {
    TezClient tezSession = null;/*from  ww  w .  j a  v a 2s  .c  om*/
    ApplicationId applicationId;
    String viewAcls = "nobody nobody_group";
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);

        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName())
                .setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);

        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
                ATSHistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs
                .makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();

        applicationId = tezSession.getAppMasterApplicationId();

        DAGClient dagClient = tezSession.submitDAG(dag);

        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
                    + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }

    TimelineDomain timelineDomain = getDomain(
            ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString());
    verifyDomainACLs(timelineDomain, Collections.singleton("nobody"), Collections.singleton("nobody_group"));

    verifyEntityDomains(applicationId, true);
}

---

From source file:org.apache.tez.dag.history.ats.acls.TestATSHistoryWithACLs.java

License:Apache License

@Test(timeout = 50000)
public void testDAGACls() throws Exception {
    TezClient tezSession = null;/*  ww  w .  j  a va 2s  .  co m*/
    ApplicationId applicationId;
    String viewAcls = "nobody nobody_group";
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);

        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName())
                .setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);
        DAGAccessControls accessControls = new DAGAccessControls();
        accessControls.setUsersWithViewACLs(Collections.singleton("nobody2"));
        accessControls.setGroupsWithViewACLs(Collections.singleton("nobody_group2"));
        dag.setAccessControls(accessControls);

        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
                ATSHistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs
                .makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();

        applicationId = tezSession.getAppMasterApplicationId();

        DAGClient dagClient = tezSession.submitDAG(dag);

        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
                    + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }

    TimelineDomain timelineDomain = getDomain(
            ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString());
    verifyDomainACLs(timelineDomain, Collections.singleton("nobody"), Collections.singleton("nobody_group"));

    timelineDomain = getDomain(
            ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString() + "_TezSleepProcessor");
    verifyDomainACLs(timelineDomain, Sets.newHashSet("nobody", "nobody2"),
            Sets.newHashSet("nobody_group", "nobody_group2"));

    verifyEntityDomains(applicationId, false);
}

---

From source file:org.apache.tez.dag.history.logging.ats.TestATSHistoryWithMiniCluster.java

License:Apache License

@Test(timeout = 50000)
public void testSimpleAMACls() throws Exception {
    TezClient tezSession = null;//from  w ww  .  j a v  a  2s.  co  m
    ApplicationId applicationId;
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);

        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName())
                .setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);

        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
                ATSHistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs
                .makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();

        applicationId = tezSession.getAppMasterApplicationId();

        DAGClient dagClient = tezSession.submitDAG(dag);

        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
                    + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }

    //    verifyEntityExistence(applicationId);
}

---

From source file:org.apache.tez.dag.history.logging.ats.TestATSHistoryWithMiniCluster.java

License:Apache License

@Test(timeout = 50000)
public void testDAGACls() throws Exception {
    TezClient tezSession = null;//from   www  .  j av  a2s.  c  om
    ApplicationId applicationId;
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);

        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName())
                .setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);

        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
                ATSHistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs
                .makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();

        applicationId = tezSession.getAppMasterApplicationId();

        DAGClient dagClient = tezSession.submitDAG(dag);

        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
                    + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }
    //    verifyEntityExistence(applicationId);
}

---

From source file:org.apache.tez.dag.history.utils.TestDAGUtils.java

License:Apache License

private DAGPlan createDAG() {
    // Create a plan with 3 vertices: A, B, C. Group(A,B)->C
    Configuration conf = new Configuration(false);
    int dummyTaskCount = 1;
    Resource dummyTaskResource = Resource.newInstance(1, 1);
    org.apache.tez.dag.api.Vertex v1 = Vertex.create("vertex1",
            ProcessorDescriptor.create("Processor").setHistoryText("vertex1 Processor HistoryText"),
            dummyTaskCount, dummyTaskResource);
    v1.addDataSource("input1", DataSourceDescriptor
            .create(InputDescriptor.create("input.class").setHistoryText("input HistoryText"), null, null));
    org.apache.tez.dag.api.Vertex v2 = Vertex.create("vertex2",
            ProcessorDescriptor.create("Processor").setHistoryText("vertex2 Processor HistoryText"),
            dummyTaskCount, dummyTaskResource);
    org.apache.tez.dag.api.Vertex v3 = Vertex.create("vertex3",
            ProcessorDescriptor.create("Processor").setHistoryText("vertex3 Processor HistoryText"),
            dummyTaskCount, dummyTaskResource);

    DAG dag = DAG.create("testDag");
    dag.setDAGInfo("dagInfo");
    String groupName1 = "uv12";
    org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
    OutputDescriptor outDesc = OutputDescriptor.create("output.class").setHistoryText("uvOut HistoryText");
    OutputCommitterDescriptor ocd = OutputCommitterDescriptor.create(OutputCommitter.class.getName());
    uv12.addDataSink("uvOut", DataSinkDescriptor.create(outDesc, ocd, null));
    v3.addDataSink("uvOut", DataSinkDescriptor.create(outDesc, ocd, null));

    GroupInputEdge e1 = GroupInputEdge.create(uv12, v3,
            EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
                    SchedulingType.SEQUENTIAL,
                    OutputDescriptor.create("dummy output class").setHistoryText("Dummy History Text"),
                    InputDescriptor.create("dummy input class").setHistoryText("Dummy History Text")),
            InputDescriptor.create("merge.class").setHistoryText("Merge HistoryText"));

    dag.addVertex(v1);//from  w  ww  .  j a va  2  s  . c om
    dag.addVertex(v2);
    dag.addVertex(v3);
    dag.addEdge(e1);
    return dag.createDag(conf, null, null, null, true);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• »