Example usage for org.apache.hadoop.yarn.conf YarnConfiguration TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES.

Prototype

String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES.

Click Source Link

Usage

From source file:org.apache.tez.dag.history.ats.acls.TestATSHistoryV15.java

License:Apache License

@Test(timeout = 50000)
public void testSimpleDAG() throws Exception {
    TezClient tezSession = null;//from www  .  jav a2s  .  co m
    ApplicationId applicationId;
    String viewAcls = "nobody nobody_group";
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);

        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName())
                .setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);

        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());

        tezConf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES, "TEZ_DAG_ID");

        tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
                ATSV15HistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs
                .makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());

        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();

        applicationId = tezSession.getAppMasterApplicationId();

        DAGClient dagClient = tezSession.submitDAG(dag);

        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: "
                    + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());

        // Verify HDFS data
        int count = verifyATSDataOnHDFS(atsActivePath, 0, applicationId);
        Assert.assertTrue("Count is: " + count, count > 0);

    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }

}

---

From source file:org.apache.tez.dag.history.logging.ats.ATSV15HistoryLoggingService.java

License:Apache License

@Override
public void serviceInit(Configuration serviceConf) throws Exception {
    Configuration conf = new Configuration(serviceConf);

    String summaryEntityTypesStr = EntityTypes.TEZ_APPLICATION + "," + EntityTypes.TEZ_APPLICATION_ATTEMPT + ","
            + EntityTypes.TEZ_DAG_ID;//from  www  .ja v  a2 s .c  o m

    // Ensure that summary entity types are defined properly for Tez.
    if (conf.getBoolean(TezConfiguration.TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES,
            TezConfiguration.TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES_DEFAULT)) {
        conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES,
                summaryEntityTypesStr);
    }

    historyLoggingEnabled = conf.getBoolean(TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED,
            TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT);
    if (!historyLoggingEnabled) {
        LOG.info("ATSService: History Logging disabled. " + TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED
                + " set to false");
        return;
    }

    if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
        timelineClient = TimelineClient.createTimelineClient();
        timelineClient.init(conf);
    } else {
        this.timelineClient = null;
        if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
                .equals(atsHistoryLoggingServiceClassName)) {
            LOG.warn(atsHistoryLoggingServiceClassName + " is disabled due to Timeline Service being disabled, "
                    + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
        }
    }
    maxTimeToWaitOnShutdown = conf.getLong(TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
            TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT);
    maxPollingTimeMillis = conf.getInt(TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT,
            TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT);
    if (maxTimeToWaitOnShutdown < 0) {
        waitForeverOnShutdown = true;
    }
    sessionDomainId = conf.get(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID);

    LOG.info("Initializing " + ATSV15HistoryLoggingService.class.getSimpleName() + " with "
            + ", maxPollingTime(ms)=" + maxPollingTimeMillis + ", waitTimeForShutdown(ms)="
            + maxTimeToWaitOnShutdown + ", TimelineACLManagerClass=" + atsHistoryACLManagerClassName);

    try {
        historyACLPolicyManager = ReflectionUtils.createClazzInstance(atsHistoryACLManagerClassName);
        historyACLPolicyManager.setConf(conf);
    } catch (TezReflectionException e) {
        LOG.warn("Could not instantiate object for " + atsHistoryACLManagerClassName
                + ". ACLs cannot be enforced correctly for history data in Timeline", e);
        if (!conf.getBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS,
                TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT)) {
            throw e;
        }
        historyACLPolicyManager = null;
    }

}

---