List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration TIMELINE_SERVICE_ENABLED
String TIMELINE_SERVICE_ENABLED
To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration TIMELINE_SERVICE_ENABLED.
Click Source Link
From source file:org.apache.metron.integration.components.YarnComponent.java
License:Apache License
@Override public void start() throws UnableToStartException { conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.set("yarn.log.dir", "target"); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName()); conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); try {/*from ww w . j a va2 s . co m*/ yarnCluster = new MiniYARNCluster(testName, 1, NUM_NMS, 1, 1, true); yarnCluster.init(conf); yarnCluster.start(); waitForNMsToRegister(); URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"); } Configuration yarnClusterConfig = yarnCluster.getConfig(); yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.writeXml(bytesOut); bytesOut.close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new File(url.getPath())); os.write(bytesOut.toByteArray()); os.close(); FileContext fsContext = FileContext.getLocalFSFileContext(); fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true); try { Thread.sleep(2000); } catch (InterruptedException e) { } } catch (Exception e) { throw new UnableToStartException("Exception setting up yarn cluster", e); } }
From source file:org.apache.metron.maas.service.Client.java
License:Apache License
private void prepareTimelineDomain() { TimelineClient timelineClient = null; if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf);/*from ww w. jav a2 s .c o m*/ timelineClient.start(); } else { LOG.warn("Cannot put the domain " + domainId + " because the timeline service is not enabled"); return; } try { TimelineDomain domain = new TimelineDomain(); domain.setId(domainId); domain.setReaders(viewACLs != null && viewACLs.length() > 0 ? viewACLs : " "); domain.setWriters(modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " "); timelineClient.putDomain(domain); LOG.info("Put the timeline domain: " + TimelineUtils.dumpTimelineRecordtoJSON(domain)); } catch (Exception e) { LOG.error("Error when putting the timeline domain", e); } finally { timelineClient.stop(); } }
From source file:org.apache.slider.core.launch.CredentialUtils.java
License:Apache License
public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken(Configuration conf, Credentials credentials) throws IOException, YarnException { if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) { LOG.debug("Timeline service enabled -fetching token"); try (TimelineClient timelineClient = TimelineClient.createTimelineClient()) { timelineClient.init(conf);/*w w w . jav a 2s . c om*/ timelineClient.start(); Token<TimelineDelegationTokenIdentifier> token = timelineClient .getDelegationToken(CredentialUtils.getRMPrincipal(conf)); credentials.addToken(token.getService(), token); return token; } } else { LOG.debug("Timeline service is disabled"); return null; } }
From source file:org.apache.tez.analyzer.TestAnalyzer.java
License:Apache License
private static void setupTezCluster() throws Exception { // make the test run faster by speeding heartbeat frequency conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true); conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName()); miniTezCluster = new MiniTezClusterWithTimeline(TestAnalyzer.class.getName(), 1, 1, 1, true); miniTezCluster.init(conf);/* w w w . ja v a 2s . co m*/ miniTezCluster.start(); yarnTimelineAddress = miniTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS); }
From source file:org.apache.tez.analyzer.TestAnalyzer.java
License:Apache License
private void createTezSessionATS() throws Exception { TezConfiguration tezConf = createCommonTezLog(); tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); tezConf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, miniTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS)); tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true); tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName()); Path remoteStagingDir = dfsCluster.getFileSystem() .makeQualified(new Path(TEST_ROOT_DIR, String.valueOf(new Random().nextInt(100000)))); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false); tezSession = TezClient.create("TestAnalyzer", tezConf, true); tezSession.start();//from w w w .ja v a 2 s.c o m }
From source file:org.apache.tez.dag.history.ats.acls.ATSV15HistoryACLPolicyManager.java
License:Apache License
private void initializeTimelineClient() { if (this.conf == null) { throw new TezUncheckedException("ATSACLManager not configured"); }/*from w w w .j ava 2 s . c om*/ if (timelineClient != null) { this.timelineClient.stop(); this.timelineClient = null; } if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { this.timelineClient = TimelineClient.createTimelineClient(); this.timelineClient.init(this.conf); this.timelineClient.start(); } else { this.timelineClient = null; if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") .equals(atsHistoryLoggingServiceClassName)) { LOG.warn(atsHistoryLoggingServiceClassName + " is disabled due to Timeline Service being disabled, " + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false"); } } try { this.user = UserGroupInformation.getCurrentUser().getShortUserName(); } catch (IOException e) { throw new TezUncheckedException("Unable to get Current User UGI", e); } }
From source file:org.apache.tez.dag.history.ats.acls.TestATSHistoryV15.java
License:Apache License
@BeforeClass public static void setup() throws IOException { try {/*from w w w . ja v a2 s. co m*/ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR); dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).racks(null).build(); remoteFs = dfsCluster.getFileSystem(); } catch (IOException io) { throw new RuntimeException("problem starting mini dfs cluster", io); } if (mrrTezCluster == null) { try { mrrTezCluster = new MiniTezClusterWithTimeline(TestATSHistoryV15.class.getName(), 1, 1, 1, true); Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS conf.setInt("yarn.nodemanager.delete.debug-delay-sec", 20000); atsActivePath = new Path("/tmp/ats/active/" + random.nextInt(100000)); Path atsDonePath = new Path("/tmp/ats/done/" + random.nextInt(100000)); conf.setDouble(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5); remoteFs.mkdirs(atsActivePath); remoteFs.mkdirs(atsDonePath); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR, remoteFs.resolvePath(atsActivePath).toString()); conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR, remoteFs.resolvePath(atsDonePath).toString()); mrrTezCluster.init(conf); mrrTezCluster.start(); } catch (Throwable e) { LOG.info("Failed to start Mini Tez Cluster", e); } } user = UserGroupInformation.getCurrentUser().getShortUserName(); timelineAddress = mrrTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS); if (timelineAddress != null) { // Hack to handle bug in MiniYARNCluster handling of webapp address timelineAddress = timelineAddress.replace("0.0.0.0", "localhost"); } }
From source file:org.apache.tez.dag.history.ats.acls.TestATSHistoryWithACLs.java
License:Apache License
@BeforeClass public static void setup() throws IOException { try {// w w w .ja v a 2 s . com conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR); dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).racks(null).build(); remoteFs = dfsCluster.getFileSystem(); } catch (IOException io) { throw new RuntimeException("problem starting mini dfs cluster", io); } if (mrrTezCluster == null) { try { mrrTezCluster = new MiniTezClusterWithTimeline(TestATSHistoryWithACLs.class.getName(), 1, 1, 1, true); Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS conf.setInt("yarn.nodemanager.delete.debug-delay-sec", 20000); mrrTezCluster.init(conf); mrrTezCluster.start(); } catch (Throwable e) { LOG.info("Failed to start Mini Tez Cluster", e); } } user = UserGroupInformation.getCurrentUser().getShortUserName(); timelineAddress = mrrTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS); if (timelineAddress != null) { // Hack to handle bug in MiniYARNCluster handling of webapp address timelineAddress = timelineAddress.replace("0.0.0.0", "localhost"); } }
From source file:org.apache.tez.dag.history.logging.ats.ATSV15HistoryLoggingService.java
License:Apache License
@Override public void serviceInit(Configuration serviceConf) throws Exception { Configuration conf = new Configuration(serviceConf); String summaryEntityTypesStr = EntityTypes.TEZ_APPLICATION + "," + EntityTypes.TEZ_APPLICATION_ATTEMPT + "," + EntityTypes.TEZ_DAG_ID;/*from w ww . ja va2s . c o m*/ // Ensure that summary entity types are defined properly for Tez. if (conf.getBoolean(TezConfiguration.TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES, TezConfiguration.TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES_DEFAULT)) { conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES, summaryEntityTypesStr); } historyLoggingEnabled = conf.getBoolean(TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED, TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT); if (!historyLoggingEnabled) { LOG.info("ATSService: History Logging disabled. " + TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED + " set to false"); return; } if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); } else { this.timelineClient = null; if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") .equals(atsHistoryLoggingServiceClassName)) { LOG.warn(atsHistoryLoggingServiceClassName + " is disabled due to Timeline Service being disabled, " + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false"); } } maxTimeToWaitOnShutdown = conf.getLong(TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS, TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT); maxPollingTimeMillis = conf.getInt(TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT, TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT); if (maxTimeToWaitOnShutdown < 0) { waitForeverOnShutdown = true; } sessionDomainId = conf.get(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID); LOG.info("Initializing " + ATSV15HistoryLoggingService.class.getSimpleName() + " with " + ", maxPollingTime(ms)=" + maxPollingTimeMillis + ", waitTimeForShutdown(ms)=" + maxTimeToWaitOnShutdown + ", TimelineACLManagerClass=" + atsHistoryACLManagerClassName); try { historyACLPolicyManager = ReflectionUtils.createClazzInstance(atsHistoryACLManagerClassName); historyACLPolicyManager.setConf(conf); } catch (TezReflectionException e) { LOG.warn("Could not instantiate object for " + atsHistoryACLManagerClassName + ". ACLs cannot be enforced correctly for history data in Timeline", e); if (!conf.getBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT)) { throw e; } historyACLPolicyManager = null; } }
From source file:org.apache.tez.dag.history.logging.ats.TestATSHistoryWithMiniCluster.java
License:Apache License
@BeforeClass public static void setup() throws IOException { try {/*www . j a v a 2 s .c o m*/ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR); dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).racks(null).build(); remoteFs = dfsCluster.getFileSystem(); } catch (IOException io) { throw new RuntimeException("problem starting mini dfs cluster", io); } if (mrrTezCluster == null) { try { mrrTezCluster = new MiniTezClusterWithTimeline(TestATSHistoryWithMiniCluster.class.getName(), 1, 1, 1, true); Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS conf.setInt("yarn.nodemanager.delete.debug-delay-sec", 20000); mrrTezCluster.init(conf); mrrTezCluster.start(); } catch (Throwable e) { LOG.info("Failed to start Mini Tez Cluster", e); } } timelineAddress = mrrTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS); if (timelineAddress != null) { // Hack to handle bug in MiniYARNCluster handling of webapp address timelineAddress = timelineAddress.replace("0.0.0.0", "localhost"); } }