List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration TIMELINE_SERVICE_ENABLED
String TIMELINE_SERVICE_ENABLED
To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration TIMELINE_SERVICE_ENABLED.
Click Source Link
From source file:cascading.flow.tez.planner.Hadoop2TezFlowStepJob.java
License:Open Source License
protected void internalNonBlockingStart() throws IOException { try {/* ww w. j av a 2s .c o m*/ if (!isTimelineServiceEnabled(jobConfiguration)) flowStep.logWarn("'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon"); TezConfiguration workingConf = new TezConfiguration(jobConfiguration); // this could be problematic flowStep.logInfo("tez session mode enabled: " + workingConf.getBoolean( TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT)); prepareEnsureStagingDir(workingConf); tezClient = TezClient.create(flowStep.getName(), workingConf, ((Hadoop2TezFlowStep) flowStep).getAllLocalResources(), null); tezClient.start(); dagClient = tezClient.submitDAG(dag); dagId = Util.returnInstanceFieldIfExistsSafe(dagClient, "dagId"); flowStep.logInfo("submitted tez dag to app master: {}, with dag id: {}", tezClient.getAppMasterApplicationId(), dagId); } catch (TezException exception) { throw new CascadingException(exception); } }
From source file:cascading.flow.tez.planner.Hadoop2TezFlowStepJob.java
License:Open Source License
private boolean isTimelineServiceEnabled(TezConfiguration workingConf) { return workingConf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED); }
From source file:cascading.platform.tez.Hadoop2TezPlatform.java
License:Open Source License
@Override public synchronized void setUp() throws IOException { if (configuration != null) return;//from w w w .jav a 2 s . co m if (!isUseCluster()) { // Current usage requirements: // 1. Clients need to set "tez.local.mode" to true when creating a TezClient instance. (For the examples this can be done via -Dtez.local.mode=true) // 2. fs.defaultFS must be set to "file:///" // 2.1 If running examples - this must be set in tez-site.xml (so that it's picked up by the client, as well as the conf instances used to configure the Inputs / Outputs). // 2.2 If using programatically (without a tez-site.xml present). All configuration instances used (to crate the client / configure Inputs / Outputs) - must have this property set. // 3. tez.runtime.optimize.local.fetch needs to be set to true (either via tez-site.xml or in all configurations used to create the job (similar to fs.defaultFS in step 2)) // 4. tez.staging-dir must be set (either programatically or via tez-site.xml). // Until TEZ-1337 goes in - the staging-dir for the job is effectively the root of the filesystem (and where inputs are read from / written to if relative paths are used). LOG.info("not using cluster"); configuration = new Configuration(); configuration.setInt(FlowRuntimeProps.GATHER_PARTITIONS, getNumGatherPartitions()); // configuration.setInt( FlowRuntimeProps.GATHER_PARTITIONS, 1 ); // deadlocks if larger than 1 configuration.set(TezConfiguration.TEZ_LOCAL_MODE, "true"); configuration.set("fs.defaultFS", "file:///"); configuration.set("tez.runtime.optimize.local.fetch", "true"); // hack to prevent deadlocks where downstream processors are scheduled before upstream configuration.setInt("tez.am.inline.task.execution.max-tasks", 3); // testHashJoinMergeIntoHashJoinAccumulatedAccumulatedMerge fails if set to 2 configuration.set(TezConfiguration.TEZ_IGNORE_LIB_URIS, "true"); // in local mode, use local classpath configuration.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, -1); configuration.set(TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS, "true"); configuration.set("tez.am.mode.session", "true"); // allows multiple TezClient instances to be used in a single jvm if (!Util.isEmpty(System.getProperty("hadoop.tmp.dir"))) configuration.set("hadoop.tmp.dir", System.getProperty("hadoop.tmp.dir")); else configuration.set("hadoop.tmp.dir", "build/test/tmp"); fileSys = FileSystem.get(configuration); } else { LOG.info("using cluster"); if (Util.isEmpty(System.getProperty("hadoop.log.dir"))) System.setProperty("hadoop.log.dir", "build/test/log"); if (Util.isEmpty(System.getProperty("hadoop.tmp.dir"))) System.setProperty("hadoop.tmp.dir", "build/test/tmp"); new File(System.getProperty("hadoop.log.dir")).mkdirs(); // ignored new File(System.getProperty("hadoop.tmp.dir")).mkdirs(); // ignored Configuration defaultConf = new Configuration(); defaultConf.setInt(FlowRuntimeProps.GATHER_PARTITIONS, getNumGatherPartitions()); defaultConf.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, -1); // defaultConf.set( TezConfiguration.TEZ_AM_LOG_LEVEL, "DEBUG" ); // defaultConf.set( TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG" ); defaultConf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); defaultConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false); defaultConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, System.getProperty("hadoop.tmp.dir")); miniDFSCluster = new MiniDFSCluster.Builder(defaultConf).numDataNodes(4).format(true).racks(null) .build(); fileSys = miniDFSCluster.getFileSystem(); Configuration tezConf = new Configuration(defaultConf); tezConf.set("fs.defaultFS", fileSys.getUri().toString()); // use HDFS tezConf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); // see MiniTezClusterWithTimeline as alternate miniTezCluster = new MiniTezCluster(getClass().getName(), 4, 1, 1); // todo: set to 4 miniTezCluster.init(tezConf); miniTezCluster.start(); configuration = miniTezCluster.getConfig(); // stats won't work after completion unless ATS is used if (setTimelineStore(configuration)) // true if ats can be loaded and configured for this hadoop version { configuration.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName()); configuration.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); configuration.set(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, "localhost:10200"); configuration.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:8188"); configuration.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, "localhost:8190"); yarnHistoryServer = new ApplicationHistoryServer(); yarnHistoryServer.init(configuration); yarnHistoryServer.start(); } } configuration.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 1); configuration.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1); configuration.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 1); Map<Object, Object> globalProperties = getGlobalProperties(); if (logger != null) globalProperties.put("log4j.logger", logger); FlowProps.setJobPollingInterval(globalProperties, 10); // should speed up tests Hadoop2TezPlanner.copyProperties(configuration, globalProperties); // copy any external properties Hadoop2TezPlanner.copyConfiguration(properties, configuration); // put all properties on the jobconf ExitUtil.disableSystemExit(); // forbidSystemExitCall(); }
From source file:cascading.stats.tez.TezNodeStats.java
License:Open Source License
private boolean withoutTimelineServer(DAGClient dagClient) { VertexStatus vertexStatus = updateProgress(dagClient, STATUS_GET_COUNTERS); if (vertexStatus == null) return false; int total = sliceStatsMap.size(); if (total == 0) // yet to be initialized logWarn("'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, task level counters cannot be retrieved"); for (int i = total; i < totalTaskCount; i++) { TezSliceStats sliceStats = new TezSliceStats(Util.createUniqueID(), kind, this.getStatus(), null); // we don't have the taskId, so we are using the id as the key sliceStatsMap.put(sliceStats.getID(), sliceStats); }// w w w. j a v a 2s .c o m // a placeholder to simulate actual slice stats for now Iterator<FlowSliceStats> iterator = sliceStatsMap.values().iterator(); for (int i = 0; i < runningTaskCount && iterator.hasNext(); i++) ((TezSliceStats) iterator.next()).setStatus(Status.RUNNING); for (int i = 0; i < succeededTaskCount && iterator.hasNext(); i++) ((TezSliceStats) iterator.next()).setStatus(Status.SUCCESSFUL); for (int i = 0; i < failedTaskCount && iterator.hasNext(); i++) ((TezSliceStats) iterator.next()).setStatus(Status.FAILED); for (int i = 0; i < killedTaskCount && iterator.hasNext(); i++) ((TezSliceStats) iterator.next()).setStatus(Status.STOPPED); List<String> diagnostics = vertexStatus.getDiagnostics(); for (String diagnostic : diagnostics) logInfo("vertex diagnostics: {}", diagnostic); return true; }
From source file:cascading.stats.tez.util.TezStatsUtil.java
License:Open Source License
private static boolean loadClass() { if (timelineClientClass != null) return true; try {//from w ww . j a v a2 s. c o m timelineClientClass = (Class<DAGClient>) Thread.currentThread().getContextClassLoader() .loadClass(TIMELINE_CLIENT_CLASS); return true; } catch (ClassNotFoundException exception) { LOG.error("'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is enabled, yet unable to load Tez YARN timeline client class: {}, ensure these dependencies are in your local CLASSPATH: tez-yarn-timeline-history, org.apache.tez:tez-yarn-timeline-history or org.apache.tez:tez-yarn-timeline-history-with-acls", TIMELINE_CLIENT_CLASS, exception); } return false; }
From source file:com.inforefiner.hdata.ApplicationMaster.java
License:Apache License
@VisibleForTesting void startTimelineClient(final Configuration conf) throws YarnException, IOException, InterruptedException { try {//w ww .j ava2s.com appSubmitterUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { // Creating the Timeline Client timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); } else { timelineClient = null; LOG.warn("Timeline service is not enabled"); } return null; } }); } catch (UndeclaredThrowableException e) { throw new YarnException(e.getCause()); } }
From source file:com.inforefiner.hdata.SubmitClient.java
License:Apache License
private void prepareTimelineDomain() { TimelineClient timelineClient = null; if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf);/*from w w w . jav a 2 s . co m*/ timelineClient.start(); } else { LOG.warn("Cannot put the domain " + domainId + " because the timeline service is not enabled"); return; } try { //TODO: we need to check and combine the existing timeline domain ACLs, //but let's do it once we have client java library to query domains. TimelineDomain domain = new TimelineDomain(); domain.setId(domainId); domain.setReaders(viewACLs != null && viewACLs.length() > 0 ? viewACLs : " "); domain.setWriters(modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " "); timelineClient.putDomain(domain); LOG.info("Put the timeline domain: " + TimelineUtils.dumpTimelineRecordtoJSON(domain)); } catch (Exception e) { LOG.error("Error when putting the timeline domain", e); } finally { timelineClient.stop(); } }
From source file:io.hops.tensorflow.Client.java
License:Apache License
private void prepareTimelineDomain() { TimelineClient timelineClient;//from w w w. j a v a 2 s . c om if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); } else { LOG.warn("Cannot put the domain " + domainId + " because the timeline service is not enabled"); return; } try { TimelineDomain domain = new TimelineDomain(); domain.setId(domainId); domain.setReaders(viewACLs != null && viewACLs.length() > 0 ? viewACLs : " "); domain.setWriters(modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " "); timelineClient.putDomain(domain); LOG.info("Put the timeline domain: " + TimelineUtils.dumpTimelineRecordtoJSON(domain)); } catch (Exception e) { LOG.error("Error when putting the timeline domain", e); } finally { timelineClient.stop(); } }
From source file:io.hops.tensorflow.TestCluster.java
License:Apache License
protected void setupInternal(int numNodeManager) throws Exception { LOG.info("Starting up YARN cluster"); conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.set("yarn.log.dir", "target"); conf.set("yarn.log-aggregation-enable", "true"); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName()); conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); conf.setBoolean(YarnConfiguration.NM_GPU_RESOURCE_ENABLED, false); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(TestCluster.class.getSimpleName(), 1, numNodeManager, 1, 1); yarnCluster.init(conf);// ww w . j ava 2 s. co m yarnCluster.start(); conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":" + yarnCluster.getApplicationHistoryServer().getPort()); waitForNMsToRegister(); URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"); } Configuration yarnClusterConfig = yarnCluster.getConfig(); yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.writeXml(bytesOut); bytesOut.close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new File(url.getPath())); os.write(bytesOut.toByteArray()); os.close(); } FileContext fsContext = FileContext.getLocalFSFileContext(); fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true); try { Thread.sleep(2000); } catch (InterruptedException e) { LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); } }
From source file:io.hops.tensorflow.TimelineHandler.java
License:Apache License
public void startClient(final Configuration conf) throws YarnException, IOException, InterruptedException { try {/*w w w. j a v a 2 s . co m*/ ugi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { // Creating the Timeline Client timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); } else { timelineClient = null; LOG.warn("Timeline service is not enabled"); } return null; } }); } catch (UndeclaredThrowableException e) { throw new YarnException(e.getCause()); } }