List of usage examples for org.apache.hadoop.mapred ClusterStatus getJobTrackerState
@Deprecated
public JobTracker.State getJobTrackerState()
JobTracker, as JobTracker.State JobTracker.State should no longer be used on M/R 2.x. From source file:com.impetus.ankush2.hadoop.monitor.JobStatusProvider.java
License:Open Source License
/** * Gets the job metrics./*w w w .j av a 2 s.com*/ * * @return List */ public Map<String, Object> getJobMetrics() throws AnkushException { String errMsg = "Unable to getch Hadoop Metrics, could not connect to Hadoop JobClient."; try { // Checking for !null if (jobClient != null) { // Creating an empty map for storing Hadoop Job Metrics information LinkedHashMap<String, Object> hadoopJobMetrics = new LinkedHashMap<String, Object>(); try { // Checking for null jobClient if (jobClient != null) { LOG.info("Fetching Hadoop Metrics Information.." + jobClient); // Get status information about the Map-Reduce cluster. ClusterStatus clusterStatus = jobClient.getClusterStatus(); // Get the current state of the JobTracker, State jobTrackerState = clusterStatus.getJobTrackerState(); // Get the number of currently running map tasks in the cluster. int mapTasks = clusterStatus.getMapTasks(); // Get the maximum capacity for running map tasks in the // cluster. int maxMapTasks = clusterStatus.getMaxMapTasks(); // Get the maximum capacity for running reduce tasks in the // cluster. int maxReduceTasks = clusterStatus.getMaxReduceTasks(); // Get the number of currently running reduce tasks in the // cluster. int reduceTasks = clusterStatus.getReduceTasks(); // Get the number of active task trackers in the cluster. int taskTrackers = clusterStatus.getTaskTrackers(); // Get the number of blacklisted task trackers in the cluster. int blackListedTrackers = clusterStatus.getBlacklistedTrackers(); long ttExpiryInterval = clusterStatus.getTTExpiryInterval(); int defaultMaps = 0; int defaultReduces = 0; try { defaultMaps = jobClient.getDefaultMaps(); defaultReduces = jobClient.getDefaultReduces(); } catch (Exception e) { //e.printStackTrace(); } // Putting Hadoop Metrics information in a map hadoopJobMetrics.put("jobTrackerState", String.valueOf(jobTrackerState)); hadoopJobMetrics.put("defaultMaps", String.valueOf(defaultMaps)); hadoopJobMetrics.put("defaultReduces", String.valueOf(defaultReduces)); hadoopJobMetrics.put("mapTasks", String.valueOf(mapTasks)); hadoopJobMetrics.put("reduceTasks", String.valueOf(reduceTasks)); hadoopJobMetrics.put("maxMapTasksCapacity", String.valueOf(maxMapTasks)); hadoopJobMetrics.put("maxReduceTasksCapacity", String.valueOf(maxReduceTasks)); hadoopJobMetrics.put("taskTrackers", String.valueOf(taskTrackers)); hadoopJobMetrics.put("blackListedTrackers", String.valueOf(blackListedTrackers)); hadoopJobMetrics.put("taskTrackerExpiryInterval", String.valueOf(ttExpiryInterval)); hadoopJobMetrics.put("schedulerType", getSchedulerType()); int totalJobSubmission = 0; // Get the jobs that are submitted. JobStatus[] jobStatus = jobClient.getAllJobs(); if (jobStatus != null) { totalJobSubmission = jobClient.getAllJobs().length; } List<Map<String, Object>> allJobsList = listAllJobs(); int totalJobRunning = getRunningJobList(allJobsList).size(); int completedJobs = getCompletedJobs(allJobsList).size(); hadoopJobMetrics.put("totalJobSubmission", String.valueOf(totalJobSubmission)); hadoopJobMetrics.put("totalJobRunning", String.valueOf(totalJobRunning)); hadoopJobMetrics.put("totalJobsCompleted", String.valueOf(completedJobs)); } else { HadoopUtils.addAndLogError(this.LOG, this.clusterConfig, errMsg, Constant.Component.Name.HADOOP); throw new AnkushException(errMsg); } } catch (AnkushException e) { throw e; } catch (Exception e) { HadoopUtils.addAndLogError(this.LOG, this.clusterConfig, errMsg, Constant.Component.Name.HADOOP, e); throw new AnkushException(errMsg); } return hadoopJobMetrics; } else { throw new AnkushException(errMsg); } } catch (AnkushException e) { throw e; } catch (Exception e) { HadoopUtils.addAndLogError(this.LOG, this.clusterConfig, errMsg, Constant.Component.Name.HADOOP, e); throw new AnkushException(errMsg); } }
From source file:com.scaleunlimited.cascading.hadoop.HadoopUtils.java
License:Apache License
/** * Utility routine that tries to ensure the cluster is "stable" (slaves have reported in) so * that it's safe to call things like maxReduceTasks. * //w w w.jav a 2 s .com * @param conf * @return * @throws IOException * @throws InterruptedException */ @SuppressWarnings("deprecation") private static ClusterStatus safeGetClusterStatus(JobConf conf) throws IOException, InterruptedException { JobClient jobClient = new JobClient(conf); int numTaskTrackers = -1; while (true) { ClusterStatus status = jobClient.getClusterStatus(); // TODO there isn't a "job tracker" in MR2, just a resource manager, and a transient // application manager for running a Hadoop job. if (status.getJobTrackerState() == State.RUNNING) { int curTaskTrackers = status.getTaskTrackers(); if (curTaskTrackers == numTaskTrackers) { return status; } else { // Things are still settling down, so keep looping. if (numTaskTrackers != -1) { LOGGER.trace(String.format("Got incremental update to number of task trackers (%d to %d)", numTaskTrackers, curTaskTrackers)); } numTaskTrackers = curTaskTrackers; } } if (!isJobLocal(conf)) { LOGGER.trace("Sleeping during status check"); Thread.sleep(STATUS_CHECK_INTERVAL); } } }
From source file:org.pentaho.hadoop.mapreduce.test.TestSubmitMapReduceJob.java
License:Open Source License
@Test public void submitJob() throws Exception { String[] args = { "hdfs://" + hostname + ":" + hdfsPort + "/junit/wordcount/input", "hdfs://" + hostname + ":" + hdfsPort + "/junit/wordcount/output" }; JobConf conf = new JobConf(); conf.setJobName("wordcount"); conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(IntWritable.class); File jar = new File("./test-res/pentaho-mapreduce-sample.jar"); URLClassLoader loader = new URLClassLoader(new URL[] { jar.toURI().toURL() }); conf.setMapperClass(//from w w w. j a v a2s.c o m (Class<? extends Mapper>) loader.loadClass("org.pentaho.hadoop.mapreduce.sample.MRWordCount$Map")); conf.setCombinerClass((Class<? extends Reducer>) loader .loadClass("org.pentaho.hadoop.mapreduce.sample.MRWordCount$Reduce")); conf.setReducerClass((Class<? extends Reducer>) loader .loadClass("org.pentaho.hadoop.mapreduce.sample.MRWordCount$Reduce")); conf.setInputFormat(TextInputFormat.class); conf.setOutputFormat(TextOutputFormat.class); FileInputFormat.setInputPaths(conf, new Path(args[0])); FileOutputFormat.setOutputPath(conf, new Path(args[1])); conf.set("fs.default.name", "hdfs://" + hostname + ":" + hdfsPort); conf.set("mapred.job.tracker", hostname + ":" + trackerPort); conf.setJarByClass(loader.loadClass("org.pentaho.hadoop.mapreduce.sample.MRWordCount")); conf.setWorkingDirectory(new Path("/tmp/wordcount")); JobClient jobClient = new JobClient(conf); ClusterStatus status = jobClient.getClusterStatus(); assertEquals(State.RUNNING, status.getJobTrackerState()); RunningJob runningJob = jobClient.submitJob(conf); System.out.print("Running " + runningJob.getJobName() + ""); while (!runningJob.isComplete()) { System.out.print("."); Thread.sleep(500); } System.out.println(); System.out.println("Finished " + runningJob.getJobName() + "."); FileObject file = fsManager.resolveFile(buildHDFSURL("/junit/wordcount/output/part-00000")); String output = IOUtils.toString(file.getContent().getInputStream()); assertEquals("Bye\t1\nGoodbye\t1\nHadoop\t2\nHello\t2\nWorld\t2\n", output); }
From source file:org.pentaho.hadoop.mapreduce.test.TransMapReduceJobTestFIXME.java
License:Open Source License
@Test public void submitJob() throws Exception { String[] args = { "hdfs://" + hostname + ":" + hdfsPort + "/junit/wordcount/input", "hdfs://" + hostname + ":" + hdfsPort + "/junit/wordcount/output" }; JobConf conf = new JobConf(); conf.setJobName("wordcount"); KettleEnvironment.init();/* www .j a v a 2 s . co m*/ TransExecutionConfiguration transExecConfig = new TransExecutionConfiguration(); TransMeta transMeta = new TransMeta("./test-res/wordcount-mapper.ktr"); TransConfiguration transConfig = new TransConfiguration(transMeta, transExecConfig); conf.set("transformation-map-xml", transConfig.getXML()); transMeta = new TransMeta("./test-res/wordcount-reducer.ktr"); transConfig = new TransConfiguration(transMeta, transExecConfig); conf.set("transformation-reduce-xml", transConfig.getXML()); conf.set("transformation-map-input-stepname", "Injector"); conf.set("transformation-map-output-stepname", "Output"); conf.set("transformation-reduce-input-stepname", "Injector"); conf.set("transformation-reduce-output-stepname", "Output"); conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(IntWritable.class); File jar = new File("./dist/pentaho-big-data-plugin-TRUNK-SNAPSHOT.jar"); URLClassLoader loader = new URLClassLoader(new URL[] { jar.toURI().toURL() }); conf.setMapperClass( (Class<? extends Mapper>) loader.loadClass("org.pentaho.hadoop.mapreduce.GenericTransMap")); conf.setCombinerClass( (Class<? extends Reducer>) loader.loadClass("org.pentaho.hadoop.mapreduce.GenericTransReduce")); conf.setReducerClass( (Class<? extends Reducer>) loader.loadClass("org.pentaho.hadoop.mapreduce.GenericTransReduce")); conf.setInputFormat(TextInputFormat.class); conf.setOutputFormat(TextOutputFormat.class); FileInputFormat.setInputPaths(conf, new Path(args[0])); FileOutputFormat.setOutputPath(conf, new Path(args[1])); conf.set("fs.default.name", "hdfs://" + hostname + ":" + hdfsPort); conf.set("mapred.job.tracker", hostname + ":" + trackerPort); conf.setJar(jar.toURI().toURL().toExternalForm()); conf.setWorkingDirectory(new Path("/tmp/wordcount")); JobClient jobClient = new JobClient(conf); ClusterStatus status = jobClient.getClusterStatus(); assertEquals(State.RUNNING, status.getJobTrackerState()); RunningJob runningJob = jobClient.submitJob(conf); System.out.print("Running " + runningJob.getJobName() + ""); while (!runningJob.isComplete()) { System.out.print("."); Thread.sleep(500); } System.out.println(); System.out.println("Finished " + runningJob.getJobName() + "."); FileObject file = fsManager.resolveFile(buildHDFSURL("/junit/wordcount/output/part-00000")); String output = IOUtils.toString(file.getContent().getInputStream()); assertEquals( "Bye\t4\nGood\t2\nGoodbye\t1\nHadoop\t2\nHello\t5\nThis\t1\nWorld\t5\nand\t1\ncounting\t1\nextra\t1\nfor\t1\nis\t1\nsome\t1\ntext\t1\nwords\t1\n", output); }
From source file:org.smartfrog.services.hadoop.components.cluster.ClusterStatusCheckerImpl.java
License:Open Source License
/** * Check the cluster status// w ww . j a v a 2 s. c o m * * @throws SFHadoopException on any problem with the checks * @return a cluster status string */ private String checkClusterStatus() throws SmartFrogException { try { JobClient cluster = createClientOnDemand(); ClusterStatus status = cluster.getClusterStatus(); StringBuilder result = new StringBuilder(); if (supportedFileSystem) { Path sysDir = cluster.getSystemDir(); URI uri = sysDir.toUri(); sfLog().info("Checking filesystem " + uri); ManagedConfiguration conf = (ManagedConfiguration) cluster.getConf(); String impl = "fs." + uri.getScheme() + ".impl"; String classname = conf.get(impl); if (classname == null) { maybeDumpConfiguration(conf); throw new SFHadoopException("File system " + uri + " will not load " + " - no configuration mapping for " + impl + " in " + conf.dump(), this, conf); } try { conf.getClassByName(classname); } catch (ClassNotFoundException e) { throw new SFHadoopException("File system " + uri + " will not load " + " - unable to locate class " + impl + " : " + e, e, this, conf); } try { result.append("Filesystem: ").append(uri).append(" ; "); FileSystem fs = cluster.getFs(); } catch (IOException e) { throw new SFHadoopException("File system " + uri + " will not load " + e, e, this, conf); } catch (IllegalArgumentException e) { throw new SFHadoopException("Bad File system URI" + e, e, this, conf); } } if (jobtrackerLive) { sfLog().info("Checking jobTracker "); JobTracker.State state = status.getJobTrackerState(); if (!state.equals(JobTracker.State.RUNNING)) { throw new SFHadoopException( "Job Tracker at " + jobtracker + " is not running. It is in the state " + state, this); } result.append("Job tracker is in state ").append(status); } checkRange(minActiveMapTasks, maxActiveMapTasks, status.getMapTasks(), "map task"); checkRange(minActiveReduceTasks, maxActiveReduceTasks, status.getReduceTasks(), "reduce task"); checkMax(maxSupportedMapTasks, status.getMaxMapTasks(), "supported max map task"); checkMax(maxSupportedReduceTasks, status.getMaxReduceTasks(), "supported max reduce task"); result.append(" Map Tasks = ").append(status.getMapTasks()); result.append(" Reduce Tasks = ").append(status.getReduceTasks()); return result.toString(); } catch (IOException e) { throw new SFHadoopException("Cannot connect to" + jobtracker, e, this); } }