List of usage examples for org.apache.hadoop.mapreduce Counter getValue
long getValue();
From source file:org.huahinframework.manager.rest.service.JobService.java
License:Apache License
/** * @param jobId//from w w w . j av a 2 s . c om * @return {@link JSONObject} * @throws IOException * @throws InterruptedException */ private Map<String, Object> getStatus(String jobId) throws IOException, InterruptedException { Map<String, Object> job = null; Cluster cluster = new Cluster(getJobConf()); for (JobStatus jobStatus : cluster.getAllJobStatuses()) { if (jobStatus.getJobID().toString().equals(jobId)) { job = JobUtils.getJob(jobStatus); Job j = cluster.getJob(jobStatus.getJobID()); if (j == null) { break; } Calendar finishTime = Calendar.getInstance(); finishTime.setTimeInMillis(j.getFinishTime()); job.put(Response.FINISH_TIME, finishTime.getTime().toString()); Map<String, Map<String, Long>> groups = new HashMap<String, Map<String, Long>>(); for (String s : j.getCounters().getGroupNames()) { CounterGroup counterGroup = j.getCounters().getGroup(s); Iterator<Counter> ite = counterGroup.iterator(); Map<String, Long> counters = new HashMap<String, Long>(); groups.put(counterGroup.getDisplayName(), counters); while (ite.hasNext()) { Counter counter = (Counter) ite.next(); counters.put(counter.getDisplayName(), counter.getValue()); } } job.put(Response.GROUPS, groups); break; } } return job; }
From source file:org.kiji.mapreduce.framework.JobHistoryKijiTable.java
License:Apache License
/** * Helper method to write individual counters to job history table's counter family. * * @param writer The {@link KijiTableWriter} for the job history table. * @param job The {@link Job} whose counters we are recording. * @throws IOException If there is an error writing to the table. */// w w w .j a v a2 s . c o m private void writeIndividualCounters(KijiTableWriter writer, Job job) throws IOException { EntityId jobEntity = mKijiTable.getEntityId(job.getJobID().toString()); Counters counters = job.getCounters(); for (String grpName : counters.getGroupNames()) { Iterator<Counter> counterIterator = counters.getGroup(grpName).iterator(); while (counterIterator.hasNext()) { Counter ctr = counterIterator.next(); writer.put(jobEntity, JOB_HISTORY_COUNTERS_FAMILY, grpName + ":" + ctr.getName(), ctr.getValue()); } } }
From source file:org.oclc.firefly.hadoop.backup.Backup.java
License:Apache License
/** * Performs a complete copy of the source hbase to the given destination * @param tables The names of the tables to backup * @param maxTries The maximum number of times to try to copy regions. * @return True if successful, false otherwise * @throws IOException If failed to interact with Hadoop * @throws ClassNotFoundException /*from w w w.j ava2 s. c om*/ * @throws InterruptedException */ public boolean doMajorCopy(String[] tables, int maxTries) throws IOException, InterruptedException, ClassNotFoundException { boolean ret = false; String username = getUsername(); short replication = (short) getInitialReplication(); // Get a list of regions from HBase // Then filter out the regions we are not extracting, and group them by table List<CatalogRow> regions = getHBaseRegions(srcConf); Map<String, List<CatalogRow>> filtered = groupAndFilter(regions, tables); List<Pair<String, HRegionInfo>> mapperInput = new ArrayList<Pair<String, HRegionInfo>>(); // Prepare the input for the mappers to use // This creates a list of region server and region pairs LOG.info("Exporting the following tables:"); for (Entry<String, List<CatalogRow>> entry : filtered.entrySet()) { String tablename = entry.getKey(); List<CatalogRow> rows = entry.getValue(); LOG.info(". " + tablename); for (CatalogRow r : rows) { String regionServer = r.getHost() + ":" + r.getPort(); HRegionInfo region = r.getHRegionInfo(); mapperInput.add(Pair.newPair(regionServer, region)); } } // Make sure we write to a directory that does not exist backupDirectoryPath = createBackupDirectory(getCurrentDateString()); LOG.info("Starting backup path: " + backupDirectoryPath); // Copy the .tableinfo files for the tables we are extracting // These files are not copied by the MR job as it only focuses on regions List<FileStatus> tableInfoFiles = getTableInfoFiles(srcFs, filtered); for (FileStatus file : tableInfoFiles) { Path srcFilePath = file.getPath(); Path relPath = new Path(BackupUtils.getFsRelativePath(srcFs, srcFilePath)); Path dstFilePath = new Path(backupDirectoryPath.toString() + relPath.toString()); BackupUtils.copy(srcFs, srcFilePath, dstFs, dstFilePath, buffer, username, replication); } // Dispatch MR job and monitor // Retry regions if necessary if (mapperInput.size() > 0) { int tries = 0; while (!ret && (maxTries == 0 || tries < maxTries)) { if (getNumMapTasks() > mapperInput.size()) { setNumMapTasks(mapperInput.size()); LOG.info("Not enough regions. Reducing number of map tasks"); } // Generate a list of mapper input files and create job List<Path> sourceFiles = createMapperInputSequenceFiles(mapperInput, getNumMapTasks(), srcFs, tries); Job job = createMRJob(srcConf, dstConf, sourceFiles, backupDirectoryPath, tries); LOG.info(job.getJobName()); LOG.info("--------------------------------------------------"); LOG.info("Number of regions : " + mapperInput.size()); LOG.info("Number of map tasks: " + getNumMapTasks()); LOG.info("Mapper input path : " + getMapInputDirectory(tries)); LOG.info("Mapper output path : " + FileOutputFormat.getOutputPath(job)); LOG.info("--------------------------------------------------"); job.waitForCompletion(true); if (job.isSuccessful()) { // Check if any regions failed Counters counters = job.getCounters(); Counter failedCounter = counters.findCounter("Backup", "FailedRegions"); long failed = failedCounter.getValue(); if (failed > 0) { LOG.info("Number of failed regions: " + failed + "."); // get a fresh list of regions to copy List<Pair<String, HRegionInfo>> failedRegions = getFailedRegions(srcFs, srcConf, tries); addCopiedRegions(mapperInput, failedRegions); mapperInput = getRemainingRegions(mapperInput, tables); for (Pair<String, HRegionInfo> pair : mapperInput) { LOG.info("Retry: " + pair.getSecond()); } if (mapperInput.size() == 0) { ret = true; backupDirectoryPath = appendEndTime(backupDirectoryPath); LOG.warn("No regions left to copy, but expected to copy more. " + "Please inspect logs/files manually for errors"); } } else { ret = true; addCopiedRegions(mapperInput, null); backupDirectoryPath = appendEndTime(backupDirectoryPath); LOG.info("MR job finished successfully"); } } else { LOG.error("An unexpected error occurred during the MR job. Please see MR logs."); break; } tries++; } if (ret) { if (verifyCopiedRegions()) { LOG.info("Verification passed succesfully"); } else { ret = false; LOG.info("Verification failed. Please inspect errors manually"); } } else { LOG.info("No attempts left. Try setting -n to a higher value, or setting it to 0"); } } if (ret) { // Set replication factor of backup directory to default. // This may not be the best solution, but let built-in shell take care of it // because it can do it recursively with out us having to rediscover all the files short finalReplication = (short) getFinalReplication(); if (replication != finalReplication) { FsShell shell = new FsShell(dstConf); String[] repArgs = { "-setrep", "-R", "-w", "" + finalReplication, backupDirectoryPath.toString() }; try { LOG.info("Setting final replication factor of backup files to " + finalReplication); shell.run(repArgs); } catch (Exception e) { LOG.warn("Could not set replication factor of backup files to " + finalReplication); } } } return ret; }
From source file:org.opencloudengine.flamingo.mapreduce.util.CounterUtils.java
License:Apache License
/** * Job? ?? Counter Map ? ./*from w ww .j a v a2s .c o m*/ * Key? <tt>GROUP_COUNTER</tt> ? Group Name? * <tt>CLEAN</tt>?, Counter <tt>VALID</tt>? Key * <tt>CLEAN_VALID</tt> ?. * * @param job Hadoop Job * @return Counter ? ? Map */ public static Map<String, String> getCounters(Job job) { Map<String, String> resultMap = new HashMap<String, String>(); try { Counters counters = job.getCounters(); Iterable<String> groupNames = counters.getGroupNames(); Iterator<String> groupIterator = groupNames.iterator(); while (groupIterator.hasNext()) { String groupName = groupIterator.next(); CounterGroup group = counters.getGroup(groupName); Iterator<Counter> counterIterator = group.iterator(); while (counterIterator.hasNext()) { Counter counter = counterIterator.next(); logger.info("[{}] {} = {}", new Object[] { group.getName(), counter.getName(), counter.getValue() }); String realName = HadoopMetrics.getMetricName(group.getName() + "_" + counter.getName()); if (!StringUtils.isEmpty(realName)) { resultMap.put(realName, String.valueOf(counter.getValue())); } } } } catch (Exception ex) { } return resultMap; }
From source file:org.openflamingo.mapreduce.util.CounterUtils.java
License:Apache License
/** * Job? ?? Counter Map ? .//w w w.java 2s . c om * Key? <tt>GROUP_COUNTER</tt> ? Group Name? * <tt>CLEAN</tt>?, Counter <tt>VALID</tt>? Key * <tt>CLEAN_VALID</tt> ?. * * @param job Hadoop Job * @return Counter ? ? Map */ public static Map<String, String> getCounters(Job job) { Map<String, String> resultMap = new HashMap<String, String>(); try { Counters counters = job.getCounters(); Collection<String> groupNames = counters.getGroupNames(); Iterator<String> groupIterator = groupNames.iterator(); while (groupIterator.hasNext()) { String groupName = groupIterator.next(); CounterGroup group = counters.getGroup(groupName); Iterator<Counter> counterIterator = group.iterator(); while (counterIterator.hasNext()) { Counter counter = counterIterator.next(); logger.info("[{}] {} = {}", new Object[] { group.getName(), counter.getName(), counter.getValue() }); String realName = HadoopMetrics.getMetricName(group.getName() + "_" + counter.getName()); if (!StringUtils.isEmpty(realName)) { resultMap.put(realName, String.valueOf(counter.getValue())); } } } } catch (Exception ex) { } return resultMap; }
From source file:org.springframework.data.hadoop.batch.mapreduce.JobTasklet.java
License:Apache License
private void saveCounters(Job job, StepContribution contribution) { Counters counters = null;/*from w w w . j a va2 s .c o m*/ try { counters = job.getCounters(); } catch (Exception ex) { if (RuntimeException.class.isAssignableFrom(ex.getClass())) { throw (RuntimeException) ex; } else { // ignore - we just can't get stats } } if (counters == null) { return; } Counter count = counters.findCounter(Task.Counter.MAP_INPUT_RECORDS); for (int i = 0; i < safeLongToInt(count.getValue()); i++) { contribution.incrementReadCount(); } count = counters.findCounter(Task.Counter.MAP_SKIPPED_RECORDS); contribution.incrementReadSkipCount(safeLongToInt(count.getValue())); count = counters.findCounter(Task.Counter.REDUCE_OUTPUT_RECORDS); contribution.incrementWriteCount(safeLongToInt(count.getValue())); count = counters.findCounter(Task.Counter.REDUCE_SKIPPED_RECORDS); for (int i = 0; i < safeLongToInt(count.getValue()); i++) { contribution.incrementWriteSkipCount(); } }
From source file:org.springframework.data.hadoop.batch.mapreduce.JobTasklet.java
License:Apache License
private static void saveJobStats(Job job, StepExecution stepExecution) { if (stepExecution == null) { return;//from www. j av a 2 s . c om } ExecutionContext executionContext = stepExecution.getExecutionContext(); String statusPrefix = "Job Status::"; executionContext.put(statusPrefix + "ID", JobUtils.getJobId(job).toString()); executionContext.put(statusPrefix + "Name", job.getJobName()); executionContext.put(statusPrefix + "Tracking URL", job.getTrackingURL()); executionContext.put(statusPrefix + "State", JobUtils.getStatus(job).toString()); try { for (String cgName : job.getCounters().getGroupNames()) { CounterGroup group = job.getCounters().getGroup(cgName); Iterator<Counter> ci = group.iterator(); while (ci.hasNext()) { Counter c = ci.next(); executionContext.put(group.getDisplayName().trim() + "::" + c.getDisplayName().trim(), c.getValue()); } } } catch (Exception ignore) { } }
From source file:org.trend.hgraph.mapreduce.pagerank.Driver.java
License:Apache License
private static int collectVeticesTotalCount(Configuration conf, String vertexTableName) throws IOException, InterruptedException, ClassNotFoundException { long totalCount = 1L; boolean success = false; Counter counter = null; String jobName = null;//from w w w . j a va2 s . c o m try { Job job = RowCounter.createSubmittableJob(conf, new String[] { vertexTableName }); if (job == null) { System.err.println("job is null"); return 1; } success = job.waitForCompletion(true); counter = job.getCounters() .findCounter("org.apache.hadoop.hbase.mapreduce.RowCounter$RowCounterMapper$Counters", "ROWS"); jobName = job.getJobName(); if (null != counter) { totalCount = counter.getValue(); conf.set(Constants.PAGE_RANK_VERTICES_TOTAL_COUNT_KEY, totalCount + ""); } LOGGER.info(Constants.PAGE_RANK_VERTICES_TOTAL_COUNT_KEY + "=" + totalCount); } catch (IOException e) { LOGGER.error("run " + jobName + " failed", e); throw e; } catch (InterruptedException e) { LOGGER.error("run " + jobName + " failed", e); throw e; } catch (ClassNotFoundException e) { LOGGER.error("run " + jobName + " failed", e); throw e; } return success ? 0 : -1; }
From source file:org.trend.hgraph.mapreduce.pagerank.GetNoColumnsRows.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (null == args || args.length == 0) { System.err.println("no any option given !!"); printUsage();/* ww w . ja v a2s. c om*/ return -1; } System.out.println("options:" + Arrays.toString(args)); boolean and = true; String cmd = null; int mustStartIdx = -1; for (int a = 0; a < args.length; a++) { cmd = args[a]; if (cmd.startsWith("-")) { if (mustStartIdx > -1) { System.err.println("option order is incorrect !!"); printUsage(); return -1; } if ("-a".equals(cmd)) { and = true; } else if ("-o".equals(cmd)) { and = false; } else { System.err.println("option is not defined !!"); printUsage(); return -1; } } else { if (mustStartIdx == -1) { mustStartIdx = a; } } } String tableName = args[mustStartIdx]; String outputPath = args[mustStartIdx + 1]; List<String> columns = new ArrayList<String>(); for (int a = mustStartIdx + 2; a < args.length; a++) { columns.add(args[a]); } LOGGER.info("tableName=" + tableName); LOGGER.info("outputPath=" + outputPath); LOGGER.info("columns=" + columns); Configuration conf = this.getConf(); conf.setBoolean(Mapper.AND_OR, and); conf.setStrings(Mapper.NO_COLUMNS, columns.toArray(new String[] {})); Job job = createSubmittableJob(conf, tableName, outputPath); boolean success = job.waitForCompletion(true); if (!success) { System.err.println("run job:" + job.getJobName() + " failed"); return -1; } // for test Counter counter = job.getCounters().findCounter( "org.trend.hgraph.mapreduce.pagerank.GetNoColumnsRows$Mapper$Counters", "COLLECTED_ROWS"); if (null != counter) { collectedRow = counter.getValue(); } return 0; }
From source file:sculptor.framework.HClient.java
License:Apache License
/** * Row count using mapreduce./*from w w w.jav a2s.co m*/ * * @param kv entity * @param ops filter conditions * @return row count if job completed successfully, -1 if failed. * @throws Exception */ public long countMR(D kv, Map<String, HCompareOp> ops) throws Exception { String jobName = "Count " + tableName; Job job = new Job(this.getConfig(), jobName); job.setJarByClass(HClient.class); // scan setting Scan scan = getRawScan(kv, ops); scan.setCacheBlocks(false); // initialize the mapper TableMapReduceUtil.initTableMapperJob(getTableName(), scan, CountMapper.class, ImmutableBytesWritable.class, Result.class, job, false); job.setNumReduceTasks(0); job.setOutputFormatClass(NullOutputFormat.class); boolean jobResult = job.waitForCompletion(true); if (!jobResult) { return -1; } Counters counters = job.getCounters(); Counter rowCounter = counters.findCounter(CountMapper.Counters.ROWS); return rowCounter.getValue(); }