Example usage for org.apache.hadoop.mapred Counters findCounter

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred Counters findCounter.

Prototype

public synchronized Counter findCounter(String group, String name) 

Source Link

Usage

From source file:com.twitter.ambrose.cascading.CascadingJob.java

License:Apache License

/**
 * helper method for counter group map retrieval
 * @param  HadoopStepStats//from  ww  w  .  j a  v a2  s .  c o  m
 * @return a map of counter name to counter value
 */
private Map<String, Long> counterGroupInfoMapHelper(HadoopStepStats stats) {
    Counters counters = new Counters();
    Map<String, Long> counterNameToValue = new HashMap<String, Long>();
    for (String groupName : stats.getCounterGroups()) { //retreiving groups
        for (String counterName : stats.getCountersFor(groupName)) { //retreiving counters in that group
            Long counterValue = stats.getCounterValue(groupName, counterName);
            counterNameToValue.put(groupName + "::" + counterName, counterValue);

            //creating counter
            Counter counter = counters.findCounter(groupName, counterName);
            counter.setValue(counterValue);
        }
    }
    setCounterGroupMap(CounterGroup.counterGroupInfoMap(counters));
    return counterNameToValue;
}

---

From source file:com.twitter.ambrose.cascading3.CascadingJob.java

License:Apache License

@JsonIgnore
public void setJobStats(HadoopStepStats stats) {
    Counters counters = new Counters();
    for (String groupName : stats.getCounterGroups()) {
        for (String counterName : stats.getCountersFor(groupName)) {
            Long counterValue = stats.getCounterValue(groupName, counterName);
            counters.findCounter(groupName, counterName).setValue(counterValue);
        }//from  w  w w  .j a va 2 s  .c om
    }
    setCounterGroupMap(CounterGroup.counterGroupsByName(counters));
}

---

From source file:com.twitter.ambrose.hive.AmbroseHiveUtil.java

License:Apache License

/**
 * Constructs Countergroups from job runtime statistics
 * /* www. j  a  va  2 s.com*/
 * @param counterNameToValue
 * @return
 */
public static Map<String, CounterGroup> counterGroupInfoMap(Map<String, Double> counterNameToValue) {

    Counters counters = new Counters();
    for (Map.Entry<String, ? extends Number> entry : counterNameToValue.entrySet()) {

        String[] cNames = entry.getKey().split("::");
        String groupName = cNames[0];
        String counterName = cNames[1];
        Counter counter = counters.findCounter(groupName, counterName);
        counter.setValue(entry.getValue().longValue());
    }
    return CounterGroup.counterGroupInfoMap(counters);
}

---

From source file:org.acacia.csr.java.LineCount.java

License:Apache License

public static void main(String[] args) throws Exception {
    /*//from   ww w .  j  av  a 2  s  .  c  o  m
      String dir1 = "/user/miyuru/wcout";
      String dir2 = "/user/miyuru/lcout";
       //We first delete the temporary directories if they exist on the HDFS
        FileSystem fs1 = FileSystem.get(new JobConf());
                
       if(fs1.exists(new Path(dir2))){
          fs1.delete(new Path(dir2), true);
       }
            
        JobConf conf = new JobConf(LineCount.class);
        conf.setJobName("LineCount");
               
        conf.setOutputKeyClass(IntWritable.class);
        conf.setOutputValueClass(IntWritable.class);
               
        conf.setMapperClass(Map.class);
        conf.setCombinerClass(Reduce.class);
        conf.setReducerClass(Reduce.class);
               
        conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(TextOutputFormat.class);
               
        FileInputFormat.setInputPaths(conf, new Path(dir1));
        FileOutputFormat.setOutputPath(conf, new Path(dir2));
               
        Job job = new Job(conf, "line count");
        job.waitForCompletion(true); 
        org.apache.hadoop.mapreduce.Counters cntr = job.getCounters();
        System .out.println("Number of lines in the file" + cntr.findCounter("org.apache.hadoop.mapred.Task$Counter", "MAP_INPUT_RECORDS").getValue());
        */

    long edgeCount = 0;
    //String dir3 = "/user/miyuru/wcout";
    String dir4 = "/user/miyuru/lcout";
    String dir5 = "/user/miyuru/input";
    //We first delete the temporary directories if they exist on the HDFS
    FileSystem fs2 = FileSystem.get(new JobConf());

    if (fs2.exists(new Path(dir4))) {
        fs2.delete(new Path(dir4), true);
    }

    JobConf conf1 = new JobConf(LineCount.class);
    conf1.setJobName("LineCount");

    conf1.setOutputKeyClass(Text.class);
    conf1.setOutputValueClass(IntWritable.class);

    conf1.setMapperClass(Map.class);
    conf1.setCombinerClass(Reduce.class);
    conf1.setReducerClass(Reduce.class);

    conf1.setInputFormat(TextInputFormat.class);
    conf1.setOutputFormat(TextOutputFormat.class);

    FileInputFormat.setInputPaths(conf1, new Path(dir5));
    FileOutputFormat.setOutputPath(conf1, new Path(dir4));

    Job job1 = new Job(conf1, "line count");
    job1.setNumReduceTasks(0);
    job1.waitForCompletion(true);
    org.apache.hadoop.mapreduce.Counters cntr = job1.getCounters();
    edgeCount = cntr.findCounter("org.apache.hadoop.mapred.Task$Counter", "MAP_INPUT_RECORDS").getValue();

    File efile = new File("/tmp/efile");

    if (efile.exists()) {
        efile.delete();
    }

    PrintWriter writer = new PrintWriter("/tmp/efile", "UTF-8");
    writer.println(edgeCount);
    writer.flush();
    writer.close();

    //edgeCount = edgeCount -1;//This is to remove the line number additionlly added to each edgelist file by HDFS. This is strange, but it happens.
    System.out.println("======>Edge count is : " + edgeCount);
    System.out.println("------Done Line Count---------------");
}

---