Example usage for org.apache.hadoop.mapred SkipBadRecords setAutoIncrMapperProcCount

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred SkipBadRecords setAutoIncrMapperProcCount.

Prototype

public static void setAutoIncrMapperProcCount(Configuration conf, boolean autoIncr) 

Source Link

Document

Set the flag which if set to true, SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS is incremented by MapRunner after invoking the map function.

Usage

From source file:edu.stolaf.cs.wmrserver.streaming.PipeMapper.java

License:Apache License

public void configure(JobConf job) {
    super.configure(job);
    //disable the auto increment of the counter. For streaming, no of 
    //processed records could be different(equal or less) than the no of 
    //records input.
    SkipBadRecords.setAutoIncrMapperProcCount(job, false);
    skipping = job.getBoolean("mapred.skip.on", false);
    String inputFormatClassName = job.getClass("mapred.input.format.class", TextInputFormat.class)
            .getCanonicalName();/*from   w w w  .jav  a  2  s.c o  m*/
    ignoreKey = inputFormatClassName.equals(TextInputFormat.class.getCanonicalName());

    try {
        mapOutputFieldSeparator = job.get("stream.map.output.field.separator", "\t").getBytes("UTF-8");
        mapInputFieldSeparator = job.get("stream.map.input.field.separator", "\t").getBytes("UTF-8");
        numOfMapOutputKeyFields = job.getInt("stream.num.map.output.key.fields", 1);
    } catch (UnsupportedEncodingException e) {
        throw new RuntimeException("The current system does not support UTF-8 encoding!", e);
    }
}

---

From source file:it.crs4.pydoop.pipes.PipesMapRunner.java

License:Apache License

/**
 * Get the new configuration.// w w w.  ja  va2  s.c  om
 * @param job the job's configuration
 */
public void configure(JobConf job) {
    this.job = job;
    //disable the auto increment of the counter. For pipes, no of processed 
    //records could be different(equal or less) than the no of records input.
    SkipBadRecords.setAutoIncrMapperProcCount(job, false);
}

---