Example usage for org.apache.hadoop.mapreduce Job getGroupingComparator

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job getGroupingComparator.

Prototype

public RawComparator<?> getGroupingComparator() 

Source Link

Document

Get the user defined RawComparator comparator for grouping keys of inputs to the reduce.

Usage

From source file:com.asakusafw.runtime.mapreduce.simple.SimpleJobRunner.java

License:Apache License

@SuppressWarnings({ "unchecked", "rawtypes" })
private void runReduce(Job job, KeyValueSorter<?, ?> sorter)
        throws ClassNotFoundException, IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    OutputFormat<?, ?> output = ReflectionUtils.newInstance(job.getOutputFormatClass(), conf);
    TaskAttemptID id = newTaskAttemptId(newReduceTaskId(job.getJobID(), 1), 0);
    Reducer<?, ?, ?, ?> reducer = ReflectionUtils.newInstance(job.getReducerClass(), conf);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("starting reducer: {0}@{1} ({2}records, {3}bytes)", //$NON-NLS-1$
                reducer.getClass().getName(), id, sorter.getRecordCount(), sorter.getSizeInBytes()));
    }/*  www  .  j  a v  a2s.  c  o m*/
    TaskAttemptContext context = newTaskAttemptContext(conf, id);
    OutputCommitter committer = output.getOutputCommitter(context);
    committer.setupTask(context);
    boolean succeed = false;
    try {
        ShuffleReader reader = new ShuffleReader(sorter, new Progress());
        try {
            RecordWriter<?, ?> writer = output.getRecordWriter(newTaskAttemptContext(conf, id));
            try {
                Reducer.Context c = newReducerContext(conf, id, reader, sorter.getKeyClass(),
                        sorter.getValueClass(), writer, committer, (RawComparator) job.getGroupingComparator());
                reducer.run(c);
            } finally {
                writer.close(newTaskAttemptContext(conf, id));
            }
        } finally {
            try {
                reader.close();
            } catch (IOException e) {
                LOG.warn(MessageFormat.format("error occurred while reducer mapper input: {0} ({1})", id,
                        job.getJobName()), e);
            }
        }
        doCommitTask(context, committer);
        succeed = true;
    } finally {
        if (succeed == false) {
            doAbortTask(context, committer);
        }
    }
}

---

From source file:org.huahinframework.unit.JobDriver.java

License:Apache License

/**
 * @param job// ww w . j a  v a 2s  . c o m
 * @return List<Pair<Key, Value>>
 * @throws InstantiationException
 * @throws IllegalAccessException
 * @throws ClassNotFoundException
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
private MapReduceDriver<WritableComparable, Writable, WritableComparable, Writable, WritableComparable, Writable> createDriver(
        Job job) throws InstantiationException, IllegalAccessException, ClassNotFoundException {
    Mapper mapper = job.getMapperClass().newInstance();
    Reducer reducer = job.getReducerClass().newInstance();
    RawComparator groupingComparator = job.getGroupingComparator();
    RawComparator sortComparator = job.getSortComparator();
    MapReduceDriver<WritableComparable, Writable, WritableComparable, Writable, WritableComparable, Writable> driver = MapReduceDriver
            .newMapReduceDriver(mapper, reducer).withKeyGroupingComparator(groupingComparator)
            .withKeyOrderComparator(sortComparator);
    driver.setConfiguration(job.getConfiguration());
    return driver;
}

---