List of usage examples for org.apache.hadoop.mapreduce.lib.reduce WrappedReducer WrappedReducer
WrappedReducer
From source file:co.cask.cdap.internal.app.runtime.batch.ReducerWrapper.java
License:Apache License
private WrappedReducer.Context createAutoFlushingContext(final Context context, final BasicMapReduceTaskContext basicMapReduceContext) { // NOTE: we will change auto-flush to take into account size of buffered data, so no need to do/test a lot with // current approach final int flushFreq = context.getConfiguration().getInt("c.reducer.flush.freq", 10000); @SuppressWarnings("unchecked") WrappedReducer.Context flushingContext = new WrappedReducer().new Context(context) { private int processedRecords = 0; @Override/*from w w w.j a v a 2 s. c om*/ public boolean nextKeyValue() throws IOException, InterruptedException { boolean result = super.nextKey(); if (++processedRecords > flushFreq) { try { LOG.info("Flushing dataset operations..."); basicMapReduceContext.flushOperations(); } catch (Exception e) { LOG.error("Failed to persist changes", e); throw Throwables.propagate(e); } processedRecords = 0; } return result; } }; return flushingContext; }
From source file:com.asakusafw.runtime.compatibility.hadoop2.JobCompatibilityHadoop2.java
License:Apache License
@Override public <KEYIN, VALUEIN, KEYOUT, VALUEOUT> Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context newReducerContext( Configuration configuration, TaskAttemptID id, RawKeyValueIterator reader, Class<KEYIN> inputKeyClass, Class<VALUEIN> inputValueClass, RecordWriter<KEYOUT, VALUEOUT> writer, OutputCommitter committer, RawComparator<KEYIN> comparator) throws IOException, InterruptedException { StatusReporter reporter = new MockStatusReporter(); ReduceContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT> context = new ReduceContextImpl<>(configuration, id, reader, reporter.getCounter("asakusafw", "inputKey"), //$NON-NLS-1$ //$NON-NLS-2$ reporter.getCounter("asakusafw", "inputValue"), //$NON-NLS-1$ //$NON-NLS-2$ writer, committer, reporter, comparator, inputKeyClass, inputValueClass); return new WrappedReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>().getReducerContext(context); }
From source file:edu.uci.ics.hyracks.dataflow.hadoop.util.MRContextUtil.java
License:Apache License
@SuppressWarnings({ "rawtypes", "unchecked" })
public Reducer.Context createReduceContext(Configuration conf, TaskAttemptID taskid, RawKeyValueIterator input,
Counter inputKeyCounter, Counter inputValueCounter, RecordWriter output, OutputCommitter committer,
StatusReporter reporter, RawComparator comparator, Class keyClass, Class valueClass)
throws HyracksDataException {
try {//from w w w .jav a2s. co m
return new WrappedReducer()
.getReducerContext(new ReduceContextImpl(conf, taskid, input, inputKeyCounter,
inputValueCounter, output, committer, reporter, comparator, keyClass, valueClass));
} catch (Exception e) {
throw new HyracksDataException(e);
}
}
From source file:org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2ReduceTask.java
License:Apache License
/** {@inheritDoc} */ @SuppressWarnings({ "ConstantConditions", "unchecked" }) @Override//from ww w . ja va 2s . co m public void run0(HadoopV2TaskContext taskCtx) throws IgniteCheckedException { OutputFormat outputFormat = null; Exception err = null; JobContextImpl jobCtx = taskCtx.jobContext(); // Set mapper index for combiner tasks if (!reduce && taskCtx.taskInfo().hasMapperIndex()) HadoopMapperUtils.mapperIndex(taskCtx.taskInfo().mapperIndex()); else HadoopMapperUtils.clearMapperIndex(); try { outputFormat = reduce || !taskCtx.job().info().hasReducer() ? prepareWriter(jobCtx) : null; Reducer reducer; if (reduce) reducer = ReflectionUtils.newInstance(jobCtx.getReducerClass(), jobCtx.getConfiguration()); else reducer = ReflectionUtils.newInstance(jobCtx.getCombinerClass(), jobCtx.getConfiguration()); try { reducer.run(new WrappedReducer().getReducerContext(hadoopContext())); if (!reduce) taskCtx.onMapperFinished(); } finally { closeWriter(); } commit(outputFormat); } catch (InterruptedException e) { err = e; Thread.currentThread().interrupt(); throw new IgniteInterruptedCheckedException(e); } catch (Exception e) { err = e; throw new IgniteCheckedException(e); } finally { if (!reduce) HadoopMapperUtils.clearMapperIndex(); if (err != null) abort(outputFormat); } }
From source file:org.apache.ignite.internal.processors.hadoop.v2.GridHadoopV2ReduceTask.java
License:Apache License
/** {@inheritDoc} */ @SuppressWarnings({ "ConstantConditions", "unchecked" }) @Override//from www . j a v a2s .c o m public void run0(GridHadoopV2TaskContext taskCtx) throws IgniteCheckedException { OutputFormat outputFormat = null; Exception err = null; JobContextImpl jobCtx = taskCtx.jobContext(); try { outputFormat = reduce || !taskCtx.job().info().hasReducer() ? prepareWriter(jobCtx) : null; Reducer reducer = ReflectionUtils.newInstance( reduce ? jobCtx.getReducerClass() : jobCtx.getCombinerClass(), jobCtx.getConfiguration()); try { reducer.run(new WrappedReducer().getReducerContext(hadoopContext())); } finally { closeWriter(); } commit(outputFormat); } catch (InterruptedException e) { err = e; Thread.currentThread().interrupt(); throw new IgniteInterruptedCheckedException(e); } catch (Exception e) { err = e; throw new IgniteCheckedException(e); } finally { if (err != null) abort(outputFormat); } }
From source file:org.apache.ignite.internal.processors.hadoop.v2.HadoopV2ReduceTask.java
License:Apache License
/** {@inheritDoc} */ @SuppressWarnings({ "ConstantConditions", "unchecked" }) @Override//from w w w . j a v a 2 s.c om public void run0(HadoopV2TaskContext taskCtx) throws IgniteCheckedException { OutputFormat outputFormat = null; Exception err = null; JobContextImpl jobCtx = taskCtx.jobContext(); try { outputFormat = reduce || !taskCtx.job().info().hasReducer() ? prepareWriter(jobCtx) : null; Reducer reducer; if (reduce) reducer = ReflectionUtils.newInstance(jobCtx.getReducerClass(), jobCtx.getConfiguration()); else reducer = ReflectionUtils.newInstance(jobCtx.getCombinerClass(), jobCtx.getConfiguration()); try { reducer.run(new WrappedReducer().getReducerContext(hadoopContext())); } finally { closeWriter(); } commit(outputFormat); } catch (InterruptedException e) { err = e; Thread.currentThread().interrupt(); throw new IgniteInterruptedCheckedException(e); } catch (Exception e) { err = e; throw new IgniteCheckedException(e); } finally { if (err != null) abort(outputFormat); } }
From source file:org.apache.tez.mapreduce.combine.MRCombiner.java
License:Apache License
private static <KEYIN, VALUEIN, KEYOUT, VALUEOUT> org.apache.hadoop.mapreduce.Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context createReduceContext( Configuration conf, TaskAttemptID mrTaskAttemptID, final TezRawKeyValueIterator rawIter, Counter combineInputKeyCounter, Counter combineInputValueCounter, RecordWriter<KEYOUT, VALUEOUT> recordWriter, MRTaskReporter reporter, RawComparator<KEYIN> comparator, Class<KEYIN> keyClass, Class<VALUEIN> valClass) throws InterruptedException, IOException { RawKeyValueIterator r = new RawKeyValueIterator() { @Override/*from w w w . j a v a 2s .c om*/ public boolean next() throws IOException { return rawIter.next(); } @Override public DataInputBuffer getValue() throws IOException { return rawIter.getValue(); } @Override public Progress getProgress() { return rawIter.getProgress(); } @Override public DataInputBuffer getKey() throws IOException { return rawIter.getKey(); } @Override public void close() throws IOException { rawIter.close(); } }; ReduceContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT> rContext = new ReduceContextImpl<KEYIN, VALUEIN, KEYOUT, VALUEOUT>( conf, mrTaskAttemptID, r, combineInputKeyCounter, combineInputValueCounter, recordWriter, null, reporter, comparator, keyClass, valClass); org.apache.hadoop.mapreduce.Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context reducerContext = new WrappedReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>() .getReducerContext(rContext); return reducerContext; }
From source file:org.apache.tez.mapreduce.processor.MRTask.java
License:Apache License
protected static <INKEY, INVALUE, OUTKEY, OUTVALUE> org.apache.hadoop.mapreduce.Reducer<INKEY, INVALUE, OUTKEY, OUTVALUE>.Context createReduceContext( org.apache.hadoop.mapreduce.Reducer<INKEY, INVALUE, OUTKEY, OUTVALUE> reducer, Configuration job, TaskAttemptID taskId, final TezRawKeyValueIterator rIter, org.apache.hadoop.mapreduce.Counter inputKeyCounter, org.apache.hadoop.mapreduce.Counter inputValueCounter, org.apache.hadoop.mapreduce.RecordWriter<OUTKEY, OUTVALUE> output, org.apache.hadoop.mapreduce.OutputCommitter committer, org.apache.hadoop.mapreduce.StatusReporter reporter, RawComparator<INKEY> comparator, Class<INKEY> keyClass, Class<INVALUE> valueClass) throws IOException, InterruptedException { RawKeyValueIterator r = new RawKeyValueIterator() { @Override//from w w w .j a v a 2 s .c om public boolean next() throws IOException { return rIter.next(); } @Override public DataInputBuffer getValue() throws IOException { return rIter.getValue(); } @Override public Progress getProgress() { return rIter.getProgress(); } @Override public DataInputBuffer getKey() throws IOException { return rIter.getKey(); } @Override public void close() throws IOException { rIter.close(); } }; org.apache.hadoop.mapreduce.ReduceContext<INKEY, INVALUE, OUTKEY, OUTVALUE> reduceContext = new ReduceContextImpl<INKEY, INVALUE, OUTKEY, OUTVALUE>( job, taskId, r, inputKeyCounter, inputValueCounter, output, committer, reporter, comparator, keyClass, valueClass); if (LOG.isDebugEnabled()) { LOG.debug("Using key class: " + keyClass + ", valueClass: " + valueClass); } org.apache.hadoop.mapreduce.Reducer<INKEY, INVALUE, OUTKEY, OUTVALUE>.Context reducerContext = new WrappedReducer<INKEY, INVALUE, OUTKEY, OUTVALUE>() .getReducerContext(reduceContext); return reducerContext; }
From source file:org.gridgain.grid.kernal.processors.hadoop.v2.GridHadoopV2ReduceTask.java
License:Open Source License
/** {@inheritDoc} */ @SuppressWarnings({ "ConstantConditions", "unchecked" }) @Override/*w w w .ja v a 2 s . c o m*/ public void run0(GridHadoopV2TaskContext taskCtx) throws GridException { OutputFormat outputFormat = null; Exception err = null; JobContextImpl jobCtx = taskCtx.jobContext(); try { outputFormat = reduce || !taskCtx.job().info().hasReducer() ? prepareWriter(jobCtx) : null; Reducer reducer = ReflectionUtils.newInstance( reduce ? jobCtx.getReducerClass() : jobCtx.getCombinerClass(), jobCtx.getConfiguration()); try { reducer.run(new WrappedReducer().getReducerContext(hadoopContext())); } finally { closeWriter(); } commit(outputFormat); } catch (InterruptedException e) { err = e; Thread.currentThread().interrupt(); throw new GridInterruptedException(e); } catch (Exception e) { err = e; throw new GridException(e); } finally { if (err != null) abort(outputFormat); } }