org.apache.hadoop.mapred.ShuffleConsumerPlugin.java Source code

Introduction

Here is the source code for org.apache.hadoop.mapred.ShuffleConsumerPlugin.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.mapred;

import java.io.IOException;
import java.util.Map;

import org.apache.hadoop.mapred.Task.CombineOutputCollector;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

/**
 * ShuffleConsumerPlugin for serving Reducers.  It may shuffle MOF files from
 * either the built-in ShuffleHandler or from a 3rd party AuxiliaryService.
 *
 */
@InterfaceAudience.LimitedPrivate("mapreduce")
@InterfaceStability.Unstable
public interface ShuffleConsumerPlugin<K, V> {

    public void init(Context<K, V> context);

    public RawKeyValueIterator run() throws IOException, InterruptedException;

    public void close();

    @InterfaceAudience.LimitedPrivate("mapreduce")
    @InterfaceStability.Unstable
    public static class Context<K, V> {
        private final org.apache.hadoop.mapreduce.TaskAttemptID reduceId;
        private final JobConf jobConf;
        private final FileSystem localFS;
        private final TaskUmbilicalProtocol umbilical;
        private final LocalDirAllocator localDirAllocator;
        private final Reporter reporter;
        private final CompressionCodec codec;
        private final Class<? extends Reducer> combinerClass;
        private final CombineOutputCollector<K, V> combineCollector;
        private final Counters.Counter spilledRecordsCounter;
        private final Counters.Counter reduceCombineInputCounter;
        private final Counters.Counter shuffledMapsCounter;
        private final Counters.Counter reduceShuffleBytes;
        private final Counters.Counter failedShuffleCounter;
        private final Counters.Counter mergedMapOutputsCounter;
        private final TaskStatus status;
        private final Progress copyPhase;
        private final Progress mergePhase;
        private final Task reduceTask;
        private final MapOutputFile mapOutputFile;
        private final Map<TaskAttemptID, MapOutputFile> localMapFiles;

        public Context(org.apache.hadoop.mapreduce.TaskAttemptID reduceId, JobConf jobConf, FileSystem localFS,
                TaskUmbilicalProtocol umbilical, LocalDirAllocator localDirAllocator, Reporter reporter,
                CompressionCodec codec, Class<? extends Reducer> combinerClass,
                CombineOutputCollector<K, V> combineCollector, Counters.Counter spilledRecordsCounter,
                Counters.Counter reduceCombineInputCounter, Counters.Counter shuffledMapsCounter,
                Counters.Counter reduceShuffleBytes, Counters.Counter failedShuffleCounter,
                Counters.Counter mergedMapOutputsCounter, TaskStatus status, Progress copyPhase,
                Progress mergePhase, Task reduceTask, MapOutputFile mapOutputFile,
                Map<TaskAttemptID, MapOutputFile> localMapFiles) {
            this.reduceId = reduceId;
            this.jobConf = jobConf;
            this.localFS = localFS;
            this.umbilical = umbilical;
            this.localDirAllocator = localDirAllocator;
            this.reporter = reporter;
            this.codec = codec;
            this.combinerClass = combinerClass;
            this.combineCollector = combineCollector;
            this.spilledRecordsCounter = spilledRecordsCounter;
            this.reduceCombineInputCounter = reduceCombineInputCounter;
            this.shuffledMapsCounter = shuffledMapsCounter;
            this.reduceShuffleBytes = reduceShuffleBytes;
            this.failedShuffleCounter = failedShuffleCounter;
            this.mergedMapOutputsCounter = mergedMapOutputsCounter;
            this.status = status;
            this.copyPhase = copyPhase;
            this.mergePhase = mergePhase;
            this.reduceTask = reduceTask;
            this.mapOutputFile = mapOutputFile;
            this.localMapFiles = localMapFiles;
        }

        public org.apache.hadoop.mapreduce.TaskAttemptID getReduceId() {
            return reduceId;
        }

        public JobConf getJobConf() {
            return jobConf;
        }

        public FileSystem getLocalFS() {
            return localFS;
        }

        public TaskUmbilicalProtocol getUmbilical() {
            return umbilical;
        }

        public LocalDirAllocator getLocalDirAllocator() {
            return localDirAllocator;
        }

        public Reporter getReporter() {
            return reporter;
        }

        public CompressionCodec getCodec() {
            return codec;
        }

        public Class<? extends Reducer> getCombinerClass() {
            return combinerClass;
        }

        public CombineOutputCollector<K, V> getCombineCollector() {
            return combineCollector;
        }

        public Counters.Counter getSpilledRecordsCounter() {
            return spilledRecordsCounter;
        }

        public Counters.Counter getReduceCombineInputCounter() {
            return reduceCombineInputCounter;
        }

        public Counters.Counter getShuffledMapsCounter() {
            return shuffledMapsCounter;
        }

        public Counters.Counter getReduceShuffleBytes() {
            return reduceShuffleBytes;
        }

        public Counters.Counter getFailedShuffleCounter() {
            return failedShuffleCounter;
        }

        public Counters.Counter getMergedMapOutputsCounter() {
            return mergedMapOutputsCounter;
        }

        public TaskStatus getStatus() {
            return status;
        }

        public Progress getCopyPhase() {
            return copyPhase;
        }

        public Progress getMergePhase() {
            return mergePhase;
        }

        public Task getReduceTask() {
            return reduceTask;
        }

        public MapOutputFile getMapOutputFile() {
            return mapOutputFile;
        }

        public Map<TaskAttemptID, MapOutputFile> getLocalMapFiles() {
            return localMapFiles;
        }
    } // end of public static class Context<K,V>

}