Example usage for org.apache.hadoop.mapreduce.lib.map WrappedMapper WrappedMapper

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.map WrappedMapper WrappedMapper.

Prototype

WrappedMapper

Source Link

Usage

From source file:co.cask.cdap.internal.app.runtime.batch.MapperWrapper.java

License:Apache License

private WrappedMapper.Context createAutoFlushingContext(final Context context,
        final BasicMapReduceTaskContext basicMapReduceContext) {
    // NOTE: we will change auto-flush to take into account size of buffered data, so no need to do/test a lot with
    //       current approach
    final int flushFreq = context.getConfiguration().getInt("c.mapper.flush.freq", 10000);

    @SuppressWarnings("unchecked")
    WrappedMapper.Context flushingContext = new WrappedMapper().new Context(context) {
        private int processedRecords = 0;

        @Override/*from ww w. j  a  v a  2  s.co  m*/
        public boolean nextKeyValue() throws IOException, InterruptedException {
            boolean result = super.nextKeyValue();
            if (++processedRecords > flushFreq) {
                try {
                    LOG.info("Flushing dataset operations...");
                    basicMapReduceContext.flushOperations();
                } catch (Exception e) {
                    LOG.error("Failed to persist changes", e);
                    throw Throwables.propagate(e);
                }
                processedRecords = 0;
            }
            return result;
        }

        @Override
        public InputSplit getInputSplit() {
            InputSplit inputSplit = super.getInputSplit();
            if (inputSplit instanceof TaggedInputSplit) {
                // expose the delegate InputSplit to the user
                inputSplit = ((TaggedInputSplit) inputSplit).getInputSplit();
            }
            return inputSplit;
        }

        @Override
        public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
            InputSplit inputSplit = super.getInputSplit();
            if (inputSplit instanceof TaggedInputSplit) {
                // expose the delegate InputFormat to the user
                return ((TaggedInputSplit) inputSplit).getInputFormatClass();
            }
            return super.getInputFormatClass();
        }
    };
    return flushingContext;
}

---

From source file:com.asakusafw.runtime.compatibility.hadoop2.JobCompatibilityHadoop2.java

License:Apache License

@Override
public <KEYIN, VALUEIN, KEYOUT, VALUEOUT> Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context newMapperContext(
        Configuration configuration, TaskAttemptID id, RecordReader<KEYIN, VALUEIN> reader,
        RecordWriter<KEYOUT, VALUEOUT> writer, OutputCommitter committer, InputSplit split)
        throws IOException, InterruptedException {
    MapContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT> context = new MapContextImpl<>(configuration, id, reader,
            writer, committer, new MockStatusReporter(), split);
    return new WrappedMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>().getMapContext(context);
}

---

From source file:com.kylinolap.job.hadoop.cube.MockupMapContext.java

License:Apache License

public static Context create(final Configuration hconf, String metadataUrl, String cubeName,
        final Object[] outKV) {

    hconf.set(BatchConstants.CFG_CUBE_NAME, cubeName);

    return new WrappedMapper().getMapContext(new MapContext() {

        @Override/*from ww  w .j ava 2  s.  c  o m*/
        public boolean nextKeyValue() throws IOException, InterruptedException {
            throw new NotImplementedException();
        }

        @Override
        public Object getCurrentKey() throws IOException, InterruptedException {
            throw new NotImplementedException();
        }

        @Override
        public Object getCurrentValue() throws IOException, InterruptedException {
            throw new NotImplementedException();
        }

        @Override
        public void write(Object key, Object value) throws IOException, InterruptedException {
            System.out.println("Write -- k:" + key + ", v:" + value);
            if (outKV != null) {
                outKV[0] = key;
                outKV[1] = value;
            }
        }

        @Override
        public OutputCommitter getOutputCommitter() {
            throw new NotImplementedException();
        }

        @Override
        public TaskAttemptID getTaskAttemptID() {
            throw new NotImplementedException();
        }

        @Override
        public void setStatus(String msg) {
            throw new NotImplementedException();
        }

        @Override
        public String getStatus() {
            throw new NotImplementedException();
        }

        @Override
        public float getProgress() {
            throw new NotImplementedException();
        }

        @Override
        public Counter getCounter(Enum<?> counterName) {
            throw new NotImplementedException();
        }

        @Override
        public Counter getCounter(String groupName, String counterName) {
            throw new NotImplementedException();
        }

        @Override
        public Configuration getConfiguration() {
            return hconf;
        }

        @Override
        public Credentials getCredentials() {
            throw new NotImplementedException();
        }

        @Override
        public JobID getJobID() {
            throw new NotImplementedException();
        }

        @Override
        public int getNumReduceTasks() {
            throw new NotImplementedException();
        }

        @Override
        public Path getWorkingDirectory() throws IOException {
            throw new NotImplementedException();
        }

        @Override
        public Class<?> getOutputKeyClass() {
            throw new NotImplementedException();
        }

        @Override
        public Class<?> getOutputValueClass() {
            throw new NotImplementedException();
        }

        @Override
        public Class<?> getMapOutputKeyClass() {
            throw new NotImplementedException();
        }

        @Override
        public Class<?> getMapOutputValueClass() {
            throw new NotImplementedException();
        }

        @Override
        public String getJobName() {
            throw new NotImplementedException();
        }

        @Override
        public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
            throw new NotImplementedException();
        }

        @Override
        public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() throws ClassNotFoundException {
            throw new NotImplementedException();
        }

        @Override
        public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() throws ClassNotFoundException {
            throw new NotImplementedException();
        }

        @Override
        public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() throws ClassNotFoundException {
            throw new NotImplementedException();
        }

        @Override
        public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() throws ClassNotFoundException {
            throw new NotImplementedException();
        }

        @Override
        public Class<? extends Partitioner<?, ?>> getPartitionerClass() throws ClassNotFoundException {
            throw new NotImplementedException();
        }

        @Override
        public RawComparator<?> getSortComparator() {
            throw new NotImplementedException();
        }

        @Override
        public String getJar() {
            throw new NotImplementedException();
        }

        @Override
        public RawComparator<?> getGroupingComparator() {
            throw new NotImplementedException();
        }

        @Override
        public boolean getJobSetupCleanupNeeded() {
            throw new NotImplementedException();
        }

        @Override
        public boolean getTaskCleanupNeeded() {
            throw new NotImplementedException();
        }

        @Override
        public boolean getProfileEnabled() {
            throw new NotImplementedException();
        }

        @Override
        public String getProfileParams() {
            throw new NotImplementedException();
        }

        @Override
        public IntegerRanges getProfileTaskRange(boolean isMap) {
            throw new NotImplementedException();
        }

        @Override
        public String getUser() {
            throw new NotImplementedException();
        }

        @Override
        public boolean getSymlink() {
            throw new NotImplementedException();
        }

        @Override
        public Path[] getArchiveClassPaths() {
            throw new NotImplementedException();
        }

        @Override
        public URI[] getCacheArchives() throws IOException {
            throw new NotImplementedException();
        }

        @Override
        public URI[] getCacheFiles() throws IOException {
            throw new NotImplementedException();
        }

        @Override
        public Path[] getLocalCacheArchives() throws IOException {
            throw new NotImplementedException();
        }

        @Override
        public Path[] getLocalCacheFiles() throws IOException {
            throw new NotImplementedException();
        }

        @Override
        public Path[] getFileClassPaths() {
            throw new NotImplementedException();
        }

        @Override
        public String[] getArchiveTimestamps() {
            throw new NotImplementedException();
        }

        @Override
        public String[] getFileTimestamps() {
            throw new NotImplementedException();
        }

        @Override
        public int getMaxMapAttempts() {
            throw new NotImplementedException();
        }

        @Override
        public int getMaxReduceAttempts() {
            throw new NotImplementedException();
        }

        @Override
        public void progress() {
            throw new NotImplementedException();
        }

        @Override
        public InputSplit getInputSplit() {
            throw new NotImplementedException();
        }

        @Override
        public RawComparator<?> getCombinerKeyGroupingComparator() {
            throw new NotImplementedException();
        }
    });
}

---

From source file:com.moz.fiji.mapreduce.platform.CDH5FijiMRBridge.java

License:Apache License

/** {@inheritDoc} */
@Override/*from   w  w w.  j a v  a2  s.c o m*/
public <KEYIN, VALUEIN, KEYOUT, VALUEOUT> Context getMapperContext(final Configuration conf,
        final TaskAttemptID taskId, final RecordReader<KEYIN, VALUEIN> reader,
        final RecordWriter<KEYOUT, VALUEOUT> writer, final OutputCommitter committer,
        final StatusReporter reporter, final InputSplit split) throws IOException, InterruptedException {
    MapContext<KEYIN, VALUEIN, KEYOUT, VALUEOUT> mapContext = new MapContextImpl<KEYIN, VALUEIN, KEYOUT, VALUEOUT>(
            conf, taskId, reader, writer, committer, reporter, split);
    return new WrappedMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>().getMapContext(mapContext);
}

---

From source file:edu.uci.ics.hyracks.dataflow.hadoop.util.MRContextUtil.java

License:Apache License

@SuppressWarnings({ "rawtypes", "unchecked" })
public Mapper.Context createMapContext(Configuration conf, TaskAttemptID taskid, RecordReader reader,
        RecordWriter writer, OutputCommitter committer, StatusReporter reporter, InputSplit split) {
    return new WrappedMapper()
            .getMapContext(new MapContextImpl(conf, taskid, reader, writer, committer, reporter, split));
}

---

From source file:org.apache.giraph.yarn.GiraphYarnTask.java

License:Apache License

/**
 * Utility to generate dummy Mapper#Context for use in Giraph internals.
 * This is the "key hack" to inject MapReduce-related data structures
 * containing YARN cluster metadata (and our GiraphConf from the AppMaster)
 * into our Giraph BSP task code.//w  w w .  j  av a  2 s .  c  o m
 * @param tid the TaskAttemptID to construct this Mapper#Context from.
 * @return sort of a Mapper#Context if you squint just right.
 */
private Context buildProxyMapperContext(final TaskAttemptID tid) {
    MapContext mc = new MapContextImpl<Object, Object, Object, Object>(conf, // our Configuration, populated back at the GiraphYarnClient.
            tid, // our TaskAttemptId, generated w/YARN app, container, attempt IDs
            null, // RecordReader here will never be used by Giraph
            null, // RecordWriter here will never be used by Giraph
            null, // OutputCommitter here will never be used by Giraph
            new TaskAttemptContextImpl.DummyReporter() { // goes in task logs for now
                @Override
                public void setStatus(String msg) {
                    LOG.info("[STATUS: task-" + bspTaskId + "] " + msg);
                }
            }, null); // Input split setting here will never be used by Giraph

    // now, we wrap our MapContext ref so we can produce a Mapper#Context
    WrappedMapper<Object, Object, Object, Object> wrappedMapper = new WrappedMapper<Object, Object, Object, Object>();
    return wrappedMapper.getMapContext(mc);
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2MapTask.java

License:Apache License

/** {@inheritDoc} */
@SuppressWarnings({ "ConstantConditions", "unchecked" })
@Override//from   w  ww. j  a  va2  s  . co  m
public void run0(HadoopV2TaskContext taskCtx) throws IgniteCheckedException {
    OutputFormat outputFormat = null;
    Exception err = null;

    JobContextImpl jobCtx = taskCtx.jobContext();

    if (taskCtx.taskInfo().hasMapperIndex())
        HadoopMapperUtils.mapperIndex(taskCtx.taskInfo().mapperIndex());
    else
        HadoopMapperUtils.clearMapperIndex();

    try {
        HadoopV2Context hadoopCtx = hadoopContext();

        InputSplit nativeSplit = hadoopCtx.getInputSplit();

        if (nativeSplit == null)
            throw new IgniteCheckedException("Input split cannot be null.");

        InputFormat inFormat = ReflectionUtils.newInstance(jobCtx.getInputFormatClass(),
                hadoopCtx.getConfiguration());

        RecordReader reader = inFormat.createRecordReader(nativeSplit, hadoopCtx);

        reader.initialize(nativeSplit, hadoopCtx);

        hadoopCtx.reader(reader);

        HadoopJobInfo jobInfo = taskCtx.job().info();

        outputFormat = jobInfo.hasCombiner() || jobInfo.hasReducer() ? null : prepareWriter(jobCtx);

        Mapper mapper = ReflectionUtils.newInstance(jobCtx.getMapperClass(), hadoopCtx.getConfiguration());

        try {
            mapper.run(new WrappedMapper().getMapContext(hadoopCtx));

            taskCtx.onMapperFinished();
        } finally {
            closeWriter();
        }

        commit(outputFormat);
    } catch (InterruptedException e) {
        err = e;

        Thread.currentThread().interrupt();

        throw new IgniteInterruptedCheckedException(e);
    } catch (Exception e) {
        err = e;

        throw new IgniteCheckedException(e);
    } finally {
        HadoopMapperUtils.clearMapperIndex();

        if (err != null)
            abort(outputFormat);
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.v2.GridHadoopV2MapTask.java

License:Apache License

/** {@inheritDoc} */
@SuppressWarnings({ "ConstantConditions", "unchecked" })
@Override//from   w  ww . ja  va2  s .c om
public void run0(GridHadoopV2TaskContext taskCtx) throws IgniteCheckedException {
    GridHadoopInputSplit split = info().inputSplit();

    InputSplit nativeSplit;

    if (split instanceof GridHadoopFileBlock) {
        GridHadoopFileBlock block = (GridHadoopFileBlock) split;

        nativeSplit = new FileSplit(new Path(block.file().toString()), block.start(), block.length(), null);
    } else
        nativeSplit = (InputSplit) taskCtx.getNativeSplit(split);

    assert nativeSplit != null;

    OutputFormat outputFormat = null;
    Exception err = null;

    JobContextImpl jobCtx = taskCtx.jobContext();

    try {
        InputFormat inFormat = ReflectionUtils.newInstance(jobCtx.getInputFormatClass(),
                hadoopContext().getConfiguration());

        RecordReader reader = inFormat.createRecordReader(nativeSplit, hadoopContext());

        reader.initialize(nativeSplit, hadoopContext());

        hadoopContext().reader(reader);

        GridHadoopJobInfo jobInfo = taskCtx.job().info();

        outputFormat = jobInfo.hasCombiner() || jobInfo.hasReducer() ? null : prepareWriter(jobCtx);

        Mapper mapper = ReflectionUtils.newInstance(jobCtx.getMapperClass(),
                hadoopContext().getConfiguration());

        try {
            mapper.run(new WrappedMapper().getMapContext(hadoopContext()));
        } finally {
            closeWriter();
        }

        commit(outputFormat);
    } catch (InterruptedException e) {
        err = e;

        Thread.currentThread().interrupt();

        throw new IgniteInterruptedCheckedException(e);
    } catch (Exception e) {
        err = e;

        throw new IgniteCheckedException(e);
    } finally {
        if (err != null)
            abort(outputFormat);
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.v2.HadoopV2MapTask.java

License:Apache License

/** {@inheritDoc} */
@SuppressWarnings({ "ConstantConditions", "unchecked" })
@Override/*w w w .  j  ava2 s  .co m*/
public void run0(HadoopV2TaskContext taskCtx) throws IgniteCheckedException {
    HadoopInputSplit split = info().inputSplit();

    InputSplit nativeSplit;

    if (split instanceof HadoopFileBlock) {
        HadoopFileBlock block = (HadoopFileBlock) split;

        nativeSplit = new FileSplit(new Path(block.file().toString()), block.start(), block.length(), null);
    } else
        nativeSplit = (InputSplit) taskCtx.getNativeSplit(split);

    assert nativeSplit != null;

    OutputFormat outputFormat = null;
    Exception err = null;

    JobContextImpl jobCtx = taskCtx.jobContext();

    try {
        InputFormat inFormat = ReflectionUtils.newInstance(jobCtx.getInputFormatClass(),
                hadoopContext().getConfiguration());

        RecordReader reader = inFormat.createRecordReader(nativeSplit, hadoopContext());

        reader.initialize(nativeSplit, hadoopContext());

        hadoopContext().reader(reader);

        HadoopJobInfo jobInfo = taskCtx.job().info();

        outputFormat = jobInfo.hasCombiner() || jobInfo.hasReducer() ? null : prepareWriter(jobCtx);

        Mapper mapper = ReflectionUtils.newInstance(jobCtx.getMapperClass(),
                hadoopContext().getConfiguration());

        try {
            mapper.run(new WrappedMapper().getMapContext(hadoopContext()));
        } finally {
            closeWriter();
        }

        commit(outputFormat);
    } catch (InterruptedException e) {
        err = e;

        Thread.currentThread().interrupt();

        throw new IgniteInterruptedCheckedException(e);
    } catch (Exception e) {
        err = e;

        throw new IgniteCheckedException(e);
    } finally {
        if (err != null)
            abort(outputFormat);
    }
}

---

From source file:org.apache.kylin.engine.mr.steps.MockupMapContext.java

License:Apache License

public static Context create(final Configuration hconf, String metadataUrl, String cubeName,
        final Object[] outKV) {

    hconf.set(BatchConstants.CFG_CUBE_NAME, cubeName);

    return new WrappedMapper().getMapContext(new MockupMapContext(hconf, outKV));
}

---

• «
• 1
• 2
• »