Example usage for org.apache.hadoop.mapreduce JobContext getConfiguration

List of usage examples for org.apache.hadoop.mapreduce JobContext getConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. JobContext
  6. getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce JobContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.asakusafw.runtime.stage.input.BridgeInputFormat.java

License:Apache License

private Class<?> extractDataClass(JobContext context, StageInput input) throws IOException {
    assert context != null;
    assert input != null;
    String value = extract(input, DirectDataSourceConstants.KEY_DATA_CLASS);
    try {/*w  ww. ja  v  a  2s  .  c om*/
        return Class.forName(value, false, context.getConfiguration().getClassLoader());
    } catch (ClassNotFoundException e) {
        throw new IOException(
                MessageFormat.format("Invalid data class: \"{1}\" (path={0})", extractBasePath(input), value),
                e);
    }
}

From source file:com.asakusafw.runtime.stage.input.BridgeInputFormat.java

License:Apache License

@SuppressWarnings("unchecked")
private Class<? extends DataFormat<?>> extractFormatClass(JobContext context, StageInput input)
        throws IOException {
    assert context != null;
    assert input != null;
    String value = extract(input, DirectDataSourceConstants.KEY_FORMAT_CLASS);
    try {//from   w  ww  . j  a  v a 2s.com
        Class<?> aClass = Class.forName(value, false, context.getConfiguration().getClassLoader());
        return (Class<? extends DataFormat<?>>) aClass.asSubclass(DataFormat.class);
    } catch (Exception e) {
        throw new IOException(
                MessageFormat.format("Invalid format class: \"{1}\" (path={0})", extractBasePath(input), value),
                e);
    }
}

From source file:com.asakusafw.runtime.stage.input.BridgeInputFormat.java

License:Apache License

@SuppressWarnings("unchecked")
private Class<? extends DataFilter<?>> extractFilterClass(JobContext context, StageInput input)
        throws IOException {
    assert context != null;
    assert input != null;
    String value = input.getAttributes().get(DirectDataSourceConstants.KEY_FILTER_CLASS);
    if (value == null) {
        return null;
    }// w  w  w  . jav  a2 s  .  c  o  m
    try {
        Class<?> aClass = Class.forName(value, false, context.getConfiguration().getClassLoader());
        return (Class<? extends DataFilter<?>>) aClass.asSubclass(DataFilter.class);
    } catch (Exception e) {
        throw new IOException(
                MessageFormat.format("Invalid format class: \"{1}\" (path={0})", extractBasePath(input), value),
                e);
    }
}

From source file:com.asakusafw.runtime.stage.input.BridgeInputFormat.java

License:Apache License

private static DirectDataSourceRepository getDataSourceRepository(JobContext context) {
    assert context != null;
    return HadoopDataSourceUtil.loadRepository(context.getConfiguration());
}

From source file:com.asakusafw.runtime.stage.input.DefaultSplitCombiner.java

License:Apache License

@Override
protected List<StageInputSplit> combine(JobContext context, List<StageInputSplit> splits)
        throws IOException, InterruptedException {
    Configuration conf = new Configuration();
    conf.withSlotsPerInput(getMaxSplitsPerMapper(context));
    conf.withPopulations(context.getConfiguration().getInt(KEY_POPULATIONS, DEFAULT_POPULATIONS));
    conf.withGenerations(context.getConfiguration().getInt(KEY_GENERATIONS, DEFAULT_GENERATIONS));
    conf.withMutations(context.getConfiguration().getFloat(KEY_MUTATION_RATIO, DEFAULT_MUTATION_RATIO));
    conf.withTinyLimit(context.getConfiguration().getLong(KEY_TINY_LIMIT, DEFAULT_TINY_LIMIT));
    conf.withAverageTimeWeight(/*from  w  ww  .  j a  va 2  s. com*/
            context.getConfiguration().getFloat(KEY_AVERAGE_TIME_WEIGHT, DEFAULT_AVERAGE_TIME_WEIGHT));
    conf.withNonLocalPenaltyRatio(
            context.getConfiguration().getFloat(KEY_NON_LOCAL_PENALTY_RATIO, DEFAULT_NON_LOCAL_PENALTY_RATIO));
    conf.withInitialLocalityRatio(
            context.getConfiguration().getFloat(KEY_INITIAL_LOCALITY_RATIO, DEFAULT_INITIAL_LOCALITY_RATIO));
    return combine(conf, splits);
}

From source file:com.asakusafw.runtime.stage.input.DefaultSplitCombiner.java

License:Apache License

private int getMaxSplitsPerMapper(JobContext context) {
    assert context != null;
    int max = context.getConfiguration().getInt(KEY_SLOTS_PER_INPUT, -1);
    if (max > 0) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("Combine split configuration: {0}={1}", //$NON-NLS-1$
                    KEY_SLOTS_PER_INPUT, max));
        }//ww  w. j a v  a  2 s .  c  o  m
        return max;
    }
    return DEFAULT_SLOTS_PER_INPUT;
}

From source file:com.asakusafw.runtime.stage.input.StageInputDriver.java

License:Apache License

/**
 * Returns the estimated input data-size.
 * @param context the current job context
 * @return the estimated input data-size in bytes
 * @throws InterruptedException if interrupted while
 * @throws IllegalArgumentException if some parameters were {@code null}
 * @since 0.6.0/*from  w w w.ja  v  a2s  . com*/
 */
public static long estimateInputSize(JobContext context) throws InterruptedException {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    long cached = context.getConfiguration().getLong(KEY_SIZE_CACHE, Integer.MIN_VALUE);
    if (cached != Integer.MIN_VALUE) {
        return cached;
    }
    long result = estimateInputSize0(context);
    if (result >= 0) {
        context.getConfiguration().setLong(KEY_SIZE_CACHE, result);
    }
    return result;
}

From source file:com.asakusafw.runtime.stage.input.StageInputFormat.java

License:Apache License

static List<StageInputSplit> computeSplits(JobContext context) throws IOException, InterruptedException {
    assert context != null;
    List<StageInput> inputs = StageInputDriver.getInputs(context.getConfiguration());
    List<StageInputSplit> cached = Cache.find(context, inputs);
    if (cached != null) {
        return cached;
    }//from  w ww . ja va2 s .c  om
    Map<FormatAndMapper, List<StageInput>> paths = groupByFormatAndMapper(inputs);
    Map<Class<? extends InputFormat<?, ?>>, InputFormat<?, ?>> formats = instantiateFormats(context,
            paths.keySet());
    Job temporaryJob = JobCompatibility.newJob(context.getConfiguration());
    List<StageInputSplit> results = new ArrayList<>();
    for (Map.Entry<FormatAndMapper, List<StageInput>> entry : paths.entrySet()) {
        FormatAndMapper formatAndMapper = entry.getKey();
        List<StageInput> current = entry.getValue();
        InputFormat<?, ?> format = formats.get(formatAndMapper.formatClass);
        List<? extends InputSplit> splits;
        if (format instanceof FileInputFormat<?, ?>) {
            FileInputFormat.setInputPaths(temporaryJob, toPathArray(current));
            splits = format.getSplits(temporaryJob);
        } else if (format instanceof BridgeInputFormat) {
            splits = ((BridgeInputFormat) format).getSplits(context, current);
        } else if (format instanceof TemporaryInputFormat<?>) {
            splits = ((TemporaryInputFormat<?>) format).getSplits(context, current);
        } else {
            splits = format.getSplits(temporaryJob);
        }
        assert format != null : formatAndMapper.formatClass.getName();
        Class<? extends Mapper<?, ?, ?, ?>> mapper = formatAndMapper.mapperClass;
        for (InputSplit split : splits) {
            Source source = new Source(split, formatAndMapper.formatClass);
            StageInputSplit wrapped = new StageInputSplit(mapper, Collections.singletonList(source));
            wrapped.setConf(context.getConfiguration());
            results.add(wrapped);
        }
    }
    Cache.put(context, inputs, results);
    return results;
}

From source file:com.asakusafw.runtime.stage.input.StageInputFormat.java

License:Apache License

private static SplitCombiner getSplitCombiner(JobContext context) {
    assert context != null;
    Class<? extends SplitCombiner> combinerClass = getSplitCombinerClass(context);
    return ReflectionUtils.newInstance(combinerClass, context.getConfiguration());
}

From source file:com.asakusafw.runtime.stage.input.StageInputFormat.java

License:Apache License

/**
 * Returns the {@link SplitCombiner} class used in the current job.
 * @param context the current job context
 * @return the {@link SplitCombiner} class
 * @since 0.7.1//from   www  .  j  a  va 2 s .co  m
 */
public static Class<? extends SplitCombiner> getSplitCombinerClass(JobContext context) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    Configuration conf = context.getConfiguration();
    String combinerType = conf.get(KEY_SPLIT_COMBINER, DEFAULT_SPLIT_COMBINER);
    if (JobCompatibility.isLocalMode(context) && combinerType.equals(DEFAULT_SPLIT_COMBINER)) {
        return ExtremeSplitCombiner.class;
    }
    Class<? extends SplitCombiner> defined = SPLIT_COMBINERS.get(combinerType);
    if (defined != null) {
        return defined;
    }
    try {
        return conf.getClassByName(combinerType).asSubclass(SplitCombiner.class);
    } catch (Exception e) {
        LOG.warn(MessageFormat.format("failed to load a combiner \"{0}\"", combinerType), e);
        return IdentitySplitCombiner.class;
    }
}