Example usage for org.apache.hadoop.mapreduce JobContext getConfiguration

List of usage examples for org.apache.hadoop.mapreduce JobContext getConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. JobContext
  6. getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce JobContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.asakusafw.runtime.stage.output.TemporaryOutputFormat.java

License:Apache License

/**
 * Configures output path./*w  ww.java2 s  .c  o  m*/
 * @param context current context
 * @param path target output path
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
public static void setOutputPath(JobContext context, Path path) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    if (path == null) {
        throw new IllegalArgumentException("path must not be null"); //$NON-NLS-1$
    }
    context.getConfiguration().set(KEY_OUTPUT_PATH, path.toString());
}

From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java

License:Apache License

/**
 * Returns the estimated resource data-size.
 * @param context the current job context
 * @return the estimated resource data-size in bytes
 * @throws InterruptedException if interrupted while
 * @throws IllegalArgumentException if some parameters were {@code null}
 * @since 0.7.1//from   w ww  . ja va  2 s  .c  o m
 */
public static long estimateResourceSize(JobContext context) throws InterruptedException {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    return context.getConfiguration().getLong(KEY_SIZE, 0L);
}

From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java

License:Apache License

/**
 * Returns the access mode for stage resources in the job.
 * @param context the current job context
 * @return the access mode/*from w ww.j a v a 2 s.  c o m*/
 * @since 0.7.1
 */
public static AccessMode getAccessMode(JobContext context) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    return AccessMode.decode(context.getConfiguration().get(KEY_ACCESS_MODE));
}

From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java

License:Apache License

/**
 * Sets the access mode for stage resources in the job.
 * @param context the current job context
 * @param mode the access mode/*from w  w w  .j  a v  a2s  .  c  o m*/
 * @since 0.7.1
 */
public static void setAccessMode(JobContext context, AccessMode mode) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    if (mode == null) {
        throw new IllegalArgumentException("mode must not be null"); //$NON-NLS-1$
    }
    context.getConfiguration().set(KEY_ACCESS_MODE, mode.encode());
}

From source file:com.asakusafw.runtime.stage.StageUtil.java

License:Apache License

/**
 * Returns whether the current job is on the local mode or not.
 * @param context the current context/*  ww  w  .  j a v a  2 s . co m*/
 * @return {@code true} if the target job is running on the local mode, otherwise {@code false}
 * @throws IllegalArgumentException if some parameters were {@code null}
 * @since 0.6.2
 */
public static boolean isLocalMode(JobContext context) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("{0}={1}", //$NON-NLS-1$
                MRConfig.FRAMEWORK_NAME, context.getConfiguration().get(MRConfig.FRAMEWORK_NAME)));
    }
    String name = context.getConfiguration().get(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
    return name.equals(MRConfig.LOCAL_FRAMEWORK_NAME);
}

From source file:com.avira.couchdoop.imp.CouchbaseViewInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException {
    List<InputSplit> inputSplits = new ArrayList<InputSplit>();

    ImportViewArgs importViewArgs;//from  www. jav a2s .c  o m
    try {
        importViewArgs = new ImportViewArgs(jobContext.getConfiguration());
    } catch (ArgsException e) {
        throw new RuntimeException("ImportViewArgs can't load settings from Hadoop Configuration");
    }
    String[] viewKeys = importViewArgs.getViewKeys();
    int viewKeysPerMapTask = (int) Math.ceil((double) viewKeys.length / importViewArgs.getNumMappers());

    LOGGER.info("Number of keys per map task is {} ({} / {})", viewKeysPerMapTask, viewKeys.length,
            importViewArgs.getNumMappers());

    CouchbaseViewInputSplit inputSplit = new CouchbaseViewInputSplit();
    int keysInCurrentSplit = 0;

    for (String viewKey : viewKeys) {
        //If we have enough keys, add the split and create a new one
        if (keysInCurrentSplit == viewKeysPerMapTask) {
            inputSplits.add(inputSplit);
            inputSplit = new CouchbaseViewInputSplit();
            keysInCurrentSplit = 0;
        }
        inputSplit.addKey(viewKey);
        keysInCurrentSplit++;
    }

    //Also add last inputSplit
    inputSplits.add(inputSplit);

    return inputSplits;
}

From source file:com.basho.riak.hadoop.RiakInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    RiakLocation[] locations = RiakConfig.getRiakLocatons(conf);

    if (locations.length == 0) {
        throw new NoRiakLocationsException();
    }/*from ww w. j a v a2  s.co m*/

    final KeyLister keyLister = RiakConfig.getKeyLister(conf);

    try {
        List<BucketKey> keys = getKeys(locations, keyLister, 0);
        List<InputSplit> splits = getSplits(keys, locations,
                getSplitSize(keys.size(), RiakConfig.getHadoopClusterSize(conf, 3)));
        return splits;
    } catch (RiakException e) {
        throw new IOException(e);
    }
}

From source file:com.baynote.kafka.hadoop.KafkaInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    final Configuration conf = context.getConfiguration();
    final String topic = getTopic(conf);
    final String group = getConsumerGroup(conf);
    return getInputSplits(conf, topic, group);
}

From source file:com.baynote.kafka.hadoop.MultipleKafkaInputFormat.java

License:Apache License

/**
 * Creates input splits for each {@link TopicConf} set up by {@link #addTopic(Job, String, String, Class)}.
 * /*ww w  .j  av  a 2  s. c  o m*/
 * <p>
 * {@inheritDoc}
 */
@Override
public List<InputSplit> getSplits(final JobContext context) throws IOException, InterruptedException {
    final Configuration conf = context.getConfiguration();
    final List<InputSplit> splits = Lists.newArrayList();
    final List<TopicConf> topicConfs = getTopics(conf);
    warnOnDuplicateTopicConsumers(topicConfs);
    for (final TopicConf topicConf : topicConfs) {
        final String topic = topicConf.getTopic();
        final String group = topicConf.getConsumerGroup();
        final Class<? extends Mapper> delegateMapper = topicConf.getMapper();
        for (final InputSplit inputSplit : getInputSplits(conf, group, topic)) {
            splits.add(new TaggedInputSplit(inputSplit, conf, KafkaInputFormat.class, delegateMapper));
        }
    }
    return splits;
}

From source file:com.blackberry.logdriver.mapreduce.boom.BoomInputFormat.java

License:Apache License

@SuppressWarnings("deprecation")
public List<InputSplit> getSplits(JobContext context) throws IOException {
    Configuration conf = context.getConfiguration();
    // Ensure we have sensible defaults for how we build blocks.
    if (conf.get("mapreduce.job.max.split.locations") == null) {
        conf.setLong("mapreduce.job.max.split.locations", MAX_SPLIT_LOCATIONS);
    }/*from   ww  w. j  av a2 s  .c  o  m*/
    if (conf.get("mapred.max.split.size") == null) {
        // Try to set the split size to the default block size. In case of
        // failure, we'll use this 128MB default.
        long blockSize = 128 * 1024 * 1024; // 128MB
        try {
            blockSize = FileSystem.get(conf).getDefaultBlockSize();
        } catch (IOException e) {
            LOG.error("Error getting filesystem to get get default block size (this does not bode well).");
        }
        conf.setLong("mapred.max.split.size", blockSize);
    }
    for (String key : new String[] { "mapreduce.job.max.split.locations", "mapred.max.split.size" }) {
        LOG.info("{} = {}", key, context.getConfiguration().get(key));
    }

    return super.getSplits(context);
}