Example usage for org.apache.hadoop.mapreduce Job getConfiguration

List of usage examples for org.apache.hadoop.mapreduce Job getConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job getConfiguration.

Prototype

public Configuration getConfiguration()

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.asakusafw.runtime.stage.configurator.AutoLocalStageConfigurator.java

License:Apache License

@Override
public void configure(Job job) throws IOException, InterruptedException {
    if (isLocal(job)) {
        return;//from  w  w w .j  a va  2  s . c  om
    }
    Configuration conf = job.getConfiguration();
    long limit = conf.getLong(KEY_LIMIT, -1L);
    if (limit < 0L) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("Auto stage localize is disabled: {0}", job.getJobName()));
        }
        return;
    }
    if (isProtected(job)) {
        return;
    }
    long estimated = StageInputDriver.estimateInputSize(job);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Auto stage localize: job={0}, limit={1}, estimated={2}",
                job.getJobName(), limit, estimated));
    }
    if (estimated < 0L || estimated > limit) {
        return;
    }

    LOG.info(MessageFormat.format("The job \"{0}\" will run in local mode", job.getJobName()));

    localize(job);
}

From source file:com.asakusafw.runtime.stage.configurator.AutoLocalStageConfigurator.java

License:Apache License

private boolean isLocal(Job job) {
    Configuration conf = job.getConfiguration();
    if (conf.get(KEY_JOBTRACKER, DEFAULT_JOBTRACKER).equals(DEFAULT_JOBTRACKER)) {
        return true;
    }/*ww w. ja v a  2s .c o m*/
    return false;
}

From source file:com.asakusafw.runtime.stage.configurator.AutoLocalStageConfigurator.java

License:Apache License

private boolean isProtected(Job job) {
    Configuration conf = job.getConfiguration();
    Set<?> finalParameters = null;
    try {// ww w.j a  v  a2  s .c o m
        Field field = Configuration.class.getDeclaredField("finalParameters");
        field.setAccessible(true);
        Object value = field.get(conf);
        if (value instanceof Set<?>) {
            finalParameters = (Set<?>) value;
        }
    } catch (Exception e) {
        LOG.debug("Exception occurred while inspecting configuration", e);
    }
    if (finalParameters == null) {
        LOG.warn("Auto stage localize does not support the current Hadoop version");
        return true;
    }
    for (String key : KEYS_REWRITE_TARGET) {
        if (finalParameters.contains(key)) {
            LOG.warn(MessageFormat
                    .format("Auto stage localize requires that configuration \"{0}\" is not final", key));
            return true;
        }
    }
    return false;
}

From source file:com.asakusafw.runtime.stage.configurator.AutoLocalStageConfigurator.java

License:Apache License

private void localize(Job job) {
    Configuration conf = job.getConfiguration();

    // reset job-tracker
    conf.set(KEY_JOBTRACKER, DEFAULT_JOBTRACKER);

    // replace local directories
    String tmpDir = conf.get(KEY_TEMPORARY_DIRECTORY, "");
    if (tmpDir.isEmpty()) {
        String name = System.getProperty("user.name", "asakusa");
        tmpDir = String.format("/tmp/hadoop-%s/autolocal", name);
    } else if (tmpDir.length() > 1 && tmpDir.endsWith("/")) {
        tmpDir = tmpDir.substring(0, tmpDir.length() - 1);
    }/*from   www. j  a v a 2 s  .co m*/
    if (conf.getBoolean(KEY_DIRECTORY_QUALIFIER, true)) {
        String qualifier = UUID.randomUUID().toString();
        tmpDir = String.format("%s/%s", tmpDir, qualifier);
    }
    LOG.info(MessageFormat.format("Substituting temporary dir: job={0}, target={1}", job.getJobName(), tmpDir));
    conf.set(KEY_LOCAL_DIR, tmpDir + "/mapred/local");
    conf.set(KEY_STAGING_DIR, tmpDir + "/mapred/staging");
}

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfigurator.java

License:Apache License

@Override
public void configure(Job job) throws IOException, InterruptedException {
    if (job.getConfiguration().getBoolean(KEY_FORCE, false)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("force enabled in-process execution: {0}", //$NON-NLS-1$
                    job.getJobName()));// ww  w.jav a2  s  . c o  m
        }
        install(job);
        return;
    }
    long limit = job.getConfiguration().getLong(KEY_LIMIT, -1L);
    if (limit < 0L) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("in-process execution is disabled: {0}", //$NON-NLS-1$
                    job.getJobName()));
        }
        return;
    }
    if (hasCustomJobRunner(job)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("custom job runner is already activated: {0}", //$NON-NLS-1$
                    job.getJobName()));
        }
        return;
    }
    long estimated = getEstimatedJobSize(job);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("estimated input data size for in-process execution: " //$NON-NLS-1$
                + "job={0}, limit={1}, estimated={2}", //$NON-NLS-1$
                job.getJobName(), limit, estimated));
    }
    if (estimated < 0L || estimated > limit) {
        return;
    }
    if (LOG.isInfoEnabled()) {
        LOG.info(MessageFormat.format("enable in-process execution: job={0}, limit={1}, estimated={2}",
                job.getJobName(), limit, estimated));
    }
    install(job);
}

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfigurator.java

License:Apache License

private boolean hasCustomJobRunner(Job job) {
    return job.getConfiguration().get(StageConstants.PROP_JOB_RUNNER) != null;
}

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfigurator.java

License:Apache License

private void install(Job job) {
    Configuration conf = job.getConfiguration();
    int prefixLength = KEY_PREFIX_REPLACE.length();
    for (Map.Entry<String, String> entry : conf.getValByRegex(PATTERN_KEY_REPLACE.pattern()).entrySet()) {
        assert entry.getKey().length() >= prefixLength;
        String key = entry.getKey().substring(prefixLength);
        if (key.isEmpty()) {
            continue;
        }//from  w w w.j a v a2  s. c om
        String value = entry.getValue();
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("activate in-process configuration: {0}=\"{1}\"->\"{2}\"", //$NON-NLS-1$
                    key, conf.get(key, ""), //$NON-NLS-1$
                    value));
        }
        conf.set(key, value);
    }
    conf.set(StageConstants.PROP_JOB_RUNNER, SimpleJobRunner.class.getName());
    StageResourceDriver.setAccessMode(job, StageResourceDriver.AccessMode.DIRECT);
    StageInputFormat.setSplitCombinerClass(job, ExtremeSplitCombiner.class);
}

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * simple case.//w ww. j  a  v a2  s  .co m
 * @throws Exception if failed
 */
@Test
public void simple() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    StageInputFormat.setSplitCombinerClass(job, IdentitySplitCombiner.class);

    new Mock(100).configure(job);
    assertThat(conf.get(StageConstants.PROP_JOB_RUNNER), is(notNullValue()));
    assertThat(StageResourceDriver.getAccessMode(job), is(StageResourceDriver.AccessMode.DIRECT));
    assertThat(StageInputFormat.getSplitCombinerClass(job), is((Object) ExtremeSplitCombiner.class));
}

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * disabled./*from   w  w w . j  a v a  2 s .  c  o  m*/
 * @throws Exception if failed
 */
@Test
public void disabled() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    Assume.assumeThat(conf.get(KEY_LIMIT), is(nullValue()));
    StageInputFormat.setSplitCombinerClass(job, IdentitySplitCombiner.class);

    new Mock(100).configure(job);
    assertThat(conf.get(StageConstants.PROP_JOB_RUNNER), is(nullValue()));
    assertThat(StageResourceDriver.getAccessMode(job), is(not(StageResourceDriver.AccessMode.DIRECT)));
    assertThat(StageInputFormat.getSplitCombinerClass(job), is(not((Object) ExtremeSplitCombiner.class)));
}

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * input size is exceeded./* w w w.ja va 2  s .c o m*/
 * @throws Exception if failed
 */
@Test
public void large() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    new Mock(101).configure(job);
    assertThat(conf.get(StageConstants.PROP_JOB_RUNNER), is(nullValue()));
}