Example usage for org.apache.hadoop.mapreduce Job getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job getConfiguration.

Prototype

public Configuration getConfiguration() 

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * activate alternative properties.//from  ww  w. ja  v  a  2  s . co m
 * @throws Exception if failed
 */
@Test
public void activate_properties() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    conf.set(KEY_PREFIX_REPLACE + "com.example.testing", "YES!");
    new Mock(100).configure(job);
    assertThat(conf.get("com.example.testing"), is("YES!"));
}

---

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * activate alternative properties.//w ww . j  av a  2  s .  co  m
 * @throws Exception if failed
 */
@Test
public void activate_properties_skip() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    conf.set(KEY_PREFIX_REPLACE + "com.example.testing", "YES!");
    new Mock(1000).configure(job);
    assertThat(conf.get("com.example.testing"), is(not("YES!")));
}

---

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

private Job newJob() {
    try {/*from w ww  .j a v a  2 s .  c om*/
        Job job = JobCompatibility.newJob(new ConfigurationProvider().newInstance());
        Assume.assumeThat(job.getConfiguration().get(StageConstants.PROP_JOB_RUNNER), is(nullValue()));
        job.setJobName("testing");
        return job;
    } catch (IOException e) {
        Assume.assumeNoException(e);
        throw new AssertionError(e);
    }
}

---

From source file:com.asakusafw.runtime.stage.input.StageInputDriver.java

License:Apache License

/**
 * Sets the input specification for this job.
 * @param job current job/*from  w  w w  .j a v  a 2  s. c  om*/
 * @param inputList each input specification
 * @throws IllegalArgumentException if some parameters were {@code null}
 * @since 0.2.5
 */
public static void set(Job job, List<StageInput> inputList) {
    if (job == null) {
        throw new IllegalArgumentException("job must not be null"); //$NON-NLS-1$
    }
    if (inputList == null) {
        throw new IllegalArgumentException("inputList must not be null"); //$NON-NLS-1$
    }
    try {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("Encoding inputs ({0} entries)", //$NON-NLS-1$
                    inputList.size()));
        }
        String encoded = encode(inputList);
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("Encoded inputs ({0} bytes)", //$NON-NLS-1$
                    encoded.length()));
        }
        job.getConfiguration().set(KEY, encoded);
    } catch (IOException e) {
        throw new IllegalArgumentException(MessageFormat.format("Failed to store input information: {0}", KEY),
                e);
    }
}

---

From source file:com.asakusafw.runtime.stage.optimizer.LibraryCopySuppressionConfigurator.java

License:Apache License

@Override
public void configure(Job job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    if (conf.getBoolean(KEY_ENABLED, DEFAULT_ENABLED) == false) {
        return;// w ww . j av  a 2 s.  c  o  m
    }
    // activates only if application launcher is used
    if (conf.getBoolean(ApplicationLauncher.KEY_LAUNCHER_USED, false) == false) {
        return;
    }
    if (JobCompatibility.isLocalMode(job) == false) {
        return;
    }
    String libraries = conf.get(KEY_CONF_LIBRARIES);
    if (libraries == null || libraries.isEmpty()) {
        return;
    }
    Set<String> loaded = new HashSet<>();
    ClassLoader loader = conf.getClassLoader();
    if (loader instanceof URLClassLoader) {
        for (URL url : ((URLClassLoader) loader).getURLs()) {
            try {
                loaded.add(url.toURI().toString());
            } catch (URISyntaxException e) {
                LOG.warn(MessageFormat.format("Failed to analyze classpath: {0}", url));
            }
        }
    }
    if (loaded.isEmpty()) {
        return;
    }
    StringBuilder result = new StringBuilder();
    for (String library : libraries.split(",")) { //$NON-NLS-1$
        if (loaded.contains(library)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Keep library: {0}", library)); //$NON-NLS-1$
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Suppress library: {0}", library)); //$NON-NLS-1$
            }
            if (result.length() != 0) {
                result.append(',');
            }
            result.append(library);
        }
    }
    if (result.length() > 0) {
        conf.set(KEY_CONF_LIBRARIES, result.toString());
    } else {
        if (CONFIGURATION_UNSET != null) {
            try {
                CONFIGURATION_UNSET.invoke(conf, KEY_CONF_LIBRARIES);
                return;
            } catch (Exception e) {
                LOG.warn(MessageFormat.format("Failed to invoke {0}", CONFIGURATION_UNSET), e);
            }
        }
        String newLibraries = selectLibraries(libraries);
        conf.set(KEY_CONF_LIBRARIES, newLibraries);
    }
}

---

From source file:com.asakusafw.runtime.stage.optimizer.ReducerSimplifierConfigurator.java

License:Apache License

@Override
public void configure(Job job) throws IOException, InterruptedException {
    int count = job.getNumReduceTasks();
    if (count <= TASKS_TINY) {
        return;//from www.jav  a 2 s  .c o  m
    }
    Configuration conf = job.getConfiguration();
    long limit = conf.getLong(KEY_TINY_LIMIT, -1L);
    if (limit < 0L) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("Reducer simplifier is disabled for tiny inputs: {0}", //$NON-NLS-1$
                    job.getJobName()));
        }
        return;
    }
    long estimated = StageInputDriver.estimateInputSize(job);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Reducer simplifier: job={0}, tiny-limit={1}, estimated={2}", //$NON-NLS-1$
                job.getJobName(), limit, estimated));
    }
    if (estimated < 0L || estimated > limit) {
        return;
    }

    LOG.info(MessageFormat.format("The number of reduce task ({0}) is configured: {1}->{2}", job.getJobName(),
            job.getNumReduceTasks(), TASKS_TINY));

    job.setNumReduceTasks(TASKS_TINY);
}

---

From source file:com.asakusafw.runtime.stage.output.StageOutputDriver.java

License:Apache License

private ResultOutput<?> buildNormalSink(String name,
        @SuppressWarnings("rawtypes") Class<? extends OutputFormat> formatClass, Class<?> keyClass,
        Class<?> valueClass, List<Counter> counters) throws IOException, InterruptedException {
    assert context != null;
    assert name != null;
    assert formatClass != null;
    assert keyClass != null;
    assert valueClass != null;
    assert counters != null;
    Job job = JobCompatibility.newJob(context.getConfiguration());
    job.setOutputFormatClass(formatClass);
    job.setOutputKeyClass(keyClass);//from www  . j av  a  2  s  .  co  m
    job.setOutputValueClass(valueClass);
    TaskAttemptContext localContext = JobCompatibility.newTaskAttemptContext(job.getConfiguration(),
            context.getTaskAttemptID());
    if (FileOutputFormat.class.isAssignableFrom(formatClass)) {
        setOutputFilePrefix(localContext, name);
    }
    OutputFormat<?, ?> format = ReflectionUtils.newInstance(formatClass, localContext.getConfiguration());
    RecordWriter<?, ?> writer = format.getRecordWriter(localContext);
    return new ResultOutput<Writable>(localContext, writer);
}

---

From source file:com.asakusafw.runtime.stage.output.StageOutputDriver.java

License:Apache License

private static void addOutput(Job job, String name, Class<?> formatClass, Class<?> keyClass,
        Class<?> valueClass) {
    assert job != null;
    assert name != null;
    assert formatClass != null;
    assert keyClass != null;
    assert valueClass != null;
    if (isValidName(name) == false) {
        throw new IllegalArgumentException(MessageFormat.format("Output name \"{0}\" is not valid", name));
    }/*from ww w.j  a v a2 s . com*/
    Configuration conf = job.getConfiguration();
    Set<String> names = new TreeSet<>(conf.getStringCollection(K_NAMES));
    if (names.contains(name)) {
        throw new IllegalArgumentException(
                MessageFormat.format("Output name \"{0}\" is already declared", name));
    }
    names.add(name);
    conf.setStrings(K_NAMES, names.toArray(new String[names.size()]));
    conf.setClass(getPropertyName(K_FORMAT_PREFIX, name), formatClass, OutputFormat.class);
    conf.setClass(getPropertyName(K_KEY_PREFIX, name), keyClass, Object.class);
    conf.setClass(getPropertyName(K_VALUE_PREFIX, name), valueClass, Object.class);
}

---

From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java

License:Apache License

/**
 * Adds a resource path into the target job object.
 * @param job the target job//w w w.  j  a  va 2s .c om
 * @param resourcePath the resource path expression (this must be accessible from task execution nodes)
 * @param resourceName the resource name
 * @throws IOException if failed to detect resources on the path
 * @throws IllegalArgumentException if some parameters are {@code null}
 */
public static void add(Job job, String resourcePath, String resourceName) throws IOException {
    if (job == null) {
        throw new IllegalArgumentException("job must not be null"); //$NON-NLS-1$
    }
    if (resourcePath == null) {
        throw new IllegalArgumentException("resourcePath must not be null"); //$NON-NLS-1$
    }
    if (resourceName == null) {
        throw new IllegalArgumentException("resourceName must not be null"); //$NON-NLS-1$
    }
    Configuration conf = job.getConfiguration();
    List<FileStatus> list = TemporaryStorage.listStatus(conf, new Path(resourcePath));
    if (list.isEmpty()) {
        throw new IOException(MessageFormat.format("Resource not found: {0}", resourcePath));
    }
    List<String> localNames = restoreStrings(conf, getLocalCacheNameKey(resourceName));
    List<String> remotePaths = restoreStrings(conf, getRemotePathKey(resourceName));
    long size = conf.getLong(KEY_SIZE, 0L);
    int index = localNames.size();
    for (FileStatus status : list) {
        String name = String.format("%s-%04d", resourceName, index++); //$NON-NLS-1$
        StringBuilder buf = new StringBuilder();
        buf.append(status.getPath().toString());
        buf.append('#');
        buf.append(name);
        String cachePath = buf.toString();

        remotePaths.add(status.getPath().toString());
        localNames.add(name);
        try {
            URI uri = new URI(cachePath);
            DistributedCache.addCacheFile(uri, conf);
        } catch (URISyntaxException e) {
            throw new IllegalStateException(e);
        }
        size += status.getLen();
    }
    conf.setStrings(getLocalCacheNameKey(resourceName), localNames.toArray(new String[localNames.size()]));
    conf.setStrings(getRemotePathKey(resourceName), remotePaths.toArray(new String[remotePaths.size()]));
    conf.setLong(KEY_SIZE, size);
    if (JobCompatibility.isLocalMode(job)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("symlinks for distributed cache will not be created in standalone mode"); //$NON-NLS-1$
        }
    } else {
        DistributedCache.createSymlink(conf);
    }
}

---

From source file:com.asakusafw.runtime.workaround.snappyjava.MacSnappyJavaWorkaround.java

License:Apache License

@Override
public void configure(Job job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    if (conf.getBoolean(KEY_ENABLED, DEFAULT_ENABLED) == false) {
        return;/*  ww  w  .  j  av  a 2  s.  com*/
    }
    install(conf.getBoolean(KEY_SKIP_ON_UNKNOWN, DEFAULT_SKIP_ON_UNKNOWN));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »