Example usage for org.apache.hadoop.mapreduce JobContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce JobContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.ambiata.ivory.operation.hadoop.MultipleInputs.java

License:Apache License

/**
 * Retrieves a map of {@link Path}s to the {@link Mapper} class that
 * should be used for them./*from  w w w  .  j  a  v a2 s.  c  o m*/
 *
 * @param job The {@link JobContext}
 * @see #addInputPath(JobConf, Path, Class, Class)
 * @return A map of paths to mappers for the job
 */
@SuppressWarnings("unchecked")
static Map<Path, Class<? extends Mapper>> getMapperTypeMap(JobContext job) {
    Configuration conf = job.getConfiguration();
    if (conf.get(DIR_MAPPERS) == null) {
        return Collections.emptyMap();
    }
    Map<Path, Class<? extends Mapper>> m = new HashMap<Path, Class<? extends Mapper>>();
    String[] pathMappings = conf.get(DIR_MAPPERS).split(",");
    for (String pathMappingEncoded : pathMappings) {
        /* WAS not decoded */
        String pathMapping = decode(pathMappingEncoded);
        String[] split = pathMapping.split(";");
        Class<? extends Mapper> mapClass;
        try {
            mapClass = (Class<? extends Mapper>) conf.getClassByName(split[1]);
        } catch (ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
        m.put(new Path(split[0]), mapClass);
    }
    return m;
}

---

From source file:com.asakusafw.bridge.hadoop.Compatibility.java

License:Apache License

/**
 * Invokes {@link JobContext#getConfiguration()}.
 * @param context the target context/*from w  w w .  ja  v  a  2  s  . co m*/
 * @return the invocation result
 * @deprecated Directly use {@code context.getConfiguration()} instead
 */
@Deprecated
public static Configuration getConfiguration(JobContext context) {
    if (JOB_CONTEXT_GET_CONFIGURATION != null) {
        try {
            return (Configuration) JOB_CONTEXT_GET_CONFIGURATION.invoke(context);
        } catch (ReflectiveOperationException e) {
            LOG.warn("failed to invoke compatible operation", e);
        }
    }
    return context.getConfiguration();
}

---

From source file:com.asakusafw.bridge.hadoop.directio.DirectFileInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    StageInfo stage = getStageInfo(conf);
    DirectFileInputInfo<?> info = extractInfo(context);
    DirectDataSourceRepository repository = getDataSourceRepository(context);
    String containerPath = repository.getContainerPath(info.basePath);
    List<DirectInputFragment> fragments = findFragments(info, repository);
    List<InputSplit> results = new ArrayList<>();
    for (DirectInputFragment fragment : fragments) {
        DirectFileInputSplit split = new DirectFileInputSplit(containerPath, info.definition, fragment,
                stage.getBatchArguments());
        ReflectionUtils.setConf(split, conf);
        results.add(split);/*from ww  w . java 2 s  .  c o  m*/
    }
    if (results.isEmpty()) {
        results.add(new NullInputSplit());
    }
    return results;
}

---

From source file:com.asakusafw.bridge.hadoop.directio.DirectFileInputFormat.java

License:Apache License

private DirectFileInputInfo<?> extractInfo(JobContext context) {
    Configuration conf = context.getConfiguration();
    String basePath = extract(conf, KEY_BASE_PATH, true, true);
    String resourcePath = extract(conf, KEY_RESOURCE_PATH, true, true);
    Class<?> dataClass = extractClass(conf, KEY_DATA_CLASS, true);
    Class<?> formatClass = extractClass(conf, KEY_FORMAT_CLASS, true);
    Class<?> filterClass = extractClass(conf, KEY_FILTER_CLASS, false);
    String optionalString = extract(conf, KEY_OPTIONAL, false, false);
    if (optionalString == null) {
        optionalString = DirectDataSourceConstants.DEFAULT_OPTIONAL;
    }/*from  ww  w .j av a  2 s. c om*/

    ResourcePattern resourcePattern = FilePattern.compile(resourcePath);
    DataDefinition<?> definition = SimpleDataDefinition.newInstance(dataClass,
            (DataFormat<?>) ReflectionUtils.newInstance(formatClass, conf), createFilter(filterClass, conf));
    boolean optional = Boolean.parseBoolean(optionalString);

    return new DirectFileInputInfo<>(basePath, resourcePattern, definition, optional);
}

---

From source file:com.asakusafw.bridge.hadoop.directio.Util.java

License:Apache License

static DirectDataSourceRepository getDataSourceRepository(JobContext context) {
    return HadoopDataSourceUtil.loadRepository(context.getConfiguration());
}

---

From source file:com.asakusafw.runtime.compatibility.hadoop1.JobCompatibilityHadoop1.java

License:Apache License

private static String getJobMode(JobContext context) {
    return context.getConfiguration().get(KEY_JOBTRACKER_ADDRESS);
}

---

From source file:com.asakusafw.runtime.compatibility.hadoop2.JobCompatibilityHadoop2.java

License:Apache License

@Override
public boolean isLocalMode(JobContext context) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }//  ww  w.j av  a  2s. c  o m
    if (isMRv1()) {
        return isLocalModeMRv1(context);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("{0}={1}", //$NON-NLS-1$
                MRConfig.FRAMEWORK_NAME, context.getConfiguration().get(MRConfig.FRAMEWORK_NAME)));
    }
    String name = context.getConfiguration().get(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
    return name.equals(MRConfig.LOCAL_FRAMEWORK_NAME);
}

---

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

private static String getTransactionId(JobContext jobContext, String datasourceId) {
    assert jobContext != null;
    assert datasourceId != null;
    String executionId = jobContext.getConfiguration().get(StageConstants.PROP_EXECUTION_ID);
    if (executionId == null) {
        executionId = jobContext.getJobID().toString();
    }//from   w w w. java2  s.co  m
    return getTransactionId(executionId);
}

---

From source file:com.asakusafw.runtime.stage.input.BridgeInputFormat.java

License:Apache License

/**
 * Computes and returns splits for the specified inputs.
 * @param context current job context//from w ww.j  a v a  2s. c o  m
 * @param inputList target input list
 * @return the computed splits
 * @throws IOException if failed to compute splits
 * @throws InterruptedException if interrupted while computing inputs
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
public List<InputSplit> getSplits(JobContext context, List<StageInput> inputList)
        throws IOException, InterruptedException {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    if (inputList == null) {
        throw new IllegalArgumentException("inputList must not be null"); //$NON-NLS-1$
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Start computing splits for Direct I/O: input={0}", //$NON-NLS-1$
                inputList.size()));
    }
    long t0 = -1L;
    if (LOG.isInfoEnabled()) {
        t0 = System.currentTimeMillis();
    }
    DirectDataSourceRepository repo = getDataSourceRepository(context);
    List<InputSplit> results = new ArrayList<>();
    Map<DirectInputGroup, List<InputPath>> patternGroups = extractInputList(context, repo, inputList);
    long totalSize = 0;
    for (Map.Entry<DirectInputGroup, List<InputPath>> entry : patternGroups.entrySet()) {
        DirectInputGroup group = entry.getKey();
        List<InputPath> paths = entry.getValue();
        DirectDataSource dataSource = repo.getRelatedDataSource(group.containerPath);
        DataDefinition<?> definition = createDataDefinition(context.getConfiguration(), group);
        for (InputPath path : paths) {
            List<DirectInputFragment> fragments = getFragments(repo, group, path, definition, dataSource);
            for (DirectInputFragment fragment : fragments) {
                totalSize += fragment.getSize();
                results.add(new BridgeInputSplit(group, fragment));
            }
        }
    }
    if (results.isEmpty()) {
        // Execute this job even if there are no input fragments.
        // It will create empty output files required by successive jobs.
        results.add(new NullInputSplit());
    }
    if (LOG.isInfoEnabled()) {
        String type = "(unknown)"; //$NON-NLS-1$
        if (patternGroups.isEmpty() == false) {
            type = patternGroups.keySet().iterator().next().dataType.getName();
        }
        long t1 = System.currentTimeMillis();
        LOG.info(MessageFormat.format(
                "found Direct I/O input splits: primary-type={0}, fragments={1}, size={2}bytes, elapsed={3}ms",
                type, results.size(), totalSize, t1 - t0));
    }
    return results;
}

---

From source file:com.asakusafw.runtime.stage.input.BridgeInputFormat.java

License:Apache License

private Map<DirectInputGroup, List<InputPath>> extractInputList(JobContext context,
        DirectDataSourceRepository repo, List<StageInput> inputList) throws IOException {
    assert context != null;
    assert repo != null;
    assert inputList != null;
    VariableTable variables = createBatchArgumentsTable(context.getConfiguration());
    Map<DirectInputGroup, List<InputPath>> results = new HashMap<>();
    for (StageInput input : inputList) {
        String fullBasePath = variables.parse(extractBasePath(input));
        String basePath = repo.getComponentPath(fullBasePath);
        FilePattern pattern = extractSearchPattern(context, variables, input);
        Class<?> dataClass = extractDataClass(context, input);
        Class<? extends DataFormat<?>> formatClass = extractFormatClass(context, input);
        Class<? extends DataFilter<?>> filterClass = extractFilterClass(context, input);
        DirectInputGroup group = new DirectInputGroup(fullBasePath, dataClass, formatClass, filterClass);
        List<InputPath> paths = results.get(group);
        if (paths == null) {
            paths = new ArrayList<>();
            results.put(group, paths);/*from  w w w  .  j  a  v  a 2s . c o m*/
        }
        paths.add(new InputPath(basePath, pattern, extractOptional(input)));
    }
    return results;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »