Example usage for org.apache.hadoop.mapreduce JobContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce JobContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.asakusafw.runtime.stage.input.StageInputFormat.java

License:Apache License

/**
 * Sets the {@link SplitCombiner} class for the current job.
 * @param context the current job context
 * @param aClass the {@link SplitCombiner} class
 * @since 0.7.1/*from  w ww.ja v a 2  s . c o  m*/
 */
public static void setSplitCombinerClass(JobContext context, Class<? extends SplitCombiner> aClass) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    if (aClass == null) {
        throw new IllegalArgumentException("aClass must not be null"); //$NON-NLS-1$
    }
    context.getConfiguration().set(KEY_SPLIT_COMBINER, aClass.getName());
}

From source file:com.asakusafw.runtime.stage.input.StageInputFormat.java

License:Apache License

private static Map<Class<? extends InputFormat<?, ?>>, InputFormat<?, ?>> instantiateFormats(JobContext context,
        Set<FormatAndMapper> pairs) throws IOException {
    assert context != null;
    assert pairs != null;
    Configuration conf = context.getConfiguration();
    Map<Class<? extends InputFormat<?, ?>>, InputFormat<?, ?>> results = new HashMap<>();
    for (FormatAndMapper pair : pairs) {
        Class<? extends InputFormat<?, ?>> type = pair.formatClass;
        if (results.containsKey(type) == false) {
            try {
                InputFormat<?, ?> instance = ReflectionUtils.newInstance(type, conf);
                results.put(type, instance);
            } catch (RuntimeException e) {
                throw new IOException(MessageFormat.format("Cannot instantiate {0}", type.getName()), e);
            }/* w  w w. ja va 2 s. c o m*/
        }
    }
    return results;
}

From source file:com.asakusafw.runtime.stage.input.TemporaryInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    return getSplits(context.getConfiguration(), getInputPaths(context));
}

From source file:com.asakusafw.runtime.stage.input.TemporaryInputFormat.java

License:Apache License

/**
 * Computes and returns splits for the specified inputs.
 * @param context current job context/*from  w  w  w  .  ja v  a2  s .com*/
 * @param inputList target input list
 * @return the computed splits
 * @throws IOException if failed to compute splits
 * @throws InterruptedException if interrupted while computing inputs
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
public List<InputSplit> getSplits(JobContext context, List<StageInput> inputList)
        throws IOException, InterruptedException {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    if (inputList == null) {
        throw new IllegalArgumentException("inputList must not be null"); //$NON-NLS-1$
    }
    List<Path> paths = new ArrayList<>();
    for (StageInput input : inputList) {
        paths.add(new Path(input.getPathString()));
    }
    return getSplits(context.getConfiguration(), paths);
}

From source file:com.asakusafw.runtime.stage.output.BridgeOutputFormat.java

License:Apache License

/**
 * Returns whether this stage has an output corresponding this format.
 * @param context current context/*from  ww w .j  av a  2 s.  c  o m*/
 * @return {@code true} if such output exists, otherwise {@code false}
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
public static boolean hasOutput(JobContext context) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    return context.getConfiguration().getRaw(KEY) != null;
}

From source file:com.asakusafw.runtime.stage.output.BridgeOutputFormat.java

License:Apache License

/**
 * Sets current output information into the current context.
 * @param context current context/*from   ww  w  .  j av  a 2s. c o m*/
 * @param outputList output information to be set
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
public static void set(JobContext context, List<StageOutput> outputList) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    if (outputList == null) {
        throw new IllegalArgumentException("outputList must not be null"); //$NON-NLS-1$
    }
    List<OutputSpec> specs = new ArrayList<>();
    for (StageOutput output : outputList) {
        List<String> deletePatterns = getDeletePatterns(output);
        OutputSpec spec = new OutputSpec(output.getName(), deletePatterns);
        specs.add(spec);
    }
    save(context.getConfiguration(), specs);
}

From source file:com.asakusafw.runtime.stage.output.BridgeOutputFormat.java

License:Apache License

private static List<OutputSpec> getSpecs(JobContext context) {
    assert context != null;
    String encoded = context.getConfiguration().getRaw(KEY);
    if (encoded == null) {
        return Collections.emptyList();
    }//w ww.  ja  v a  2 s . c  o  m
    VariableTable table = getVariableTable(context);
    try {
        ByteArrayInputStream source = new ByteArrayInputStream(encoded.getBytes(ASCII));
        DataInputStream input = new DataInputStream(new GZIPInputStream(new Base64InputStream(source)));
        long version = WritableUtils.readVLong(input);
        if (version != SERIAL_VERSION) {
            throw new IOException(MessageFormat.format("Invalid StageOutput version: framework={0}, saw={1}",
                    SERIAL_VERSION, version));
        }
        List<OutputSpec> results = new ArrayList<>();
        int specCount = WritableUtils.readVInt(input);
        for (int specIndex = 0; specIndex < specCount; specIndex++) {
            String basePath = WritableUtils.readString(input);
            try {
                basePath = table.parse(basePath);
            } catch (IllegalArgumentException e) {
                throw new IllegalStateException(MessageFormat.format("Invalid basePath: {0}", basePath), e);
            }
            int patternCount = WritableUtils.readVInt(input);
            List<String> patterns = new ArrayList<>();
            for (int patternIndex = 0; patternIndex < patternCount; patternIndex++) {
                String pattern = WritableUtils.readString(input);
                try {
                    pattern = table.parse(pattern);
                } catch (IllegalArgumentException e) {
                    throw new IllegalStateException(
                            MessageFormat.format("Invalid delete pattern: {0}", pattern), e);
                }
                patterns.add(pattern);
            }
            results.add(new OutputSpec(basePath, patterns, true));
        }
        return results;
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
}

From source file:com.asakusafw.runtime.stage.output.BridgeOutputFormat.java

License:Apache License

static VariableTable getVariableTable(JobContext context) {
    assert context != null;
    String arguments = context.getConfiguration().get(StageConstants.PROP_ASAKUSA_BATCH_ARGS, ""); //$NON-NLS-1$
    VariableTable variables = new VariableTable(VariableTable.RedefineStrategy.IGNORE);
    variables.defineVariables(arguments);
    return variables;
}

From source file:com.asakusafw.runtime.stage.output.TemporaryOutputFormat.java

License:Apache License

@Override
public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }/*from www . j  a v  a  2 s.  c om*/
    Path path = getOutputPath(context);
    if (TemporaryOutputFormat.getOutputPath(context) == null) {
        throw new IOException("Temporary output path is not set");
    }
    TokenCache.obtainTokensForNamenodes(context.getCredentials(), new Path[] { path },
            context.getConfiguration());
    if (path.getFileSystem(context.getConfiguration()).exists(path)) {
        throw new IOException(MessageFormat.format("Output directory {0} already exists", path));
    }
}

From source file:com.asakusafw.runtime.stage.output.TemporaryOutputFormat.java

License:Apache License

/**
 * Returns the output path./* w  w w  . j a v  a2  s .c o  m*/
 * @param context current context
 * @return the path
 * @throws IllegalArgumentException if some parameters were {@code null}
 * @see #setOutputPath(JobContext, Path)
 */
public static Path getOutputPath(JobContext context) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    String pathString = context.getConfiguration().get(KEY_OUTPUT_PATH);
    if (pathString == null) {
        return null;
    }
    return new Path(pathString);
}