Example usage for org.apache.hadoop.mapreduce JobContext getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce JobContext getConfiguration.

Prototype

public Configuration getConfiguration();

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.zinnia.nectar.util.hadoop.inputformat.FirstNLineInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
    // TODO Auto-generated method stub
    List<InputSplit> inputSplits = new ArrayList<InputSplit>();
    for (FileStatus status : listStatus(job)) {
        inputSplits.addAll(getSplitsForFile(status, job.getConfiguration(), getNumLinesPerSplit(job)));
    }//from   w  w w  .j  av  a  2 s .  c o m

    return inputSplits;

}

From source file:com.zjy.mongo.MongoInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(final JobContext context) throws IOException {
    final Configuration conf = context.getConfiguration();
    try {//from w ww . j a va 2 s . c om
        MongoSplitter splitterImpl = MongoSplitterFactory.getSplitter(conf);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Using " + splitterImpl.toString() + " to calculate splits.");
        }
        return splitterImpl.calculateSplits();
    } catch (SplitFailedException spfe) {
        throw new IOException(spfe);
    }
}

From source file:cz.seznam.euphoria.hadoop.input.DataSourceInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext jc) throws IOException, InterruptedException {

    initialize(jc.getConfiguration());
    return source.getPartitions().stream().map(SourceSplit::new).collect(Collectors.toList());
}

From source file:cz.seznam.euphoria.hadoop.output.DataSinkOutputFormat.java

License:Apache License

private void instantiateSink(JobContext jc) throws IOException {
    if (sink == null) {
        String sinkBytes = jc.getConfiguration().get(DATA_SINK, null);
        if (sinkBytes == null) {
            throw new IllegalStateException(
                    "Invalid output spec, call `DataSinkOutputFormat#configure` before passing "
                            + " the configuration to output");
        }/*  w  w w.  j a  v a2 s . co  m*/
        try {
            sink = fromBase64(sinkBytes);
            sink.initialize();
        } catch (ClassNotFoundException ex) {
            throw new IOException(ex);
        }
    }
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputCommitter.java

License:Apache License

/**
 * Create the temporary directory that is the root of all of the task work
 * directories.//from   w  w w . j a  v a 2s  .  co  m
 *
 * @param context the job's context
 */
public void setupJob(JobContext context) throws IOException {
    if (this.outputPath != null && this.tempPath != null) {
        Path tmpDir = new Path(this.tempPath, HirodsFileOutputCommitter.TEMP_DIR_NAME);
        FileSystem fileSys = tmpDir.getFileSystem(context.getConfiguration());
        if (!fileSys.mkdirs(tmpDir)) {
            LOG.error("Mkdirs failed to create " + tmpDir.toString());
        }
    }
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputFormat.java

License:Apache License

/**
 * Is the job output compressed?// w  w w.  j  av  a2  s.  c o  m
 *
 * @param job the Job to look in
 * @return <code>true</code> if the job output should be compressed,
 * <code>false</code> otherwise
 */
public static boolean getCompressOutput(JobContext job) {
    return job.getConfiguration().getBoolean("edu.arizona.cs.hadoop.fs.irods.mapred.output.compress", false);
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputFormat.java

License:Apache License

/**
 * Get the {@link CompressionCodec} for compressing the job outputs.
 *
 * @param job the {@link Job} to look in
 * @param defaultValue the {@link CompressionCodec} to return if not set
 * @return the {@link CompressionCodec} to be used to compress the job
 * outputs/*w  w w  .  j  av  a 2  s  .  c o m*/
 * @throws IllegalArgumentException if the class was specified, but not
 * found
 */
public static Class<? extends CompressionCodec> getOutputCompressorClass(JobContext job,
        Class<? extends CompressionCodec> defaultValue) {
    Class<? extends CompressionCodec> codecClass = defaultValue;
    Configuration conf = job.getConfiguration();
    String name = conf.get("edu.arizona.cs.hadoop.fs.irods.mapred.output.compression.codec");
    if (name != null) {
        try {
            codecClass = conf.getClassByName(name).asSubclass(CompressionCodec.class);
        } catch (ClassNotFoundException e) {
            throw new IllegalArgumentException("Compression codec " + name + " was not found.", e);
        }
    }
    return codecClass;
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputFormat.java

License:Apache License

/**
 * Get the {@link Path} to the output directory for the map-reduce job.
 *
 * @return the {@link Path} to the output directory for the map-reduce job.
 * @see HirodsFileOutputFormat#getWorkOutputPath(TaskInputOutputContext)
 *//*from  ww w. j a  v a2  s  .  c  o m*/
public static Path getOutputPath(JobContext job) {
    String name = job.getConfiguration().get("edu.arizona.cs.hadoop.fs.irods.mapred.output.dir");
    return name == null ? null : new Path(name);
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputFormat.java

License:Apache License

public static Path getTempPath(JobContext job) {
    String name = HirodsConfigUtils.getIrodsOutputBufferedPath(job.getConfiguration());
    return name == null ? null : new Path(name);
}

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsMultipleOutputs.java

License:Apache License

private static Class<?> getNamedOutputKeyClass(JobContext job, String namedOutput) {
    return job.getConfiguration().getClass(MO_PREFIX + namedOutput + KEY, null, WritableComparable.class);
}