Example usage for org.apache.hadoop.mapreduce Job getConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job getConfiguration.

Prototype

public Configuration getConfiguration()

Source Link

Document

Return the configuration for the job.

Usage

From source file:com.cg.mapreduce.myfpgrowth.PFPGrowth.java

License:Apache License

private static Job initJob(Configuration conf) {
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/core-site.xml"));
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/hdfs-default.xml"));
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/hdfs-site.xml"));
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/yarn-default.xml"));
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/yarn-site.xml"));
    conf.addResource(new Path("D:/program/hadoop-2.6.0/etc/hadoop/mapred-site.xml"));
    conf.set("HADOOP_USER_NAME", "hadoop");
    conf.set("mapred.reduce.tasks", "3");
    Job job = null;
    try {//from   w  ww  .  ja  v a 2s  .  c o m
        File jarFile = EJob.createTempJar("bin");
        EJob.addClasspath("D:/program/hadoop-2.6.0/etc/hadoop/");
        ClassLoader classLoader = EJob.getClassLoader();
        Thread.currentThread().setContextClassLoader(classLoader);
        job = new Job(conf, "PFP");
        ((JobConf) job.getConfiguration()).setJar(jarFile.toString());
    } catch (IOException e) {
        e.printStackTrace();
    }
    return job;
}

From source file:com.chinamobile.bcbsp.io.BSPFileInputFormat.java

License:Apache License

/**
 * Set a PathFilter to be applied to the input paths for the map-reduce job.
 *
 * @param job//  ww  w.  ja va 2s.com
 *        the job to modify
 * @param filter
 *        the PathFilter class use for filtering the input paths.
 */
public static void setInputPathFilter(Job job, Class<? extends PathFilter> filter) {
    job.getConfiguration().setClass("mapred.input.pathFilter.class", filter, PathFilter.class);
}

From source file:com.chinamobile.bcbsp.io.BSPFileInputFormat.java

License:Apache License

/**
 * Set the minimum input split size TODO This function is disable
 *
 * @param job//w  w  w.  jav a  2s  .  c om
 *        the job to modify
 * @param size
 *        the minimum size
 */
public static void setMinInputSplitSize(Job job, long size) {
    job.getConfiguration().setLong("mapred.min.split.size", size);
}

From source file:com.chinamobile.bcbsp.io.BSPFileInputFormat.java

License:Apache License

/**
 * Get the minimum split size TODO This function is disable
 *
 * @param job/*from w  ww  . j  av  a 2  s .c om*/
 *        the job
 * @return the minimum number of bytes that can be in a split
 */
public static long getMinSplitSize(Job job) {
    return job.getConfiguration().getLong("mapred.min.split.size", 1L);
}

From source file:com.chinamobile.bcbsp.io.BSPFileInputFormat.java

License:Apache License

/**
 * Set the maximum split size TODO This function is disable
 *
 * @param job//from  w  w w  . j av a 2 s .co m
 *        the job to modify
 * @param size
 *        the maximum split size
 */
public static void setMaxInputSplitSize(Job job, long size) {
    job.getConfiguration().setLong("mapred.max.split.size", size);
}

From source file:com.chinamobile.bcbsp.io.BSPFileInputFormat.java

License:Apache License

/**
 * Get the maximum split size. TODO This function is disable
 *
 * @param context// ww w  . j  a  va 2s.com
 *        the job to look at.
 * @return the maximum number of bytes a split can include
 */
public static long getMaxSplitSize(Job context) {
    return context.getConfiguration().getLong("mapred.max.split.size", Long.MAX_VALUE);
}

From source file:com.ci.backports.avro.mapreduce.AvroJob.java

License:Apache License

/** Configure a job's map input schema. */
public static void setInputSchema(Job job, Schema schema) {
    job.getConfiguration().set(INPUT_SCHEMA_CONFIG_FIELD, schema.toString());
}

From source file:com.ci.backports.avro.mapreduce.AvroJob.java

License:Apache License

/** Configure a job's map output key schema. */
public static void setMapOutputKeySchema(Job job, Schema schema) {
    job.setMapOutputKeyClass(AvroKey.class);
    job.getConfiguration().set(KEY_MAP_OUTPUT_SCHEMA_CONFIG_FIELD, schema.toString());
    job.setGroupingComparatorClass(AvroKeyComparator.class);
    job.setSortComparatorClass(AvroKeyComparator.class);
    addAvroSerialization(job.getConfiguration());
}

From source file:com.ci.backports.avro.mapreduce.AvroJob.java

License:Apache License

/** Configure a job's map output value schema. */
public static void setMapOutputValueSchema(Job job, Schema schema) {
    job.setMapOutputValueClass(AvroValue.class);
    job.getConfiguration().set(VALUE_MAP_OUTPUT_SCHEMA_CONFIG_FIELD, schema.toString());
    addAvroSerialization(job.getConfiguration());
}

From source file:com.ci.backports.avro.mapreduce.AvroJob.java

License:Apache License

/** Configure a job's output schema. */
public static void setOutputSchema(Job job, Schema schema) {
    job.setOutputKeyClass(AvroKey.class);
    job.setOutputValueClass(NullWritable.class);
    job.getConfiguration().set(OUTPUT_SCHEMA_CONFIG_FIELD, schema.toString());
    addAvroSerialization(job.getConfiguration());
}