Example usage for org.apache.hadoop.mapreduce Job setJobName

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setJobName.

Prototype

public void setJobName(String name) throws IllegalStateException 

Source Link

Document

Set the user-specified job name.

Usage

From source file:hadoop.examples.DBCountPageView.java

License:Apache License

public int run(String[] args) throws Exception {

    String driverClassName = DRIVER_CLASS;
    String url = DB_URL;//from   w  w  w  .  ja  va2 s.  c  o  m

    if (args.length > 1) {
        driverClassName = args[0];
        url = args[1];
    }

    initialize(driverClassName, url);
    Configuration conf = getConf();

    DBConfiguration.configureDB(conf, driverClassName, url);

    Job job = new Job(conf);

    job.setJobName("Count Pageviews of URLs");
    job.setJarByClass(DBCountPageView.class);
    job.setMapperClass(PageviewMapper.class);
    job.setCombinerClass(LongSumReducer.class);
    job.setReducerClass(PageviewReducer.class);

    DBInputFormat.setInput(job, AccessRecord.class, "Access", null, "url", AccessFieldNames);

    DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);

    job.setOutputKeyClass(PageviewRecord.class);
    job.setOutputValueClass(NullWritable.class);
    int ret;
    try {
        ret = job.waitForCompletion(true) ? 0 : 1;
        boolean correct = verify();
        if (!correct) {
            throw new RuntimeException("Evaluation was not correct!");
        }
    } finally {
        shutdown();
    }
    return ret;
}

---

From source file:hadoop.examples.mapreduce.WordCountV2.java

License:Open Source License

public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    List<String> other_args = new ArrayList<String>();
    args = new GenericOptionsParser(conf, args).getRemainingArgs();

    for (int i = 0; i < args.length; i++) {
        if ("-skip".equals(args[i])) {
            DistributedCache.addCacheFile(new Path(args[++i]).toUri(), conf);
            conf.setBoolean("wordcount.skip.patterns", true);
        } else if ("-D".equals(args[i])) {
            String[] arr = args[++i].split("=");
            conf.setBoolean(arr[0], Boolean.valueOf(arr[1]));
        } else/*w w  w  .java2s.  co  m*/
            other_args.add(args[i]);
    }

    Job job = new Job(conf);
    job.setJarByClass(WordCountV2.class);
    job.setJobName("word count version 2");
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(TokenizeMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    FileInputFormat.setInputPaths(job, new Path(other_args.get(0)));
    FileOutputFormat.setOutputPath(job, new Path(other_args.get(1)));
    return job.waitForCompletion(true) ? 0 : 1;
}

---

From source file:hadoop.SleepJob.java

License:Apache License

public Job createJob(int numMapper, int numReducer, long mapSleepTime, int mapSleepCount, long reduceSleepTime,
        int reduceSleepCount) throws IOException {
    Configuration conf = getConf();
    conf.setLong(MAP_SLEEP_TIME, mapSleepTime);
    conf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime);
    conf.setInt(MAP_SLEEP_COUNT, mapSleepCount);
    conf.setInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
    conf.setInt(MRJobConfig.NUM_MAPS, numMapper);
    Job job = new Job(conf, "sleep");
    job.setNumReduceTasks(numReducer);//from  w  ww  .j  a v  a 2 s .  c o m
    job.setJarByClass(SleepJob.class);
    job.setNumReduceTasks(numReducer);
    job.setMapperClass(SleepMapper.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(NullWritable.class);
    job.setReducerClass(SleepReducer.class);
    job.setOutputFormatClass(NullOutputFormat.class);
    job.setInputFormatClass(SleepInputFormat.class);
    job.setPartitionerClass(SleepJobPartitioner.class);
    job.setSpeculativeExecution(false);
    job.setJobName("Sleep job");
    FileInputFormat.addInputPath(job, new Path("ignored"));
    return job;
}

---

From source file:hadoop.table.CountTableRows.java

License:Open Source License

@Override
@SuppressWarnings("deprecation")
public int run(String[] args) throws Exception {

    final Configuration conf = getConf();

    /*//w w w .j  av  a  2s  . com
     * Instantiate the logger using the configuration info (if any)
     * specified on the command line. The 'hadoop' command line
     * interpreter places the values of the desired system properties
     * in the Configuration object above, not the system properties on
     * the JVM. As a result, because the logger retrieves its config
     * from the JVM's system properties (not the MapReduce Configuration),
     * the desired logger can be instantiated only after the necessary
     * logger config values are retrieved from the MapReduce config
     * and placed in the system properties of the JVM.
     *
     * Note that once the logger configuration system properties are set
     * on the JVM, they are available to all other loggers that may be
     * created by classes instantiated in the JVM; for example, any
     * utility classes employed by this class.
     */
    if (conf != null) {
        final String loggingConfig = conf.get("java.util.logging.config.file");
        if (loggingConfig != null) {
            System.setProperty("java.util.logging.config.file", loggingConfig);
        }

        final String loggingFormat = conf.get("java.util.logging.SimpleFormatter.format");
        if (loggingFormat != null) {
            System.setProperty("java.util.logging.SimpleFormatter.format", loggingFormat);
        }
    }

    /* Since the JVM properties are set, the logger can now be created. */
    logger = Logger.getLogger(THIS_CLASS_NAME);

    final Job job = new Job(conf);
    job.setJarByClass(CountTableRows.class);
    job.setJobName("Count Table Rows");

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);

    job.setInputFormatClass(TableInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    TableInputFormat.setKVStoreName(args[0]);
    TableInputFormat.setKVHelperHosts(new String[] { args[1] });
    TableInputFormat.setTableName(args[2]);

    FileOutputFormat.setOutputPath(job, new Path(args[3]));

    /*
     * Handle the optional client-side security information. If
     * accessing a secure store, then this information must be
     * specified using the last 2 arguments input on the command
     * line; where,
     * args[4] = name of the client side login file
     * args[5] = name of the server side login file
     */
    if (args.length >= 6) {
        logger.info("using a SECURE KVStore [" + args[4] + ", " + args[5] + "]");
        KVSecurityUtil.createLocalKVSecurity(args[4], args[5]);
        TableInputFormat.setKVSecurity(KVSecurityUtil.getServerLoginFlnm(),
                KVSecurityUtil.getPasswordCredentials(), KVSecurityUtil.getClientTrustFlnm());
    } else {
        logger.info("using a NON-SECURE KVStore");
    }

    if (conf != null) {
        final String primaryKeyProperty = conf.get(ParamConstant.PRIMARY_KEY.getName());
        if (primaryKeyProperty != null) {
            TableInputFormat.setPrimaryKeyProperty(primaryKeyProperty);
        }
    }

    final boolean success = job.waitForCompletion(true);
    return success ? 0 : 1;
}

---

From source file:hadoop.TestingDriver.java

License:Open Source License

public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    String input = conf.get("gc.TestingDriver.input");
    String output = conf.get("gc.TestingDriver.output");
    String jobname = conf.get("gc.TestingDriver.name");
    String dataset = conf.get("gc.TestingDriver.dataset");

    if (input == null || output == null || dataset == null || jobname == null) {
        System.out.println(" Incorrect parameters ");
        System.exit(0);/*  ww w .  j  a va 2  s .co  m*/
    }

    conf = addPathToDC(conf, conf.get("gc.TestingDriver.dataset") + "*");

    Job job = new Job(conf);
    job.setJarByClass(TestingDriverMapper.class);
    job.setJobName(jobname);

    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(PIFArray.class);

    job.setMapperClass(TestingDriverMapper.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(PIFArray.class);

    job.setInputFormatClass(SequenceFileInputFormat.class);
    job.setCombinerClass(TestingDriverReducer.class);
    job.setReducerClass(TestingDriverReducer.class);

    job.setOutputFormatClass(TextOutputFormat.class);

    FileInputFormat.setInputPaths(job, input);

    FileOutputFormat.setOutputPath(job, new Path(output));

    System.out.println(" Input dir = " + conf.get("gc.TestingDriver.input"));
    System.out.println(" Output dir = " + conf.get("gc.TestingDriver.output"));
    System.out.println(" Testing Input = " + conf.get("gc.TestingDriver.dataset"));
    System.out.println(" Name = " + conf.get("gc.TestingDriver.name"));

    if (job.waitForCompletion(true) == false) {
        System.err.println(" Job " + jobname + " Failed (miserably)");
        System.exit(2);
    }

    return 0;
}

---

From source file:hadoop.TrainingDriver.java

License:Open Source License

public int run(String[] args) throws Exception {
    Configuration conf = getConf();

    String input = conf.get("gc.TrainingDriver.input");
    String output = conf.get("gc.TrainingDriver.output");
    String dataset = conf.get("gc.TrainingDriver.dataset");
    String jobname = conf.get("gc.TrainingDriver.name");

    if (input == null || output == null || dataset == null || jobname == null) {
        System.out.println(" Incorrect parameters ");
        System.exit(0);//from ww  w  .ja va2 s .  co  m
    }

    conf = addPathToDC(conf, conf.get("gc.TrainingDriver.dataset") + "*");

    Job job = new Job(conf);
    job.setJarByClass(TrainingDriver.class);
    job.setJobName(jobname);

    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(WeightParameter.class);

    job.setMapperClass(TrainingDriverMapper.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(WeightParameter.class);

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    SequenceFileOutputFormat.setOutputCompressionType(job, SequenceFile.CompressionType.RECORD);
    job.setNumReduceTasks(0);

    FileInputFormat.setInputPaths(job, input);
    FileOutputFormat.setOutputPath(job, new Path(output));

    System.out.println(" Input dir = " + input);
    System.out.println(" Output dir = " + output);
    System.out.println(" Training Input = " + dataset);
    System.out.println(" Name = " + jobname);

    if (job.waitForCompletion(true) == false) {
        System.err.println(" Job " + jobname + " Failed (miserably)");
        System.exit(2);
    }
    return 0;
}

---

From source file:hr.fer.tel.rovkp.homework02.task01.Program.java

public static void main(String[] args) throws Exception {
    Stopwatch timer = new Stopwatch();
    timer.start();/*from   w w  w.j  ava  2  s  .c  o  m*/

    if (args.length != 2) {
        timer.stop();
        System.err.println("Usage: <jar> <input path> <output path>");
        return;
    }

    Job job = Job.getInstance();
    job.setJarByClass(Program.class);
    job.setJobName("TripTimes");

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(TripTimesMapper.class);
    // job.setCombinerClass(TripTimesReducer.class);
    job.setReducerClass(TripTimesReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(TripTimesTuple.class);

    // job.setNumReduceTasks(1);

    job.waitForCompletion(true);
    timer.stop();
    System.out.println("Total time: " + timer.elapsedTime(TimeUnit.SECONDS) + "s");
}

---

From source file:hr.fer.tel.rovkp.homework02.task02.Program.java

public static void main(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.println("Usage: <jar> <input path> <output path>");
        return;/*from  w w  w. j av a2s  .c  o  m*/
    }

    Job job = Job.getInstance();
    job.setJarByClass(Program.class);
    job.setJobName("Locations");

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(LocationsMapper.class);
    job.setPartitionerClass(LocationsPartitioner.class);
    job.setReducerClass(LocationsReducer.class);
    job.setNumReduceTasks(6);

    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(Text.class);

    MultipleOutputs.addNamedOutput(job, "bins", TextOutputFormat.class, NullWritable.class, Text.class);
    job.waitForCompletion(true);
}

---

From source file:hr.fer.tel.rovkp.homework02.task03.Program.java

public static void main(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.println("Usage: <jar> <input path> <output path>");
        return;//w  ww .j  ava 2  s. com
    }

    Job firstJob = Job.getInstance();
    firstJob.setJarByClass(Program.class);
    firstJob.setJobName("Locations");

    FileInputFormat.addInputPath(firstJob, new Path(args[0]));
    FileOutputFormat.setOutputPath(firstJob, new Path(INTERMEDIATE_PATH));

    firstJob.setMapperClass(LocationsMapper.class);
    firstJob.setPartitionerClass(LocationsPartitioner.class);
    firstJob.setReducerClass(LocationsReducer.class);
    firstJob.setNumReduceTasks(6);

    firstJob.setOutputKeyClass(IntWritable.class);
    firstJob.setOutputValueClass(Text.class);

    MultipleOutputs.addNamedOutput(firstJob, "bins", TextOutputFormat.class, NullWritable.class, Text.class);

    int code = firstJob.waitForCompletion(true) ? 0 : 1;

    System.out.println("First job return code: " + code);

    if (code == 0) {

        Job job1 = run(INTERMEDIATE_PATH + "center1", args[1] + "/1");
        Job job2 = run(INTERMEDIATE_PATH + "not_center1", args[1] + "/2");
        Job job3 = run(INTERMEDIATE_PATH + "center2", args[1] + "/3");
        Job job4 = run(INTERMEDIATE_PATH + "not_center2", args[1] + "/4");
        Job job5 = run(INTERMEDIATE_PATH + "center4", args[1] + "/5");
        Job job6 = run(INTERMEDIATE_PATH + "not_center4", args[1] + "/6");

        while (!(job1.isComplete() && job2.isComplete() && job3.isComplete() && job4.isComplete()
                && job5.isComplete() && job6.isComplete())) {
            Thread.sleep(2000);
        }
    }
    FileSystem.get(firstJob.getConfiguration()).delete(new Path(INTERMEDIATE_PATH), true);
}

---

From source file:hr.fer.tel.rovkp.homework02.task03.Program.java

private static Job run(String pathIn, String pathOut)
        throws IOException, InterruptedException, ClassNotFoundException {
    Job nextJob = Job.getInstance();
    nextJob.setJarByClass(Program.class);
    nextJob.setJobName("TripTimes");

    FileInputFormat.addInputPath(nextJob, new Path(pathIn));
    FileOutputFormat.setOutputPath(nextJob, new Path(pathOut));

    nextJob.setMapperClass(TripTimesMapper.class);
    nextJob.setReducerClass(TripTimesReducer.class);

    nextJob.setOutputKeyClass(Text.class);
    nextJob.setOutputValueClass(TripTimesTuple.class);

    nextJob.submit();/*w  w  w. j av a 2  s.  c o m*/

    return nextJob;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »