Example usage for org.apache.hadoop.mapreduce Job submit

List of usage examples for org.apache.hadoop.mapreduce Job submit

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. submit

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job submit.

Prototype

public void submit() throws IOException, InterruptedException, ClassNotFoundException

Source Link

Document

Submit the job to the cluster and return immediately.

Usage

From source file:gobblin.runtime.mapreduce.MRTask.java

License:Apache License

@Override
public void run() {

    try {//from  w w  w. j av  a  2s  .c  o  m
        Job job = createJob();

        job.submit();
        this.eventSubmitter.submit(Events.MR_JOB_STARTED_EVENT, Events.JOB_URL, job.getTrackingURL());
        job.waitForCompletion(false);

        if (job.isSuccessful()) {
            this.eventSubmitter.submit(Events.MR_JOB_SUCCESSFUL, Events.JOB_URL, job.getTrackingURL());
            this.workingState = WorkUnitState.WorkingState.SUCCESSFUL;
        } else {
            this.eventSubmitter.submit(Events.MR_JOB_FAILED, Events.JOB_URL, job.getTrackingURL());
            this.workingState = WorkUnitState.WorkingState.FAILED;
        }
    } catch (Throwable t) {
        log.error("Failed to run MR job.", t);
        this.eventSubmitter.submit(Events.MR_JOB_FAILED, Events.FAILURE_CONTEXT, t.getMessage());
        this.workingState = WorkUnitState.WorkingState.FAILED;
    }
}

From source file:hr.fer.tel.rovkp.homework02.task03.Program.java

private static Job run(String pathIn, String pathOut)
        throws IOException, InterruptedException, ClassNotFoundException {
    Job nextJob = Job.getInstance();
    nextJob.setJarByClass(Program.class);
    nextJob.setJobName("TripTimes");

    FileInputFormat.addInputPath(nextJob, new Path(pathIn));
    FileOutputFormat.setOutputPath(nextJob, new Path(pathOut));

    nextJob.setMapperClass(TripTimesMapper.class);
    nextJob.setReducerClass(TripTimesReducer.class);

    nextJob.setOutputKeyClass(Text.class);
    nextJob.setOutputValueClass(TripTimesTuple.class);

    nextJob.submit();

    return nextJob;
}

From source file:io.covert.binary.analysis.BinaryAnalysisJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    if (args.length != 2) {
        usage("");
    }/*from w  w  w. j a v  a 2s  . co m*/

    String inDir = args[0];
    String outDir = args[1];

    Configuration conf = getConf();
    for (String name : requiredSettings) {
        if (conf.get(name) == null)
            usage("Missing required setting: " + name);
    }

    Job job = new Job(conf);
    job.setJobName(BinaryAnalysisJob.class.getName() + " inDir=" + inDir + ", outDir=" + outDir);
    job.setJarByClass(getClass());

    job.setMapperClass(BinaryAnalysisMapper.class);
    job.setNumReduceTasks(0);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(BytesWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(SequenceFileInputFormat.class);
    SequenceFileInputFormat.setInputPaths(job, new Path(inDir));

    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    SequenceFileOutputFormat.setOutputPath(job, new Path(outDir));
    job.submit();

    int retVal = job.waitForCompletion(true) ? 0 : 1;
    return retVal;
}

From source file:io.covert.dns.collection.CollectionJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    if (args.length != 4) {
        usage("");
    }//from w w  w  .j a  va2  s .c o m

    String dclass = args[0];
    String types = args[1];
    String inDir = args[2];
    String outDir = args[3];

    Configuration conf = getConf();

    if (conf.get("dns.collection.num.resolvers") == null)
        conf.setInt("dns.collection.num.resolvers", 50);
    if (conf.get("dns.collection.nameservers") == null)
        conf.set("dns.collection.nameservers", "127.0.0.1");

    Job job = new Job(conf);
    job.setJobName(CollectionJob.class.getSimpleName() + ": types=" + types + ", dclass=" + dclass + " inDir="
            + inDir + ", outDir=" + outDir + ", resolvers=" + conf.get("dns.collection.nameservers"));
    job.setJarByClass(getClass());

    job.setMapperClass(CollectionMapper.class);
    job.setNumReduceTasks(0);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(BytesWritable.class);

    job.setInputFormatClass(DnsRequestInputFormat.class);
    DnsRequestInputFormat.setInputPaths(job, new Path(inDir));
    DnsRequestInputFormat.configure(job, dclass.toUpperCase(), Arrays.asList(types.split(",")),
            Arrays.asList(""));

    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    SequenceFileOutputFormat.setOutputPath(job, new Path(outDir));
    SequenceFileOutputFormat.setCompressOutput(job, true);
    job.submit();

    int retVal = job.waitForCompletion(true) ? 0 : 1;

    CounterGroup counters = job.getCounters().getGroup(CollectionMapper.RESOLVER_GROUP);
    Counter constructMessageMS = counters.findCounter(CollectionMapper.CONSTRUCT_MESSAGE_MS);
    Counter parseResponseMS = counters.findCounter(CollectionMapper.PARSE_RESPONSE_MS);
    Counter performRequestMS = counters.findCounter(CollectionMapper.PERFORM_REQUEST_MS);
    Counter totalRequestHandlingMS = counters.findCounter(CollectionMapper.TOTAL_REQUEST_HANDLING_MS);

    Log.info("Total ConstructMessage percent: "
            + (double) (constructMessageMS.getValue() * 100L) / ((double) totalRequestHandlingMS.getValue()));
    Log.info("Total ParseResponse percent:    "
            + (double) (parseResponseMS.getValue() * 100L) / ((double) totalRequestHandlingMS.getValue()));
    Log.info("Total PerformRequest percent:   "
            + (double) (performRequestMS.getValue() * 100L) / ((double) totalRequestHandlingMS.getValue()));

    return retVal;
}

From source file:io.covert.dns.extract.ExtractorJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    if (args.length != 3) {
        usage("");
    }//from w ww .ja  v  a  2s .com

    String expression = args[0];
    String inDir = args[1];
    String outDir = args[2];

    Configuration conf = getConf();
    conf.set(ExtractorMapper.EXTRACTOR_JEXL_EXPRESSION, expression);

    Job job = new Job(conf);
    job.setJobName(ExtractorJob.class.getSimpleName() + ": inDir=" + inDir + ", outDir=" + outDir
            + ", expression=[" + expression + "]");
    job.setJarByClass(getClass());

    job.setMapperClass(ExtractorMapper.class);
    job.setReducerClass(UniqueKeyOnlyReducer.class);
    job.setNumReduceTasks(new JobClient(new JobConf(conf)).getClusterStatus().getTaskTrackers());

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(SequenceFileInputFormat.class);
    FileInputFormat.setInputPaths(job, new Path(inDir));

    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    SequenceFileOutputFormat.setOutputPath(job, new Path(outDir));
    SequenceFileOutputFormat.setCompressOutput(job, true);
    job.submit();

    int retVal = job.waitForCompletion(true) ? 0 : 1;
    return retVal;
}

From source file:io.covert.dns.filtering.FilterJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    if (args.length != 3) {
        usage("");
    }//  ww w .  ja va  2  s  .c  o m

    String filter = args[0];
    String inDir = args[1];
    String outDir = args[2];

    Configuration conf = getConf();
    conf.set(FilterMapper.FILTER_JEXL_EXPRESSION, filter);

    Job job = new Job(conf);
    job.setJobName(FilterJob.class.getSimpleName() + ": inDir=" + inDir + ", outDir=" + outDir + ", filter=["
            + filter + "]");
    job.setJarByClass(getClass());

    job.setMapperClass(FilterMapper.class);
    job.setReducerClass(Reducer.class); // Identity Reduce...
    job.setNumReduceTasks(new JobClient(new JobConf(conf)).getClusterStatus().getTaskTrackers());

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(SequenceFileInputFormat.class);
    FileInputFormat.setInputPaths(job, new Path(inDir));

    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    SequenceFileOutputFormat.setOutputPath(job, new Path(outDir));
    SequenceFileOutputFormat.setCompressOutput(job, true);
    job.submit();

    int retVal = job.waitForCompletion(true) ? 0 : 1;
    return retVal;
}

From source file:io.covert.dns.geo.GeoJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    if (args.length != 4) {
        usage("");
    }//  w  w  w .  ja  v  a2  s  . c  om

    String dbfile = args[0];
    String asnDbfile = args[1];
    String inDir = args[2];
    String outDir = args[3];

    Configuration conf = getConf();
    conf.set("maxmind.geo.database.file", dbfile);
    conf.set("maxmind.asn.database.file", asnDbfile);

    Job job = new Job(conf);
    job.setJobName(GeoJob.class.getSimpleName() + ": dbfile=" + dbfile + ", asnDB=" + asnDbfile + " inDir="
            + inDir + ", outDir=" + outDir);
    job.setJarByClass(getClass());

    job.setMapperClass(GeoMapper.class);
    job.setNumReduceTasks(0);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(SequenceFileInputFormat.class);
    FileInputFormat.setInputPaths(job, new Path(inDir));

    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    SequenceFileOutputFormat.setOutputPath(job, new Path(outDir));
    SequenceFileOutputFormat.setCompressOutput(job, true);
    job.submit();

    int retVal = job.waitForCompletion(true) ? 0 : 1;
    return retVal;
}

From source file:io.covert.dns.parse.ParseJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    String inDir = args[0];//from   w w  w . ja  v  a 2 s. c o  m
    String outDir = args[1];

    Configuration conf = getConf();

    Job job = new Job(conf);
    job.setJobName(ParseJob.class.getSimpleName() + ": inDir=" + inDir + ", outDir=" + outDir);
    job.setJarByClass(getClass());

    job.setMapperClass(ParseMapper.class);
    job.setReducerClass(UniqueKeyOnlyReducer.class);
    job.setNumReduceTasks(new JobClient(new JobConf(conf)).getClusterStatus().getTaskTrackers());

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(SequenceFileInputFormat.class);
    FileInputFormat.setInputPaths(job, new Path(inDir));

    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    SequenceFileOutputFormat.setOutputPath(job, new Path(outDir));
    SequenceFileOutputFormat.setCompressOutput(job, true);
    job.submit();

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:io.covert.dns.storage.StorageJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    String inDir = args[0];/*  w  w w  .j  av  a 2 s.  c  o  m*/
    Configuration conf = getConf();
    Job job = new Job(conf);
    job.setJarByClass(getClass());
    job.setJobName(StorageJob.class.getSimpleName() + ": inDir=" + inDir);

    job.setMapperClass(StorageMapper.class);
    job.setNumReduceTasks(0);

    job.setInputFormatClass(SequenceFileInputFormat.class);
    FileInputFormat.setInputPaths(job, new Path(inDir));

    // This job doesn't write output via Hadoop, it uses the configured storage modules
    job.setOutputFormatClass(NullOutputFormat.class);
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(NullWritable.class);
    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(NullWritable.class);

    job.submit();

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:io.covert.util.FileFormatToConverterJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    if (args.length != 1) {
        usage("");
    }//  ww w  . j  a  v a  2  s .  c om

    String inDir = args[0];

    Configuration conf = getConf();

    if (conf.get("stream.process.command") == null) {
        conf.set("stream.process.command", "/opt/decompress.sh");
    }

    Job job = new Job(conf);
    job.setJobName(FileFormatToConverterJob.class.getName() + " inDir=" + inDir);
    job.setJarByClass(getClass());

    job.setMapperClass(ConvertMapper.class);
    job.setNumReduceTasks(0);

    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(NullWritable.class);
    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(NullWritable.class);

    job.setInputFormatClass(TextInputFormat.class);
    TextInputFormat.setInputPaths(job, new Path(inDir));

    job.setOutputFormatClass(NullOutputFormat.class);
    job.submit();

    int retVal = job.waitForCompletion(true) ? 0 : 1;
    return retVal;
}