Example usage for org.apache.hadoop.mapreduce Job isSuccessful

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job isSuccessful.

Prototype

public boolean isSuccessful() throws IOException 

Source Link

Document

Check if the job completed successfully.

Usage

From source file:org.apache.accumulo.examples.mapreduce.RowHash.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Job job = Job.getInstance(getConf());
    job.setJobName(this.getClass().getName());
    job.setJarByClass(this.getClass());
    Opts opts = new Opts();
    opts.parseArgs(RowHash.class.getName(), args);
    job.setInputFormatClass(AccumuloInputFormat.class);
    opts.setAccumuloConfigs(job);/*  www  .j  a va 2 s .c  om*/

    String col = opts.column;
    int idx = col.indexOf(":");
    Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
    Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
    if (cf.getLength() > 0)
        AccumuloInputFormat.fetchColumns(job, Collections.singleton(new Pair<>(cf, cq)));

    job.setMapperClass(HashDataMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Mutation.class);

    job.setNumReduceTasks(0);

    job.setOutputFormatClass(AccumuloOutputFormat.class);

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.examples.mapreduce.TableToFile.java

License:Apache License

@Override
public int run(String[] args)
        throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
    Job job = Job.getInstance(getConf());
    job.setJobName(this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());
    Opts opts = new Opts();
    opts.parseArgs(getClass().getName(), args);

    job.setInputFormatClass(AccumuloInputFormat.class);
    opts.setAccumuloConfigs(job);//www.j a v  a  2s .c  o m

    HashSet<Pair<Text, Text>> columnsToFetch = new HashSet<>();
    for (String col : opts.columns.split(",")) {
        int idx = col.indexOf(":");
        Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
        Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
        if (cf.getLength() > 0)
            columnsToFetch.add(new Pair<>(cf, cq));
    }
    if (!columnsToFetch.isEmpty())
        AccumuloInputFormat.fetchColumns(job, columnsToFetch);

    job.setMapperClass(TTFMapper.class);
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(Text.class);

    job.setNumReduceTasks(0);

    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, new Path(opts.output));

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.examples.mapreduce.TeraSortIngest.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Job job = Job.getInstance(getConf());
    job.setJobName("TeraSortCloud");
    job.setJarByClass(this.getClass());
    Opts opts = new Opts();
    opts.parseArgs(TeraSortIngest.class.getName(), args);

    job.setInputFormatClass(RangeInputFormat.class);
    job.setMapperClass(SortGenMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Mutation.class);

    job.setNumReduceTasks(0);/*from w w w. ja  v  a  2s  . c o m*/

    job.setOutputFormatClass(AccumuloOutputFormat.class);
    opts.setAccumuloConfigs(job);
    BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10L * 1000 * 1000);
    AccumuloOutputFormat.setBatchWriterOptions(job, bwConfig);

    Configuration conf = job.getConfiguration();
    conf.setLong(NUMROWS, opts.numRows);
    conf.setInt("cloudgen.minkeylength", opts.minKeyLength);
    conf.setInt("cloudgen.maxkeylength", opts.maxKeyLength);
    conf.setInt("cloudgen.minvaluelength", opts.minValueLength);
    conf.setInt("cloudgen.maxvaluelength", opts.maxValueLength);
    conf.set("cloudgen.tablename", opts.getTableName());

    if (opts.splits != 0)
        conf.setInt(NUMSPLITS, opts.splits);

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.examples.mapreduce.UniqueColumns.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(UniqueColumns.class.getName(), args);

    String jobName = this.getClass().getSimpleName() + "_" + System.currentTimeMillis();

    Job job = Job.getInstance(getConf());
    job.setJobName(jobName);//  w w  w  .jav a  2 s  .  c  om
    job.setJarByClass(this.getClass());

    String clone = opts.getTableName();
    Connector conn = null;

    opts.setAccumuloConfigs(job);

    if (opts.offline) {
        /*
         * this example clones the table and takes it offline. If you plan to run map reduce jobs over a table many times, it may be more efficient to compact the
         * table, clone it, and then keep using the same clone as input for map reduce.
         */

        conn = opts.getConnector();
        clone = opts.getTableName() + "_" + jobName;
        conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap<String, String>(),
                new HashSet<String>());
        conn.tableOperations().offline(clone);

        AccumuloInputFormat.setOfflineTableScan(job, true);
        AccumuloInputFormat.setInputTableName(job, clone);
    }

    job.setInputFormatClass(AccumuloInputFormat.class);

    job.setMapperClass(UMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setCombinerClass(UReducer.class);
    job.setReducerClass(UReducer.class);

    job.setNumReduceTasks(opts.reducers);

    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, new Path(opts.output));

    job.waitForCompletion(true);

    if (opts.offline) {
        conn.tableOperations().delete(clone);
    }

    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.examples.simple.filedata.CharacterHistogram.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Job job = JobUtil.getJob(getConf());
    job.setJobName(this.getClass().getSimpleName());
    job.setJarByClass(this.getClass());

    Opts opts = new Opts();
    opts.parseArgs(CharacterHistogram.class.getName(), args);

    job.setInputFormatClass(ChunkInputFormat.class);
    opts.setAccumuloConfigs(job);//  w w w  .jav a  2  s  .  c  om
    job.getConfiguration().set(VIS, opts.visibilities.toString());

    job.setMapperClass(HistMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Mutation.class);

    job.setNumReduceTasks(0);

    job.setOutputFormatClass(AccumuloOutputFormat.class);

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.examples.simple.mapreduce.NGramIngest.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(getClass().getName(), args);

    Job job = JobUtil.getJob(getConf());
    job.setJobName(getClass().getSimpleName());
    job.setJarByClass(getClass());//from  w ww .  j a  va2  s  .  c  om

    opts.setAccumuloConfigs(job);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(AccumuloOutputFormat.class);

    job.setMapperClass(NGramMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Mutation.class);

    job.setNumReduceTasks(0);
    job.setSpeculativeExecution(false);

    if (!opts.getConnector().tableOperations().exists(opts.getTableName())) {
        log.info("Creating table " + opts.getTableName());
        opts.getConnector().tableOperations().create(opts.getTableName());
        SortedSet<Text> splits = new TreeSet<Text>();
        String numbers[] = "1 2 3 4 5 6 7 8 9".split("\\s");
        String lower[] = "a b c d e f g h i j k l m n o p q r s t u v w x y z".split("\\s");
        String upper[] = "A B C D E F G H I J K L M N O P Q R S T U V W X Y Z".split("\\s");
        for (String[] array : new String[][] { numbers, lower, upper }) {
            for (String s : array) {
                splits.add(new Text(s));
            }
        }
        opts.getConnector().tableOperations().addSplits(opts.getTableName(), splits);
    }

    TextInputFormat.addInputPath(job, new Path(opts.inputDirectory));
    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.examples.simple.mapreduce.RegexExample.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(getClass().getName(), args);

    Job job = JobUtil.getJob(getConf());
    job.setJobName(getClass().getSimpleName());
    job.setJarByClass(getClass());//  w w w  .  j a  va2 s  . c o m

    job.setInputFormatClass(AccumuloInputFormat.class);
    opts.setAccumuloConfigs(job);

    IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class);
    RegExFilter.setRegexs(regex, opts.rowRegex, opts.columnFamilyRegex, opts.columnQualifierRegex,
            opts.valueRegex, false);
    AccumuloInputFormat.addIterator(job, regex);

    job.setMapperClass(RegexMapper.class);
    job.setMapOutputKeyClass(Key.class);
    job.setMapOutputValueClass(Value.class);

    job.setNumReduceTasks(0);

    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, new Path(opts.destination));

    System.out.println("setRowRegex: " + opts.rowRegex);
    System.out.println("setColumnFamilyRegex: " + opts.columnFamilyRegex);
    System.out.println("setColumnQualifierRegex: " + opts.columnQualifierRegex);
    System.out.println("setValueRegex: " + opts.valueRegex);

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.examples.simple.mapreduce.RowHash.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Job job = JobUtil.getJob(getConf());
    job.setJobName(this.getClass().getName());
    job.setJarByClass(this.getClass());
    Opts opts = new Opts();
    opts.parseArgs(RowHash.class.getName(), args);
    job.setInputFormatClass(AccumuloInputFormat.class);
    opts.setAccumuloConfigs(job);/* w  ww .  ja  v a 2s  .co m*/

    String col = opts.column;
    int idx = col.indexOf(":");
    Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
    Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
    if (cf.getLength() > 0)
        AccumuloInputFormat.fetchColumns(job, Collections.singleton(new Pair<Text, Text>(cf, cq)));

    job.setMapperClass(HashDataMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Mutation.class);

    job.setNumReduceTasks(0);

    job.setOutputFormatClass(AccumuloOutputFormat.class);

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.examples.simple.mapreduce.TableToFile.java

License:Apache License

@Override
public int run(String[] args)
        throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
    Job job = JobUtil.getJob(getConf());
    job.setJobName(this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());
    Opts opts = new Opts();
    opts.parseArgs(getClass().getName(), args);

    job.setInputFormatClass(AccumuloInputFormat.class);
    opts.setAccumuloConfigs(job);/*from ww  w.j a va 2s.c  o  m*/

    HashSet<Pair<Text, Text>> columnsToFetch = new HashSet<Pair<Text, Text>>();
    for (String col : opts.columns.split(",")) {
        int idx = col.indexOf(":");
        Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
        Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
        if (cf.getLength() > 0)
            columnsToFetch.add(new Pair<Text, Text>(cf, cq));
    }
    if (!columnsToFetch.isEmpty())
        AccumuloInputFormat.fetchColumns(job, columnsToFetch);

    job.setMapperClass(TTFMapper.class);
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(Text.class);

    job.setNumReduceTasks(0);

    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, new Path(opts.output));

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Job job = JobUtil.getJob(getConf());
    job.setJobName("TeraSortCloud");
    job.setJarByClass(this.getClass());
    Opts opts = new Opts();
    opts.parseArgs(TeraSortIngest.class.getName(), args);

    job.setInputFormatClass(RangeInputFormat.class);
    job.setMapperClass(SortGenMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Mutation.class);

    job.setNumReduceTasks(0);//from   w w  w . ja  v a2s . co  m

    job.setOutputFormatClass(AccumuloOutputFormat.class);
    opts.setAccumuloConfigs(job);
    BatchWriterConfig bwConfig = new BatchWriterConfig().setMaxMemory(10L * 1000 * 1000);
    AccumuloOutputFormat.setBatchWriterOptions(job, bwConfig);

    Configuration conf = job.getConfiguration();
    conf.setLong(NUMROWS, opts.numRows);
    conf.setInt("cloudgen.minkeylength", opts.minKeyLength);
    conf.setInt("cloudgen.maxkeylength", opts.maxKeyLength);
    conf.setInt("cloudgen.minvaluelength", opts.minValueLength);
    conf.setInt("cloudgen.maxvaluelength", opts.maxValueLength);
    conf.set("cloudgen.tablename", opts.getTableName());

    if (args.length > 10)
        conf.setInt(NUMSPLITS, opts.splits);

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• »