Example usage for org.apache.hadoop.mapreduce Job isSuccessful

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job isSuccessful.

Prototype

public boolean isSuccessful() throws IOException 

Source Link

Document

Check if the job completed successfully.

Usage

From source file:org.apache.accumulo.examples.simple.mapreduce.UniqueColumns.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(UniqueColumns.class.getName(), args);

    String jobName = this.getClass().getSimpleName() + "_" + System.currentTimeMillis();

    Job job = JobUtil.getJob(getConf());
    job.setJobName(jobName);/*w  w  w  . j ava2  s  . co m*/
    job.setJarByClass(this.getClass());

    String clone = opts.getTableName();
    Connector conn = null;

    opts.setAccumuloConfigs(job);

    if (opts.offline) {
        /*
         * this example clones the table and takes it offline. If you plan to run map reduce jobs over a table many times, it may be more efficient to compact the
         * table, clone it, and then keep using the same clone as input for map reduce.
         */

        conn = opts.getConnector();
        clone = opts.getTableName() + "_" + jobName;
        conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap<String, String>(),
                new HashSet<String>());
        conn.tableOperations().offline(clone);

        AccumuloInputFormat.setOfflineTableScan(job, true);
        AccumuloInputFormat.setInputTableName(job, clone);
    }

    job.setInputFormatClass(AccumuloInputFormat.class);

    job.setMapperClass(UMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setCombinerClass(UReducer.class);
    job.setReducerClass(UReducer.class);

    job.setNumReduceTasks(opts.reducers);

    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, new Path(opts.output));

    job.waitForCompletion(true);

    if (opts.offline) {
        conn.tableOperations().delete(clone);
    }

    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.server.master.LogSort.java

License:Apache License

public static void main(String[] args) throws Exception {
    long startTime = System.currentTimeMillis();
    log.info("Job started");
    Job job = startSort(false, args);
    log.info("The job finished after " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds.");
    if (!job.isSuccessful())
        System.exit(1);/*from w ww.  j a va 2 s .  co m*/
}

---

From source file:org.apache.accumulo.server.metanalysis.FilterMeta.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    String jobName = this.getClass().getSimpleName() + "_" + System.currentTimeMillis();

    Job job = new Job(getConf(), jobName);
    job.setJarByClass(this.getClass());

    Path paths[] = new Path[args.length - 1];
    for (int i = 0; i < paths.length; i++) {
        paths[i] = new Path(args[i]);
    }//from   www .  j  av a2  s  .co m

    job.setInputFormatClass(LogFileInputFormat.class);
    LogFileInputFormat.setInputPaths(job, paths);

    job.setOutputFormatClass(LogFileOutputFormat.class);
    LogFileOutputFormat.setOutputPath(job, new Path(args[args.length - 1]));

    job.setMapperClass(FilterMapper.class);

    job.setNumReduceTasks(0);

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.server.metanalysis.IndexMeta.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(IndexMeta.class.getName(), args);

    String jobName = this.getClass().getSimpleName() + "_" + System.currentTimeMillis();

    Job job = new Job(getConf(), jobName);
    job.setJarByClass(this.getClass());

    List<String> logFiles = Arrays.asList(args).subList(4, args.length);
    Path paths[] = new Path[logFiles.size()];
    int count = 0;
    for (String logFile : logFiles) {
        paths[count++] = new Path(logFile);
    }/*from w w  w.  j ava 2s. c  o m*/

    job.setInputFormatClass(LogFileInputFormat.class);
    LogFileInputFormat.setInputPaths(job, paths);

    job.setNumReduceTasks(0);

    job.setOutputFormatClass(AccumuloOutputFormat.class);
    AccumuloOutputFormat.setZooKeeperInstance(job, opts.instance, opts.zookeepers);
    AccumuloOutputFormat.setConnectorInfo(job, opts.principal, opts.getToken());
    AccumuloOutputFormat.setCreateTables(job, false);

    job.setMapperClass(IndexMapper.class);

    Connector conn = opts.getConnector();

    try {
        conn.tableOperations().create("createEvents");
    } catch (TableExistsException tee) {
        Logger.getLogger(IndexMeta.class).warn("Table createEvents exists");
    }

    try {
        conn.tableOperations().create("tabletEvents");
    } catch (TableExistsException tee) {
        Logger.getLogger(IndexMeta.class).warn("Table tabletEvents exists");
    }

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.server.test.continuous.ContinuousMoru.java

License:Apache License

@Override
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    if (args.length != 13) {
        throw new IllegalArgumentException("Usage : " + ContinuousMoru.class.getName()
                + " <instance name> <zookeepers> <user> <pass> <table> <min> <max> <max cf> <max cq> <max mem> <max latency> <num threads> <max maps>");
    }/*from   w  w w  .j  a v  a 2 s.  co m*/

    String instance = args[0];
    String zookeepers = args[1];
    String user = args[2];
    String pass = args[3];
    String table = args[4];
    String min = args[5];
    String max = args[6];
    String max_cf = args[7];
    String max_cq = args[8];
    String maxMem = args[9];
    String maxLatency = args[10];
    String numThreads = args[11];
    String maxMaps = args[12];

    Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());

    job.setInputFormatClass(AccumuloInputFormat.class);
    AccumuloInputFormat.setInputInfo(job.getConfiguration(), user, pass.getBytes(), table,
            new Authorizations());
    AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), instance, zookeepers);

    // set up ranges
    try {
        AccumuloInputFormat.setRanges(job.getConfiguration(),
                new ZooKeeperInstance(instance, zookeepers).getConnector(user, pass.getBytes())
                        .tableOperations().splitRangeByTablets(table, new Range(), Integer.parseInt(maxMaps)));
        AccumuloInputFormat.disableAutoAdjustRanges(job.getConfiguration());
    } catch (Exception e) {
        throw new IOException(e);
    }

    job.setMapperClass(CMapper.class);

    job.setNumReduceTasks(0);

    job.setOutputFormatClass(AccumuloOutputFormat.class);
    Configuration conf = job.getConfiguration();
    AccumuloOutputFormat.setOutputInfo(conf, user, pass.getBytes(), false, table);
    AccumuloOutputFormat.setZooKeeperInstance(conf, instance, zookeepers);
    AccumuloOutputFormat.setMaxLatency(conf, (int) (Integer.parseInt(maxLatency) / 1000.0));
    AccumuloOutputFormat.setMaxMutationBufferSize(conf, Long.parseLong(maxMem));
    AccumuloOutputFormat.setMaxWriteThreads(conf, Integer.parseInt(numThreads));

    conf.setLong(MIN, Long.parseLong(min));
    conf.setLong(MAX, Long.parseLong(max));
    conf.setInt(MAX_CF, Integer.parseInt(max_cf));
    conf.setInt(MAX_CQ, Integer.parseInt(max_cq));
    conf.set(CI_ID, UUID.randomUUID().toString());

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.server.test.continuous.ContinuousVerify.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 9) {
        throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName()
                + " <instance name> <zookeepers> <user> <pass> <table> <output dir> <max mappers> <num reducers> <scan offline>");
    }//  www .j a v  a 2  s. c o m

    String instance = args[0];
    String zookeepers = args[1];
    String user = args[2];
    String pass = args[3];
    String table = args[4];
    String outputdir = args[5];
    String maxMaps = args[6];
    String reducers = args[7];
    boolean scanOffline = Boolean.parseBoolean(args[8]);

    Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());

    String clone = table;
    Connector conn = null;
    if (scanOffline) {
        Random random = new Random();
        clone = table + "_" + String.format("%016x", Math.abs(random.nextLong()));
        ZooKeeperInstance zki = new ZooKeeperInstance(instance, zookeepers);
        conn = zki.getConnector(user, pass.getBytes());
        conn.tableOperations().clone(table, clone, true, new HashMap<String, String>(), new HashSet<String>());
        conn.tableOperations().offline(clone);
    }

    job.setInputFormatClass(AccumuloInputFormat.class);
    AccumuloInputFormat.setInputInfo(job.getConfiguration(), user, pass.getBytes(), clone,
            new Authorizations());
    AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), instance, zookeepers);
    AccumuloInputFormat.setScanOffline(job.getConfiguration(), scanOffline);
    // set up ranges
    try {
        Set<Range> ranges = new ZooKeeperInstance(instance, zookeepers).getConnector(user, pass.getBytes())
                .tableOperations().splitRangeByTablets(table, new Range(), Integer.parseInt(maxMaps));
        AccumuloInputFormat.setRanges(job.getConfiguration(), ranges);
        AccumuloInputFormat.disableAutoAdjustRanges(job.getConfiguration());
    } catch (Exception e) {
        throw new IOException(e);
    }

    job.setMapperClass(CMapper.class);
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(VLongWritable.class);

    job.setReducerClass(CReducer.class);
    job.setNumReduceTasks(Integer.parseInt(reducers));

    job.setOutputFormatClass(TextOutputFormat.class);

    job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);

    TextOutputFormat.setOutputPath(job, new Path(outputdir));

    job.waitForCompletion(true);

    if (scanOffline) {
        conn.tableOperations().delete(clone);
    }

    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.server.test.randomwalk.multitable.CopyTool.java

License:Apache License

public int run(String[] args) throws Exception {
    Job job = new Job(getConf(), this.getClass().getSimpleName());
    job.setJarByClass(this.getClass());

    if (job.getJar() == null) {
        log.error("M/R requires a jar file!  Run mvn package.");
        return 1;
    }/*  w  w  w  .j  av  a2 s .co  m*/

    job.setInputFormatClass(AccumuloInputFormat.class);
    AccumuloInputFormat.setInputInfo(job.getConfiguration(), args[0], args[1].getBytes(), args[2],
            new Authorizations());
    AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), args[3], args[4]);

    job.setMapperClass(SeqMapClass.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Mutation.class);

    job.setNumReduceTasks(0);

    job.setOutputFormatClass(AccumuloOutputFormat.class);
    AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[0], args[1].getBytes(), true, args[5]);
    AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[3], args[4]);

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.server.test.randomwalk.sequential.MapRedVerifyTool.java

License:Apache License

public int run(String[] args) throws Exception {
    Job job = new Job(getConf(), this.getClass().getSimpleName());
    job.setJarByClass(this.getClass());

    if (job.getJar() == null) {
        log.error("M/R requires a jar file!  Run mvn package.");
        return 1;
    }//  w w  w  .  j a  v  a  2  s  . c o m

    job.setInputFormatClass(AccumuloInputFormat.class);
    AccumuloInputFormat.setInputInfo(job.getConfiguration(), args[0], args[1].getBytes(), args[2],
            new Authorizations());
    AccumuloInputFormat.setZooKeeperInstance(job.getConfiguration(), args[3], args[4]);

    job.setMapperClass(SeqMapClass.class);
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(IntWritable.class);

    job.setReducerClass(SeqReduceClass.class);
    job.setNumReduceTasks(1);

    job.setOutputFormatClass(AccumuloOutputFormat.class);
    AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[0], args[1].getBytes(), true, args[5]);
    AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[3], args[4]);

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.server.test.randomwalk.shard.SortTool.java

License:Apache License

public int run(String[] args) throws Exception {
    Job job = new Job(getConf(), this.getClass().getSimpleName());
    job.setJarByClass(this.getClass());

    if (job.getJar() == null) {
        log.error("M/R requires a jar file!  Run mvn package.");
        return 1;
    }/*from  w  ww  .j av a 2 s  . c o  m*/

    job.setInputFormatClass(SequenceFileInputFormat.class);
    SequenceFileInputFormat.setInputPaths(job, seqFile);

    job.setPartitionerClass(KeyRangePartitioner.class);
    KeyRangePartitioner.setSplitFile(job, splitFile);

    job.setMapOutputKeyClass(Key.class);
    job.setMapOutputValueClass(Value.class);

    job.setNumReduceTasks(splits.size() + 1);

    job.setOutputFormatClass(AccumuloFileOutputFormat.class);
    AccumuloFileOutputFormat.setOutputPath(job, new Path(outputDir));

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

From source file:org.apache.accumulo.server.util.CountRowKeys.java

License:Apache License

@Override
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    if (args.length != 2) {
        System.out.println("Usage: CountRowKeys tableName outputPath");
        return 1;
    }//  w  ww  .j a  va2s.  c  o m

    Job job = new Job(getConf(), this.getClass().getName());
    job.setJarByClass(this.getClass());

    job.setInputFormatClass(SequenceFileInputFormat.class);
    SequenceFileInputFormat.addInputPath(job,
            new Path(ServerConstants.getTablesDir() + "/" + args[0] + "/*/*/data"));

    job.setMapperClass(MyMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(NullWritable.class);

    job.setReducerClass(MyReducer.class);

    TextOutputFormat.setOutputPath(job, new Path(args[1]));

    job.waitForCompletion(true);
    return job.isSuccessful() ? 0 : 1;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• »