Example usage for org.apache.hadoop.mapreduce MRJobConfig MAPREDUCE_JOB_USER_CLASSPATH

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce MRJobConfig MAPREDUCE_JOB_USER_CLASSPATH_FIRST.

Prototype

String MAPREDUCE_JOB_USER_CLASSPATH_FIRST

To view the source code for org.apache.hadoop.mapreduce MRJobConfig MAPREDUCE_JOB_USER_CLASSPATH_FIRST.

Click Source Link

Usage

From source file:mvm.rya.joinselect.mr.JoinSelectAggregate.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    String inPath1 = conf.get(PROSPECTS_OUTPUTPATH);
    String inPath2 = conf.get(SPO_OUTPUTPATH);
    String auths = conf.get(AUTHS);
    String outPath = conf.get(OUTPUTPATH);

    assert inPath1 != null && inPath2 != null && outPath != null;

    Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());
    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);

    JoinSelectStatsUtil.initJoinMRJob(job, inPath1, inPath2, JoinSelectAggregateMapper.class, outPath, auths);

    job.setSortComparatorClass(JoinSelectSortComparator.class);
    job.setGroupingComparatorClass(JoinSelectGroupComparator.class);
    job.setPartitionerClass(JoinSelectPartitioner.class);
    job.setReducerClass(JoinReducer.class);
    job.setNumReduceTasks(32);/* w w  w. j  a v a 2s  .c  o m*/
    job.waitForCompletion(true);

    return job.isSuccessful() ? 0 : 1;

}

From source file:mvm.rya.joinselect.mr.JoinSelectProspectOutput.java

License:Apache License

@Override
public int run(String[] args)
        throws AccumuloSecurityException, IOException, ClassNotFoundException, InterruptedException {

    Configuration conf = getConf();
    String inTable = conf.get(PROSPECTS_TABLE);
    String auths = conf.get(AUTHS);
    String outPath = conf.get(PROSPECTS_OUTPUTPATH);

    assert inTable != null && outPath != null;

    Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());
    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);

    JoinSelectStatsUtil.initTabToSeqFileJob(job, inTable, outPath, auths);
    job.setMapperClass(CardinalityMapper.class);

    job.setNumReduceTasks(0);/* w w w  .  java  2s . co  m*/

    job.waitForCompletion(true);

    return job.isSuccessful() ? 0 : 1;

}

From source file:mvm.rya.joinselect.mr.JoinSelectSpoTableOutput.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    Configuration conf = getConf();
    String inTable = conf.get(SPO_TABLE);
    String auths = conf.get(AUTHS);
    String outPath = conf.get(SPO_OUTPUTPATH);

    assert inTable != null && outPath != null;

    Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());
    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);

    JoinSelectStatsUtil.initTabToSeqFileJob(job, inTable, outPath, auths);
    job.setMapperClass(JoinSelectMapper.class);
    job.setNumReduceTasks(0);/*from  www .  j  ava  2s.  co  m*/
    job.waitForCompletion(true);

    return job.isSuccessful() ? 0 : 1;

}

From source file:org.apache.accumulo.test.mrit.IntegrationTestMapReduce.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    // read a list of tests from the input, and print out the results
    if (args.length != 2) {
        System.err.println("Wrong number of args: <input> <output>");
        return 1;
    }//from w ww. j  a  v  a  2s .c  o m
    Configuration conf = getConf();
    Job job = Job.getInstance(conf, "accumulo integration test runner");
    conf = job.getConfiguration();

    // some tests take more than 10 minutes
    conf.setLong(MRJobConfig.TASK_TIMEOUT, 20 * 60 * 1000);

    // minicluster uses a lot of ram
    conf.setInt(MRJobConfig.MAP_MEMORY_MB, 4000);

    // hadoop puts an ancient version of jline on the classpath
    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);

    // no need to run a test multiple times
    job.setSpeculativeExecution(false);

    // read one line at a time
    job.setInputFormatClass(NLineInputFormat.class);
    NLineInputFormat.setNumLinesPerSplit(job, 1);

    // run the test
    job.setJarByClass(IntegrationTestMapReduce.class);
    job.setMapperClass(TestMapper.class);

    // group test by result code
    job.setReducerClass(TestReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:org.apache.rya.joinselect.mr.JoinSelectProspectOutput.java

License:Apache License

@Override
public int run(final String[] args)
        throws AccumuloSecurityException, IOException, ClassNotFoundException, InterruptedException {

    final Configuration conf = getConf();
    final String inTable = conf.get(PROSPECTS_TABLE);
    final String auths = conf.get(AUTHS);
    final String outPath = conf.get(PROSPECTS_OUTPUTPATH);

    assert inTable != null && outPath != null;

    final Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());
    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);

    JoinSelectStatsUtil.initTabToSeqFileJob(job, inTable, outPath, auths);
    job.setMapperClass(CardinalityMapper.class);

    job.setNumReduceTasks(0);//from w  ww . java  2  s . c o m

    job.waitForCompletion(true);

    return job.isSuccessful() ? 0 : 1;

}

Example usage for org.apache.hadoop.mapreduce MRJobConfig MAPREDUCE_JOB_USER_CLASSPATH_FIRST

Introduction

Prototype

Usage