Example usage for org.apache.hadoop.mapreduce Job setJarByClass

List of usage examples for org.apache.hadoop.mapreduce Job setJarByClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. setJarByClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setJarByClass.

Prototype

public void setJarByClass(Class<?> cls)

Source Link

Document

Set the Jar by finding where a given class came from.

Usage

From source file:Assignment2_P3_GenderMovieCount.GenderMovieRatingDriver.java

/**
 * @param args the command line arguments
 *///from   w w  w. j a v a2  s .c o  m
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Gender Movie Rating Count");
    job.setJarByClass(GenderMovieRatingDriver.class);
    job.setMapperClass(GenderMovieRating_Mapper.class);
    job.setCombinerClass(GenderMovieRating_Reducer.class);
    job.setReducerClass(GenderMovieRating_Reducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:Assignment2_P4_MovieRatingCount.MovieRatingDriver.java

/**
 * @param args the command line arguments
 *//*  ww  w . java  2s  .c  o m*/
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Movie Rating Count");
    job.setJarByClass(MovieRatingDriver.class);
    job.setMapperClass(MovieRating_Mapper.class);
    job.setCombinerClass(MovieRating_Reducer.class);
    job.setReducerClass(MovieRating_Reducer.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:Assignment2_P5_IPAddressCount.IPAddressDriver.java

/**
 * @param args the command line arguments
 *///from  ww w.  j  a  v a2 s .c om
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "IP Address Count");
    job.setJarByClass(IPAddressDriver.class);
    job.setMapperClass(IPAddress_Mapper.class);
    job.setCombinerClass(IPAddress_Reducer.class);
    job.setReducerClass(IPAddress_Reducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:Assignment3_P2_MergeStockAverageCount.StockPriceMergeDriver.java

/**
 * @param args the command line arguments
 *//*from w ww.  j a  va  2 s  .  com*/
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();

    // local file system handle
    FileSystem local = FileSystem.getLocal(conf);

    // hdfs file system handle
    FileSystem hdfs = FileSystem.get(conf);

    // local input directory
    Path inputDir = new Path(args[0]);

    // hdfs i/p  directory
    Path inputDir1 = new Path(args[1]);

    // local input files in local dir
    FileStatus[] inputFiles = local.listStatus(inputDir);

    // o/p stream
    FSDataOutputStream out = hdfs.create(inputDir1);

    // open each file and extract contents of file
    for (int i = 0; i < inputFiles.length; i++) {
        System.out.println("File name ----------------------------------------------------------------> "
                + inputFiles[i].getPath().getName());
        FSDataInputStream in = local.open(inputFiles[i].getPath());
        byte buffer[] = new byte[256];
        int bytesRead = 0;

        // extract all contents of file
        while ((bytesRead = in.read(buffer)) > 0) {
            out.write(buffer, 0, bytesRead);
        }

        // close input stream
        in.close();
    }

    Job job = Job.getInstance(conf, "Average Stock Price");
    job.setJarByClass(StockPriceMergeDriver.class);
    job.setMapperClass(StockPriceMerge_Mapper.class);
    job.setCombinerClass(StockPriceMerge_Reducer.class);
    job.setReducerClass(StockPriceMerge_Reducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(FloatWritable.class);
    FileInputFormat.addInputPath(job, new Path(args[1])); // above programs output will be input for mapper
    FileOutputFormat.setOutputPath(job, new Path(args[2]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:Assignment3_P4_DateStock.DateStockDriver.java

/**
 * @param args the command line arguments
 *///from   www  .  j a  v a2  s . c  o m
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Average Stock Price");
    job.setJarByClass(DateStockDriver.class);
    job.setMapperClass(DateStock_Mapper.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(DateStock_CompositeValueWritable.class);

    job.setReducerClass(DateStock_Reducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:Assignment3_P5_Top25Movies.Top25MovieRatingDriver.java

/**
 * @param args the command line arguments
 *//*from  www.j av  a 2  s . com*/
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job1 = Job.getInstance(conf, "Movie Rating Count");
    job1.setJarByClass(Top25MovieRatingDriver.class);

    // the usual - get basic mapred ready
    job1.setMapperClass(Top25MovieRating_Mapper.class);
    job1.setCombinerClass(Top25MovieRating_Reducer.class);
    job1.setReducerClass(Top25MovieRating_Reducer.class);

    // this will basically out -> movieId, average rating
    job1.setOutputKeyClass(IntWritable.class);
    job1.setOutputValueClass(FloatWritable.class);

    FileInputFormat.addInputPath(job1, new Path(args[0]));
    FileOutputFormat.setOutputPath(job1, new Path(args[1]));
    boolean complete = job1.waitForCompletion(true);

    // here's where we sort
    Configuration conf2 = new Configuration();
    Job job2 = Job.getInstance(conf2, "Movie Rating Count");
    if (complete) {
        job2.setJarByClass(Top25MovieRatingDriver.class);

        // namesake fellow, take it and go types - mostly useless
        job2.setMapperClass(Top25MovieRating_Mapper1.class);
        job2.setMapOutputKeyClass(FloatWritable.class);
        job2.setMapOutputValueClass(IntWritable.class);

        // this is where we would ideally sort descendingly
        job2.setSortComparatorClass(Top25MovieRating_SortComparator.class);

        // o/p top 25, man
        job2.setNumReduceTasks(1);
        job2.setReducerClass(Top25MovieRating_Reducer1.class);
        job2.setOutputKeyClass(FloatWritable.class);
        job2.setOutputValueClass(IntWritable.class);

        FileInputFormat.addInputPath(job2, new Path(args[1]));
        FileOutputFormat.setOutputPath(job2, new Path(args[2]));
        System.exit(job2.waitForCompletion(true) ? 0 : 1);
    }
}

From source file:Assignment4_P2_StockAverageWithCombiner.StockAverageDriver.java

/**
 * @param args the command line arguments
 *///  ww w .j  a va  2 s.  c o  m
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Average Stock Price");
    job.setJarByClass(StockAverageDriver.class);

    job.setMapperClass(StockAverage_Mapper.class);
    job.setCombinerClass(StockAverage_Combiner.class);
    job.setReducerClass(StockAverage_Reducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(StockAverage_CompositeValueWritable.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:Assignment4_P3_InMemoryStdDeviation.MovieRatingStdDevDriver.java

/**
 * @param args the command line arguments
 *///  ww  w.j av a  2  s  .  co  m
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Movie Rating Standard Deviation");
    job.setJarByClass(MovieRatingStdDevDriver.class);

    job.setMapperClass(MovieRatingStdDev_Mapper.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(FloatWritable.class);

    job.setReducerClass(MovieRatingStdDev_Reducer.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(Text.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:Assignment4_P4_MemoryConscious.MovieRatingMemConsciousDriver.java

/**
 * @param args the command line arguments
 *///from  w w w  .  j  a va 2  s  .  c o m
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Movie Rating Mem Conscious Standard Deviation");
    job.setJarByClass(MovieRatingMemConsciousDriver.class);

    job.setMapperClass(MovieRatingMemConscious_Mapper.class);
    job.setCombinerClass(MovingRatingMemConscious_Combiner.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(SortedMapWritable.class);

    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(Text.class);
    job.setReducerClass(MovieRatingMemConscious_Reducer.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:Assignment5_P2_DistinctIPAddress.DistinctIPAddressDriver.java

/**
 * @param args the command line arguments
 *///from w ww.  j a va2s. c o m
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "IP Address Count");
    job.setJarByClass(DistinctIPAddressDriver.class);
    job.setMapperClass(DistinctIPAddress_Mapper.class);
    job.setCombinerClass(DistinctIPAddress_Reducer.class);
    job.setReducerClass(DistinctIPAddress_Reducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}