Example usage for org.apache.hadoop.mapred JobConf JobConf

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf JobConf.

Prototype

public JobConf(boolean loadDefaults) 

Source Link

Document

A new map/reduce configuration where the behavior of reading from the default resources can be turned off.

Usage

From source file:gen_rank.java

License:LGPL

public static void runjob(String input, String output) throws Exception {
    JobConf conf = new JobConf(gen_rank.class);
    conf.setJobName("Preparing_data");

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    conf.setMapperClass(Map.class);
    conf.setCombinerClass(Reduce.class);
    conf.setReducerClass(Reduce.class);

    conf.setInputFormat(TextInputFormat.class);
    conf.setOutputFormat(TextOutputFormat.class);

    FileInputFormat.setInputPaths(conf, new Path(input));
    FileOutputFormat.setOutputPath(conf, new Path(output));

    JobClient.runJob(conf);/*from w  w w .  j  a  va2  s .  c  o m*/
}

---

From source file:GapDeduceRunner.java

License:Apache License

public static void main(String[] args) throws IOException {
    JobConf conf = new JobConf(GapDeduceRunner.class);
    conf.setJobName("gapdeduce");

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(Text.class);

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    conf.setMapperClass(Gapper.class);
    conf.setReducerClass(Deducer.class);

    // KeyValueTextInputFormat treats each line as an input record, 
    // and splits the line by the tab character to separate it into key and value 
    conf.setInputFormat(KeyValueTextInputFormat.class);
    conf.setOutputFormat(TextOutputFormat.class);

    FileInputFormat.setInputPaths(conf, new Path(args[0]));
    FileOutputFormat.setOutputPath(conf, new Path(args[1]));

    JobClient.runJob(conf);/*from   w ww  . j av  a  2s  .  co m*/
}

---

From source file:LicenseStatewiseCount.java

public static void main(String[] args) throws Exception {

    JobConf conf = new JobConf(ParkingTicketStatewiseCount.class);
    conf.setJobName("Statecounts");

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(IntWritable.class);

    conf.setMapperClass(Map.class);
    conf.setReducerClass(Reduce.class);

    conf.setInputFormat(TextInputFormat.class);
    conf.setOutputFormat(TextOutputFormat.class);

    FileInputFormat.setInputPaths(conf, new Path(args[0]));
    FileOutputFormat.setOutputPath(conf, new Path(args[1]));

    JobClient.runJob(conf);/*from   w w  w  . ja  va2s  . co  m*/

}

---

From source file:TestFormatStorageRecordReader.java

License:Open Source License

public static void main(String[] argv) throws IOException {
    try {//w  w w .  j  a  va  2 s  . c om
        String path1 = "se_test/fs/basic/f1/kt/";
        String path2 = "se_test/fs/basic/f2/";

        initFormatData();

        JobConf conf1 = new JobConf(TestFormatStorageRecordReader.class);
        JobConf conf2 = new JobConf(TestFormatStorageRecordReader.class);

        FormatStorageSerDe serDe1 = initSerDe(conf1);
        FormatStorageSerDe serDe2 = initSerDe(conf2);

        StandardStructObjectInspector oi1 = (StandardStructObjectInspector) serDe1.getObjectInspector();
        List<? extends StructField> fieldRefs1 = oi1.getAllStructFieldRefs();

        StandardStructObjectInspector oi2 = (StandardStructObjectInspector) serDe2.getObjectInspector();
        List<? extends StructField> fieldRefs2 = oi2.getAllStructFieldRefs();

        InputFormat inputFormat = new FormatStorageInputFormat();
        RecordReader<WritableComparable, Writable> currRecReader1 = getRecReader(conf1, path1);
        WritableComparable key;
        Writable value;

        key = currRecReader1.createKey();
        value = currRecReader1.createValue();
        System.out.println("currRecReader1. output....");
        while (currRecReader1.next(key, value)) {
            ((Record) value).show();
            System.out.println("end value.show");
            Object row = serDe1.deserialize((Record) value);
            Record record = (Record) serDe1.serialize(row, oi1);
            record.show();

        }
        /*
        RecordReader<WritableComparable, Writable> currRecReader2 = getRecReader(conf2, path2);                
        key = currRecReader2.createKey();
        value = currRecReader2.createValue();
        System.out.println("currRecReader2. output....");
        while (currRecReader2.next(key, value))
        {
        ((Record)value).show();
        }
                
        RecordReader<WritableComparable, Writable> currRecReader3 = getRecReader(conf1, path1);            
        key = currRecReader3.createKey();
        value = currRecReader3.createValue();
        System.out.println("currRecReader3. output....");
        while (currRecReader3.next(key, value))
        {
        ((Record)value).show();
        }
        */
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }
}

---

From source file:DijikstraAlgo.java

License:GNU General Public License

public static void run(String[] args) throws Exception {
    IN = "hdfs://10.8.3.161:9000/user/sagar/input/";
    OUT = "hdfs://10.8.3.161:9000/user/sagar/output/";
    String input = IN;//from  w w  w .ja va  2 s .  c o  m
    String output = OUT + System.nanoTime();
    String MAX_SPLIT_SIZE = args[0];
    boolean isdone = false;

    // Reiteration again and again till the convergence
    while (isdone == false) {
        JobConf conf = new JobConf(DijikstraAlgo.class);
        conf.setJobName("Dijikstra");
        // conf.set("mapred.max.split.size", MAX_SPLIT_SIZE);
        conf.setOutputKeyClass(LongWritable.class);
        conf.setOutputValueClass(Text.class);
        conf.setMapperClass(Map.class);
        conf.setReducerClass(Reduce.class);
        conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(TextOutputFormat.class);

        FileInputFormat.setInputPaths(conf, new Path(input));
        FileOutputFormat.setOutputPath(conf, new Path(output));

        JobClient.runJob(conf);

        input = output + "/part-00000";
        isdone = true;// set the job to NOT run again!
        Path ofile = new Path(input);
        FileSystem fs = FileSystem.get(new URI("hdfs://10.8.3.165:9000"), conf);
        //FileSystem fs = FileSystem.get(new Configuration());
        BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(ofile)));
        HashMap<Integer, Integer> imap = new HashMap<Integer, Integer>();
        String line = br.readLine();
        // Read the current output file and put it into HashMap
        while (line != null) {
            String[] sp = line.split("\t| ");
            int node = Integer.parseInt(sp[0]);
            int distance = Integer.parseInt(sp[1]);
            imap.put(node, distance);
            line = br.readLine();
        }
        br.close();

        // Check for convergence condition if any node is still left then
        // continue else stop
        Iterator<Integer> itr = imap.keySet().iterator();
        while (itr.hasNext()) {
            int key = itr.next();
            int value = imap.get(key);
            if (value >= 125) {
                isdone = false;
            }
        }
        input = output;
        output = OUT + System.nanoTime();
    }
}

---

From source file:Text2FormatStorageMR.java

License:Open Source License

@SuppressWarnings("deprecation")
public static void main(String[] args) throws Exception {

    if (args.length != 2) {
        System.out.println("FormatFileMR <input> <output>");
        System.exit(-1);// www  .jav  a  2s  . c o m
    }

    JobConf conf = new JobConf(FormatStorageMR.class);

    conf.setJobName("Text2FormatMR");

    conf.setNumMapTasks(1);
    conf.setNumReduceTasks(4);

    conf.setOutputKeyClass(LongWritable.class);
    conf.setOutputValueClass(Unit.Record.class);

    conf.setMapperClass(TextFileTestMapper.class);
    conf.setReducerClass(FormatFileTestReducer.class);

    conf.setInputFormat(TextInputFormat.class);
    conf.setOutputFormat(FormatStorageOutputFormat.class);
    conf.set("mapred.output.compress", "flase");

    Head head = new Head();
    initHead(head);

    head.toJobConf(conf);

    FileInputFormat.setInputPaths(conf, args[0]);
    Path outputPath = new Path(args[1]);
    FileOutputFormat.setOutputPath(conf, outputPath);

    FileSystem fs = outputPath.getFileSystem(conf);
    fs.delete(outputPath, true);

    JobClient jc = new JobClient(conf);
    RunningJob rj = null;
    rj = jc.submitJob(conf);

    String lastReport = "";
    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS");
    long reportTime = System.currentTimeMillis();
    long maxReportInterval = 3 * 1000;
    while (!rj.isComplete()) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        }

        int mapProgress = Math.round(rj.mapProgress() * 100);
        int reduceProgress = Math.round(rj.reduceProgress() * 100);

        String report = " map = " + mapProgress + "%,  reduce = " + reduceProgress + "%";

        if (!report.equals(lastReport) || System.currentTimeMillis() >= reportTime + maxReportInterval) {

            String output = dateFormat.format(Calendar.getInstance().getTime()) + report;
            System.out.println(output);
            lastReport = report;
            reportTime = System.currentTimeMillis();
        }
    }

    System.exit(0);

}

---

From source file:TestFormatStorageInputFormat.java

License:Open Source License

public static void main(String[] argv) throws IOException, SerDeException {
    try {//from  w w  w . ja  v  a2  s .  c om
        if (argv.length != 2) {
            System.out.println("TestFormatStorageInputFormat <input> <output>");
            System.exit(-1);
        }

        JobConf conf = new JobConf(TestFormatStorageInputFormat.class);

        conf.setJobName("TestFormatStorageInputFormat");

        conf.setNumMapTasks(1);
        conf.setNumReduceTasks(1);

        conf.setOutputKeyClass(LongWritable.class);
        conf.setOutputValueClass(Unit.Record.class);

        conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(FormatStorageOutputFormat.class);
        conf.set("mapred.output.compress", "flase");

        conf.set("mapred.input.dir", argv[0]);

        Head head = new Head();
        initHead(head);

        head.toJobConf(conf);

        FormatStorageSerDe serDe = initSerDe(conf);
        StandardStructObjectInspector oi = (StandardStructObjectInspector) serDe.getObjectInspector();
        List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();

        FileInputFormat.setInputPaths(conf, argv[0]);
        Path outputPath = new Path(argv[1]);
        FileOutputFormat.setOutputPath(conf, outputPath);

        InputFormat inputFormat = new FormatStorageInputFormat();
        InputSplit[] inputSplits = inputFormat.getSplits(conf, 1);
        if (inputSplits.length == 0) {
            System.out.println("inputSplits is empty");
            return;
        } else {
            System.out.println("get Splits:" + inputSplits.length);
        }

        int size = inputSplits.length;
        System.out.println("getSplits return size:" + size);
        for (int i = 0; i < size; i++) {
            FormatStorageSplit split = (FormatStorageSplit) inputSplits[i];
            System.out.printf("split:" + i + "offset:" + split.getStart() + "len:" + split.getLength() + "path:"
                    + conf.get(ConstVar.InputPath) + "beginLine:" + split.getBeginLine() + "endLine:"
                    + split.getEndLine() + "\n");
        }

        {
            int totalDelay = 0;
            RecordReader<WritableComparable, Writable> currRecReader = null;
            for (int i = 0; i < inputSplits.length; i++) {
                currRecReader = inputFormat.getRecordReader(inputSplits[i], conf, Reporter.NULL);

                WritableComparable key;
                Writable value;

                key = currRecReader.createKey();
                value = currRecReader.createValue();

                long begin = System.currentTimeMillis();
                int count = 0;
                while (currRecReader.next(key, value)) {
                    Record record = (Record) value;

                    Object row = serDe.deserialize(record);
                    count++;
                }
                long end = System.currentTimeMillis();

                long delay = (end - begin) / 1000;
                totalDelay += delay;
                System.out.println(count + " record read over, delay " + delay + " s");
            }

            System.out.println("total delay:" + totalDelay);
        }
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }
}

---

From source file:MapReduce3.java

public static void main(String[] args) throws Exception {

    ////from  w  w  w .  j av  a2  s  .c o  m
    String dst = "hdfs://localhost:9000/data/2006a.csv";

    //??
    //  String dstOut = "hdfs://localhost:9000/mapreduce/result3/1";
    String dstOut = "/Users/wendyzhuo/NetBeansProjects/final_Hadoop/src/output3/1";
    String outFiles = "/Users/wendyzhuo/NetBeansProjects/final_Hadoop/src/output3/2";
    Configuration hadoopConfig = new Configuration();

    hadoopConfig.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());

    hadoopConfig.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());

    Job job = new Job(hadoopConfig);
    Job job2 = new Job(hadoopConfig);

    FileInputFormat.addInputPath(job, new Path(dst));
    FileOutputFormat.setOutputPath(job, new Path(dstOut));
    FileInputFormat.addInputPath(job2, new Path(dstOut));
    FileOutputFormat.setOutputPath(job2, new Path(outFiles));

    JobConf map1Conf = new JobConf(false);
    ChainMapper.addMapper(job, TempMapper.class, LongWritable.class, Text.class, CompositeKey_wd.class,
            IntWritable.class, map1Conf);
    JobConf reduceConf = new JobConf(false);
    ChainReducer.setReducer(job, TempReducer.class, CompositeKey_wd.class, IntWritable.class,
            CompositeKey_wd.class, IntWritable.class, reduceConf);

    JobConf map2Conf = new JobConf(false);
    ChainMapper.addMapper(job2, TempMapper2.class, LongWritable.class, Text.class, IntWritable.class,
            CompositeKey_wd.class, map2Conf);
    JobConf map3Conf = new JobConf(false);
    ChainReducer.setReducer(job2, TempReduce2.class, IntWritable.class, CompositeKey_wd.class, Text.class,
            IntWritable.class, map3Conf);
    //       
    //  JobClient.runJob(job);

    //MapperReducer?
    //        job.setMapperClass(TempMapper.class);
    //
    //        job.setReducerClass(TempReducer.class);

    //?KeyValue
    job.setOutputKeyClass(CompositeKey_wd.class);

    job.setOutputValueClass(IntWritable.class);

    job2.setMapOutputKeyClass(IntWritable.class);
    job2.setMapOutputValueClass(CompositeKey_wd.class);

    //  job2.setSortComparatorClass(LongWritable.DecreasingComparator.class);

    //job?
    job.waitForCompletion(true);
    System.out.println("Finished1");
    job2.waitForCompletion(true);
    System.out.println("Finished2");

}

---

From source file:DistribCountingDriver.java

License:Apache License

public int run(String args[]) throws Exception {
    long job_start_time, job_end_time;
    long job_runtime;

    JobConf conf = new JobConf(getConf());

    int minFreqPercent = Integer.parseInt(args[0]);
    int datasetSize = Integer.parseInt(args[1]);
    conf.setInt("DISTRCOUNT.datasetSize", datasetSize);
    conf.setInt("DISTRCOUNT.minFreqPercent", minFreqPercent);

    conf.setBoolean("mapred.reduce.tasks.speculative.execution", false);
    conf.setInt("mapred.task.timeout", 60000000);

    conf.setJarByClass(DistribCountingDriver.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(IntWritable.class);

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    conf.setMapperClass(DistribCountingMapper.class);
    conf.setCombinerClass(DistribCountingCombiner.class);
    conf.setReducerClass(DistribCountingReducer.class);

    conf.setInputFormat(SequenceFileInputFormat.class);
    SequenceFileInputFormat.addInputPath(conf, new Path(args[2]));
    FileOutputFormat.setOutputPath(conf, new Path(args[3]));

    job_start_time = System.currentTimeMillis();
    JobClient.runJob(conf);/* ww  w. j ava  2  s . c o m*/
    job_end_time = System.currentTimeMillis();

    job_runtime = (job_end_time - job_start_time) / 1000;

    System.out.println("total job runtime (seconds): " + job_runtime);

    return 0;
}

---

From source file:TestColumnStorageOutputFormat.java

License:Open Source License

public static void main(String[] argv) throws IOException {
    try {//w  ww  . ja  v  a 2  s.c  o m
        if (argv.length != 2) {
            System.out.println("TestColumnStorageOutputFormat <output> <count>");
            System.exit(-1);
        }

        JobConf conf = new JobConf(TestColumnStorageOutputFormat.class);

        conf.setJobName("TestColumnStorageOutputFormat");

        conf.setNumMapTasks(1);
        conf.setNumReduceTasks(1);

        conf.setOutputKeyClass(LongWritable.class);
        conf.setOutputValueClass(Unit.Record.class);

        conf.setOutputFormat(ColumnStorageOutputFormat.class);
        conf.set("mapred.output.compress", "flase");

        conf.set("mapred.output.dir", argv[0]);

        Head head = new Head();
        initHead(head);

        head.toJobConf(conf);

        Path outputPath = new Path(argv[0]);
        FileOutputFormat.setOutputPath(conf, outputPath);

        FileSystem fs = FileSystem.get(conf);
        MyColumnOutputFormat output = new MyColumnOutputFormat(head, conf, outputPath);

        long begin = System.currentTimeMillis();
        int count = Integer.valueOf(argv[1]);
        String string = "hello konten";
        for (int i = 0; i < count; i++) {
            Record record = new Record((short) 210);

            for (short j = 0; j < 30; j++) {
                record.addValue(new FieldValue((byte) 1, (short) (j * 7 + 0)));
                record.addValue(new FieldValue((short) 2, (short) (j * 7 + 1)));
                record.addValue(new FieldValue((int) 3, (short) (j * 7 + 2)));
                record.addValue(new FieldValue((long) 4, (short) (j * 7 + 3)));
                record.addValue(new FieldValue((float) 5.5, (short) (j * 7 + 4)));
                record.addValue(new FieldValue((double) 6.6, (short) (j * 7 + 5)));
                record.addValue(new FieldValue((double) 7.7, (short) (j * 7 + 6)));

            }
            output.doWrite(record);

            if (i % 100000 == 0) {
                long end = System.currentTimeMillis();
                System.out.println(i + "record write, delay:" + (end - begin) / 1000 + "s");
            }
        }

        long end = System.currentTimeMillis();
        System.out.println(count + "record write over, delay:" + (end - begin) / 1000 + "s");
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• »