Java tutorial
/** * Copyright 2011 Nube Technologies * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR * CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and limitations under the License. */ package co.nubetech.hiho.dedup; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.log4j.Logger; import co.nubetech.hiho.common.HIHOException; public class DedupJob extends Configured implements Tool { final static Logger logger = Logger.getLogger(co.nubetech.hiho.dedup.DedupJob.class); private String inputFormat = null; private String dedupBy = null; // "key" or "value" private String inputKeyClassName = null; private String inputValueClassName = null; private String inputPath = null; private String outputPath = null; private String outputFormat = null; private String delimiter = ","; private int column = 1; private long totalRecordsRead; private long badRecords; private long output; private long duplicateRecords; public long getTotalRecordsRead() { return totalRecordsRead; } public void setTotalRecordsRead(long totalRecordsRead) { this.totalRecordsRead = totalRecordsRead; } public long getBadRecords() { return badRecords; } public void setBadRecords(long badRecords) { this.badRecords = badRecords; } public long getOutput() { return output; } public void setOutput(long output) { this.output = output; } public long getDuplicateRecords() { return duplicateRecords; } public void setDuplicateRecords(long duplicateRecords) { this.duplicateRecords = duplicateRecords; } public void populateConfiguration(String[] args) { for (int i = 0; i < args.length - 1; i++) { if ("-inputFormat".equals(args[i])) { inputFormat = args[++i]; if (inputFormat.equals("org.apache.hadoop.mapreduce.lib.input.TextInputFormat")) { inputKeyClassName = "org.apache.hadoop.io.LongWritable"; inputValueClassName = "org.apache.hadoop.io.Text"; } else if (inputFormat.equals("co.nubetech.hiho.dedup.DelimitedTextInputFormat")) { inputKeyClassName = "org.apache.hadoop.io.Text"; inputValueClassName = "org.apache.hadoop.io.Text"; outputFormat = "co.nubetech.hiho.mapreduce.lib.output.NoKeyOnlyValueOutputFormat"; } else if (inputFormat.equals("org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat")) { outputFormat = "org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat"; } } else if ("-dedupBy".equals(args[i])) { dedupBy = args[++i]; } else if ("-inputKeyClassName".equals(args[i])) { inputKeyClassName = args[++i]; } else if ("-inputValueClassName".equals(args[i])) { inputValueClassName = args[++i]; } else if ("-inputPath".equals(args[i])) { inputPath = args[++i]; } else if ("-outputPath".equals(args[i])) { outputPath = args[++i]; } else if ("-delimiter".equals(args[i])) { delimiter = args[++i]; } else if ("-column".equals(args[i])) { column = Integer.parseInt(args[++i]); } else if ("-outputFormat".equals(args[i])) { outputFormat = args[++i]; } } } public void checkMandatoryConfs() throws HIHOException { if (inputFormat == null) { throw new HIHOException("The provided input format is empty, please specify inputFormat"); } if (dedupBy == null) { throw new HIHOException("The provided value of dedupBy is empty, please specify either key or value"); } if ((!dedupBy.equals("key")) && (!dedupBy.equals("value"))) { throw new HIHOException( "The provided value of dedupBy is Incorrect, please specify either key or value"); } if (inputKeyClassName == null) { throw new HIHOException("The provided input key class name is empty, please specify inputKeyClassName"); } if (inputValueClassName == null) { throw new HIHOException( "The provided input value class name is empty, please specify inputValueClassName"); } if (inputPath == null) { throw new HIHOException("The provided input path is empty, please specify inputPath"); } if (outputPath == null) { throw new HIHOException("The provided output path is empty, please specify outputPath"); } if (outputFormat == null) { System.out.println(outputFormat); throw new HIHOException("The provided output format is empty, please specify outputFormat"); } } @Override public int run(String[] args) throws Exception { Configuration conf = getConf(); populateConfiguration(args); try { checkMandatoryConfs(); } catch (HIHOException e1) { e1.printStackTrace(); throw new Exception(e1); } Job job = new Job(conf); job.setJobName("Dedup job"); job.setJarByClass(DedupJob.class); Class inputFormatClass = Class.forName(inputFormat); Class outputFormatClass = Class.forName(outputFormat); Class inputKeyClass = Class.forName(inputKeyClassName); Class inputValueClass = Class.forName(inputValueClassName); if (dedupBy.equals("key")) { job.setMapperClass(DedupKeyMapper.class); job.setReducerClass(DedupKeyReducer.class); job.setMapOutputValueClass(inputValueClass); } else if (dedupBy.equals("value")) { job.setMapperClass(DedupValueMapper.class); job.setReducerClass(DedupValueReducer.class); job.setMapOutputValueClass(inputKeyClass); } job.setInputFormatClass(inputFormatClass); if (inputFormat.equals("co.nubetech.hiho.dedup.DelimitedTextInputFormat")) { DelimitedTextInputFormat.setProperties(job, delimiter, column); } job.setMapOutputKeyClass(HihoTuple.class); job.setOutputKeyClass(inputKeyClass); job.setOutputValueClass(inputValueClass); job.setPartitionerClass(HihoHashPartitioner.class); FileInputFormat.setInputPaths(job, inputPath); job.setOutputFormatClass(outputFormatClass); FileOutputFormat.setOutputPath(job, new Path(outputPath)); try { logger.debug("Output format class is " + job.getOutputFormatClass()); logger.debug("Class is " + ReflectionUtils .newInstance(job.getOutputFormatClass(), job.getConfiguration()).getClass().getName()); job.waitForCompletion(false); if (job.isComplete()) { Counters counters = job.getCounters(); totalRecordsRead = counters.findCounter(DedupRecordCounter.TOTAL_RECORDS_READ).getValue(); badRecords = counters.findCounter(DedupRecordCounter.BAD_RECORD).getValue(); output = counters.findCounter(DedupRecordCounter.OUTPUT).getValue(); duplicateRecords = totalRecordsRead - output; logger.info("Total records read are: " + totalRecordsRead); logger.info("Bad Records are: " + badRecords); logger.info("Output records are: " + output); logger.info("Duplicate records are: " + duplicateRecords); } } catch (Exception e) { e.printStackTrace(); } return 0; } public static void main(String[] args) throws Exception { DedupJob job = new DedupJob(); int res = ToolRunner.run(new Configuration(), job, args); System.exit(res); } }