List of usage examples for org.apache.hadoop.mapreduce Job setMapOutputKeyClass
public void setMapOutputKeyClass(Class<?> theClass) throws IllegalStateException
From source file:com.marklogic.hadoop.csvtoxml.CsvToXML.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); @SuppressWarnings("deprecation") Job job = new Job(conf); job.setJobName("ProcessCSVtoXML_job"); System.out.println("After the JobName Updates"); job.setJarByClass(CsvToXML.class); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(ContentMapper.class); job.setMapOutputKeyClass(DocumentURI.class); job.setMapOutputValueClass(Text.class); job.setOutputFormatClass(ContentOutputFormat.class); System.out.println("Made it past external jar dependencies nodes"); FileInputFormat.setInputPaths(job, new Path(otherArgs[1])); conf = job.getConfiguration();//from ww w. j a v a2 s.co m conf.addResource(otherArgs[0]); System.out.println("After the conf.set"); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.marklogic.mapreduce.examples.BinaryReader.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); if (args.length < 2) { System.err.println("Usage: BinaryReader configFile outputDir"); System.exit(2);/*from w w w . j av a 2 s . c o m*/ } String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Job job = Job.getInstance(conf, "binary reader"); job.setJarByClass(BinaryReader.class); job.setInputFormatClass(DocumentInputFormat.class); job.setMapperClass(DocMapper.class); job.setMapOutputKeyClass(DocumentURI.class); job.setMapOutputValueClass(BytesWritable.class); job.setOutputFormatClass(BinaryOutputFormat.class); job.setOutputKeyClass(DocumentURI.class); job.setOutputValueClass(BytesWritable.class); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); conf = job.getConfiguration(); conf.addResource(otherArgs[0]); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.marklogic.mapreduce.examples.ContentLoader.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); if (args.length < 2) { System.err.println("Usage: ContentLoader configFile inputDir"); System.exit(2);/* w ww . ja v a2s. c o m*/ } String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Job job = Job.getInstance(conf, "content loader"); job.setJarByClass(ContentLoader.class); job.setInputFormatClass(ContentInputFormat.class); job.setMapperClass(ContentMapper.class); job.setMapOutputKeyClass(DocumentURI.class); job.setMapOutputValueClass(Text.class); job.setOutputFormatClass(ContentOutputFormat.class); ContentInputFormat.setInputPaths(job, new Path(otherArgs[1])); conf = job.getConfiguration(); conf.addResource(otherArgs[0]); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.marklogic.mapreduce.examples.ContentReader.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); if (args.length < 2) { System.err.println("Usage: ContentReader configFile outputDir"); System.exit(2);//from w w w . j a va2 s. c o m } String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Job job = Job.getInstance(conf, "content reader"); job.setJarByClass(ContentReader.class); job.setInputFormatClass(DocumentInputFormat.class); job.setMapperClass(DocMapper.class); job.setMapOutputKeyClass(DocumentURI.class); job.setMapOutputValueClass(DatabaseDocument.class); job.setOutputFormatClass(CustomOutputFormat.class); CustomOutputFormat.setOutputPath(job, new Path(otherArgs[1])); conf = job.getConfiguration(); conf.addResource(otherArgs[0]); conf.setClass(MarkLogicConstants.INPUT_SSL_OPTIONS_CLASS, SslOptions.class, SslConfigOptions.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.marklogic.mapreduce.examples.HelloWorld.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Job job = Job.getInstance(conf, "hello world"); job.setJarByClass(HelloWorld.class); // Map related configuration job.setInputFormatClass(DocumentInputFormat.class); job.setMapperClass(MyMapper.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(Text.class); // Reduce related configuration job.setReducerClass(MyReducer.class); job.setOutputFormatClass(ContentOutputFormat.class); job.setOutputKeyClass(DocumentURI.class); job.setOutputValueClass(Text.class); conf = job.getConfiguration();/* ww w .j a v a 2s .c o m*/ conf.addResource("marklogic-hello-world.xml"); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.marklogic.mapreduce.examples.LinkCount.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); if (args.length < 2) { System.err.println("Usage: LinkCount configFile outputDir"); System.exit(2);/*from www . ja va2 s .c om*/ } String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Job job = Job.getInstance(conf, "link count"); job.setJarByClass(LinkCount.class); job.setInputFormatClass(ValueInputFormat.class); job.setMapperClass(RefMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputFormatClass(TextOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); conf = job.getConfiguration(); conf.addResource(otherArgs[0]); conf.setClass(MarkLogicConstants.INPUT_VALUE_CLASS, Text.class, Writable.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.marklogic.mapreduce.examples.LinkCountCooccurrences.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); if (args.length < 2) { System.err.println("Usage: LinkCountCooccurrences configFile outputDir"); System.exit(2);/*ww w .ja va2s. com*/ } String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Job job = Job.getInstance(conf, "link count cooccurrences"); job.setJarByClass(LinkCountCooccurrences.class); job.setInputFormatClass(KeyValueInputFormat.class); job.setMapperClass(RefMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputFormatClass(TextOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); conf = job.getConfiguration(); conf.addResource(otherArgs[0]); conf.setClass(MarkLogicConstants.INPUT_KEY_CLASS, Text.class, Writable.class); conf.setClass(MarkLogicConstants.INPUT_VALUE_CLASS, Text.class, Writable.class); conf.setClass(MarkLogicConstants.INPUT_LEXICON_FUNCTION_CLASS, HrefTitleMap.class, ElemAttrValueCooccurrences.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.marklogic.mapreduce.examples.LinkCountHDFS.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); if (args.length < 2) { System.err.println("Usage: LinkCountHDFS inputDir outputDir"); System.exit(2);//from w ww . j a v a 2s . com } String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Job job = Job.getInstance(conf, "link count hdfs"); job.setJarByClass(LinkCountHDFS.class); job.setInputFormatClass(HDFSInputFormat.class); job.setMapperClass(RefMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(IntSumReducer.class); job.setOutputFormatClass(TextOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); HDFSInputFormat.setInputPaths(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.marklogic.mapreduce.examples.LinkCountInDoc.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); if (args.length < 1) { System.err.println("Usage: LinkCountInDoc configFile"); System.exit(2);/* w ww. j a v a2 s .com*/ } String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Job job = Job.getInstance(conf, "link count in doc"); job.setJarByClass(LinkCountInDoc.class); job.setInputFormatClass(NodeInputFormat.class); job.setMapperClass(RefMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(IntSumReducer.class); job.setOutputFormatClass(NodeOutputFormat.class); job.setOutputKeyClass(NodePath.class); job.setOutputValueClass(MarkLogicNode.class); conf = job.getConfiguration(); conf.addResource(otherArgs[0]); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:com.marklogic.mapreduce.examples.LinkCountInProperty.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); if (args.length < 1) { System.err.println("Usage: LinkCountInProperty configFile"); System.exit(2);/*from w w w . j a va 2s. com*/ } String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Job job = Job.getInstance(conf, "link count in property"); job.setJarByClass(LinkCountInProperty.class); job.setInputFormatClass(ValueInputFormat.class); job.setMapperClass(RefMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(IntSumReducer.class); job.setOutputFormatClass(PropertyOutputFormat.class); job.setOutputKeyClass(DocumentURI.class); job.setOutputValueClass(MarkLogicNode.class); conf = job.getConfiguration(); conf.addResource(otherArgs[0]); conf.setClass(MarkLogicConstants.INPUT_VALUE_CLASS, Text.class, Writable.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }