List of usage examples for org.apache.hadoop.mapred JobConf setCombinerClass
public void setCombinerClass(Class<? extends Reducer> theClass)
From source file:org.sf.xrime.algorithms.partitions.connected.weakly.alg_2.VertexLabelPropagation.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { JobConf conf = new JobConf(context, VertexLabelPropagation.class); conf.setJobName("VertexLabelPropagation"); // the keys are vertex identifiers (strings) conf.setOutputKeyClass(Text.class); // the values are adjacent vertexes with labels (Writable) conf.setOutputValueClass(LabeledAdjSetVertex.class); // mapper, combiner, reducer, all show up. conf.setMapperClass(MapClass.class); conf.setCombinerClass(CombineClass.class); conf.setReducerClass(ReduceClass.class); // makes the file format suitable for machine processing. conf.setInputFormat(SequenceFileInputFormat.class); conf.setOutputFormat(SequenceFileOutputFormat.class); // Enable compression. conf.setCompressMapOutput(true);//from w w w . j av a2 s .c o m conf.setMapOutputCompressorClass(GzipCodec.class); try { FileInputFormat.setInputPaths(conf, getSource().getPath()); FileOutputFormat.setOutputPath(conf, getDestination().getPath()); } catch (IllegalAccessException e1) { throw new ProcessorExecutionException(e1); } conf.setNumMapTasks(getMapperNum()); conf.setNumReduceTasks(getReducerNum()); try { this.runningJob = JobClient.runJob(conf); } catch (IOException e) { throw new ProcessorExecutionException(e); } }
From source file:org.sf.xrime.algorithms.partitions.connected.weakly.alg_2.VertexSetSummarize.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { JobConf conf = new JobConf(context, VertexSetSummarize.class); conf.setJobName("VertexSetSummarize"); // the keys are vertex labels (strings) conf.setOutputKeyClass(Text.class); // the values are vertex sets (Writable) conf.setOutputValueClass(VertexSet.class); conf.setMapperClass(MapClass.class); conf.setCombinerClass(ReduceClass.class); conf.setReducerClass(ReduceClass.class); // makes the file format suitable for machine processing. conf.setInputFormat(SequenceFileInputFormat.class); conf.setOutputFormat(SequenceFileOutputFormat.class); // Enable compression. conf.setCompressMapOutput(true);/*from w w w . ja va2s . c om*/ conf.setMapOutputCompressorClass(GzipCodec.class); try { FileInputFormat.setInputPaths(conf, getSource().getPath()); FileOutputFormat.setOutputPath(conf, getDestination().getPath()); } catch (IllegalAccessException e1) { throw new ProcessorExecutionException(e1); } conf.setNumMapTasks(getMapperNum()); conf.setNumReduceTasks(getReducerNum()); try { this.runningJob = JobClient.runJob(conf); } catch (IOException e) { throw new ProcessorExecutionException(e); } }
From source file:org.sf.xrime.algorithms.statistics.LargestLabeledSetWithLabelDegree.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { JobConf conf = new JobConf(context, LargestLabeledSetWithLabelDegree.class); conf.setJobName("LargestLabeledSetWithLabelDegree"); // the keys are a pseudo one ("Largest_Degree") conf.setOutputKeyClass(Text.class); // the values are degrees (ints) conf.setOutputValueClass(IntWritable.class); conf.setMapperClass(MapClass.class); conf.setCombinerClass(ReduceClass.class); conf.setReducerClass(ReduceClass.class); // The format of input data is generated with WritableSerialization. conf.setInputFormat(SequenceFileInputFormat.class); try {// w w w .j a va 2 s.c o m FileInputFormat.setInputPaths(conf, getSource().getPath()); FileOutputFormat.setOutputPath(conf, getDestination().getPath()); } catch (IllegalAccessException e1) { throw new ProcessorExecutionException(e1); } conf.setNumMapTasks(getMapperNum()); // Only one reducer is permitted, or the largest value will be wrong. conf.setNumReduceTasks(1); conf.setCompressMapOutput(true); conf.setMapOutputCompressorClass(GzipCodec.class); try { this.runningJob = JobClient.runJob(conf); } catch (IOException e) { throw new ProcessorExecutionException(e); } }
From source file:org.sf.xrime.algorithms.statistics.LargestVertexDegree.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { JobConf conf = new JobConf(context, LargestVertexDegree.class); conf.setJobName("LargestDegree"); // the keys are a pseudo one ("Largest_Degree") conf.setOutputKeyClass(Text.class); // the values are degrees (ints) conf.setOutputValueClass(IntWritable.class); conf.setMapperClass(MapClass.class); conf.setCombinerClass(ReduceClass.class); conf.setReducerClass(ReduceClass.class); // The format of input data is generated with WritableSerialization. conf.setInputFormat(SequenceFileInputFormat.class); try {/*from www . j a va 2s . c o m*/ FileInputFormat.setInputPaths(conf, getSource().getPath()); FileOutputFormat.setOutputPath(conf, getDestination().getPath()); } catch (IllegalAccessException e1) { throw new ProcessorExecutionException(e1); } conf.setNumMapTasks(getMapperNum()); // Only one reducer is permitted, or the largest value will be wrong. conf.setNumReduceTasks(1); conf.setCompressMapOutput(true); conf.setMapOutputCompressorClass(GzipCodec.class); try { this.runningJob = JobClient.runJob(conf); } catch (IOException e) { throw new ProcessorExecutionException(e); } }
From source file:org.sf.xrime.algorithms.statistics.VertexEdgeCounter.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { try {/*from w ww .ja v a2 s. com*/ JobConf conf = new JobConf(context, AverageVertexDegree.class); conf.setJobName("AverageDegree"); // the keys are a pseudo one ("Average_Degree") conf.setOutputKeyClass(Text.class); // the values are degrees (ints) conf.setOutputValueClass(LongWritable.class); conf.setMapperClass(MapClass.class); conf.setCombinerClass(ReduceClass.class); // No combiner is permitted. conf.setReducerClass(ReduceClass.class); // The format of input data is generated with WritableSerialization. conf.setInputFormat(SequenceFileInputFormat.class); conf.setOutputFormat(SequenceFileOutputFormat.class); FileInputFormat.setInputPaths(conf, getSource().getPath()); FileOutputFormat.setOutputPath(conf, getDestination().getPath()); conf.setNumMapTasks(getMapperNum()); // Only one reducer is permitted, or the return value will be wrong. conf.setNumReduceTasks(1); // Set the possible CounterFilter class if (counterFilterClass != null) { conf.setClass(counterFilterKey, counterFilterClass, CounterFilter.class); } this.runningJob = JobClient.runJob(conf); FileSystem fs = getDestination().getPath().getFileSystem(conf); Path dataPath = new Path(getDestination().getPath().toString() + "/part-00000"); SequenceFile.Reader reader = new SequenceFile.Reader(fs, dataPath, conf); Writable key = ReflectionUtils.newInstance(reader.getKeyClass().asSubclass(WritableComparable.class), conf); Writable value = ReflectionUtils.newInstance(reader.getValueClass().asSubclass(Writable.class), conf); while (reader.next(key, value)) { addCounter(((Text) key).toString(), ((LongWritable) value).get()); } reader.close(); } catch (IOException e) { throw new ProcessorExecutionException(e); } catch (IllegalAccessException e) { throw new ProcessorExecutionException(e); } }
From source file:org.sf.xrime.algorithms.statistics.VertexEdgeDoubleCounter.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { try {// w ww. j a va 2s. c o m JobConf conf = new JobConf(context, AverageVertexDegree.class); conf.setJobName("AverageDegree"); conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(DoubleWritable.class); conf.setMapperClass(MapClass.class); conf.setCombinerClass(ReduceClass.class); conf.setReducerClass(ReduceClass.class); // The format of input data is generated with WritableSerialization. conf.setInputFormat(SequenceFileInputFormat.class); conf.setOutputFormat(SequenceFileOutputFormat.class); FileInputFormat.setInputPaths(conf, getSource().getPath()); FileOutputFormat.setOutputPath(conf, getDestination().getPath()); conf.setNumMapTasks(getMapperNum()); // Only one reducer is permitted, or the return value will be wrong. conf.setNumReduceTasks(1); // Set the possible CounterFilter class if (doubleCounterFilterClass != null) { conf.setClass(counterFilterKey, doubleCounterFilterClass, DoubleCounterFilter.class); } this.runningJob = JobClient.runJob(conf); FileSystem fs = getDestination().getPath().getFileSystem(conf); Path dataPath = new Path(getDestination().getPath().toString() + "/part-00000"); SequenceFile.Reader reader = new SequenceFile.Reader(fs, dataPath, conf); Writable key = ReflectionUtils.newInstance(reader.getKeyClass().asSubclass(WritableComparable.class), conf); Writable value = ReflectionUtils.newInstance(reader.getValueClass().asSubclass(Writable.class), conf); while (reader.next(key, value)) { addCounter(((Text) key).toString(), ((DoubleWritable) value).get()); } reader.close(); } catch (IOException e) { throw new ProcessorExecutionException(e); } catch (IllegalAccessException e) { throw new ProcessorExecutionException(e); } }
From source file:org.sf.xrime.algorithms.transform.vertex.AdjVertex2VertexSetTransformer.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { JobConf jobConf = new JobConf(conf, AdjVertex2VertexSetTransformer.class); jobConf.setJobName("AdjVertex2VertexSetTransformer"); // the keys are vertex identifiers (strings) jobConf.setOutputKeyClass(Text.class); // the values are vertex sets (Writable) jobConf.setOutputValueClass(VertexSet.class); jobConf.setMapperClass(MapClass.class); jobConf.setCombinerClass(ReduceClass.class); jobConf.setReducerClass(ReduceClass.class); // makes the file format suitable for machine processing. jobConf.setInputFormat(SequenceFileInputFormat.class); jobConf.setOutputFormat(SequenceFileOutputFormat.class); FileInputFormat.setInputPaths(jobConf, srcPath); FileOutputFormat.setOutputPath(jobConf, destPath); jobConf.setNumMapTasks(mapperNum);/*from w w w. ja v a 2s. co m*/ jobConf.setNumReduceTasks(reducerNum); jobConf.setCompressMapOutput(true); jobConf.setMapOutputCompressorClass(GzipCodec.class); try { this.runningJob = JobClient.runJob(jobConf); } catch (IOException e) { throw new ProcessorExecutionException(e); } }
From source file:org.sf.xrime.algorithms.transform.vertex.OutAdjVertex2LabeledSetWithLabelTransformer.java
License:Apache License
@SuppressWarnings("deprecation") @Override/*from ww w. j a va 2 s. co m*/ public void execute() throws ProcessorExecutionException { // Create a JobConf with default settings. JobConf jobConf = new JobConf(conf, OutAdjVertex2LabeledSetWithLabelTransformer.class); jobConf.setJobName("OutAdjVertex2LabeledSetWithLabelTransformer"); // the keys are vertex identifiers (strings) jobConf.setOutputKeyClass(Text.class); // the values are adjacent vertexes with labels (Writable) jobConf.setOutputValueClass(LabeledAdjSetVertexWithTwoHopLabel.class); jobConf.setMapperClass(MapClass.class); jobConf.setCombinerClass(ReduceClass.class); jobConf.setReducerClass(ReduceClass.class); // makes the file format suitable for machine processing. jobConf.setInputFormat(SequenceFileInputFormat.class); jobConf.setOutputFormat(SequenceFileOutputFormat.class); // Enable compression. jobConf.setCompressMapOutput(true); jobConf.setMapOutputCompressorClass(GzipCodec.class); FileInputFormat.setInputPaths(jobConf, srcPath); FileOutputFormat.setOutputPath(jobConf, destPath); jobConf.setNumMapTasks(mapperNum); jobConf.setNumReduceTasks(reducerNum); try { this.runningJob = JobClient.runJob(jobConf); System.out.println("Output Node Num. =" + this.runningJob.getCounters().getCounter(Counter.VertexNum)); } catch (IOException e) { throw new ProcessorExecutionException(e); } }
From source file:org.sf.xrime.algorithms.transform.vertex.OutAdjVertex2SetWithLabelTransformer.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { // Create a JobConf with default settings. JobConf jobConf = new JobConf(conf, OutAdjVertex2SetWithLabelTransformer.class); jobConf.setJobName("OutAdjVertex2AdjSetVertexWithLabelTransformer"); // the keys are vertex identifiers (strings) jobConf.setOutputKeyClass(Text.class); // the values are adjacent vertexes with labels (Writable) jobConf.setOutputValueClass(AdjSetVertexWithTwoHopLabel.class); jobConf.setMapperClass(MapClass.class); jobConf.setCombinerClass(ReduceClass.class); jobConf.setReducerClass(ReduceClass.class); // makes the file format suitable for machine processing. jobConf.setInputFormat(SequenceFileInputFormat.class); jobConf.setOutputFormat(SequenceFileOutputFormat.class); // Enable compression. jobConf.setCompressMapOutput(true);//from w w w. j ava 2 s.com jobConf.setMapOutputCompressorClass(GzipCodec.class); FileInputFormat.setInputPaths(jobConf, srcPath); FileOutputFormat.setOutputPath(jobConf, destPath); jobConf.setNumMapTasks(mapperNum); jobConf.setNumReduceTasks(reducerNum); try { this.runningJob = JobClient.runJob(jobConf); } catch (IOException e) { throw new ProcessorExecutionException(e); } }
From source file:org.sf.xrime.preprocessing.smth.Raw2InAdjVertexTransformer.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { JobConf jobConf = new JobConf(conf, Raw2InAdjVertexTransformer.class); jobConf.setJobName("Smth-InAdjVertexesList"); jobConf.setMapperClass(MapClass.class); jobConf.setCombinerClass(ReduceClass.class); jobConf.setReducerClass(ReduceClass.class); // makes the file format suitable for machine processing. jobConf.setOutputFormat(SequenceFileOutputFormat.class); // the keys are author names (strings) jobConf.setOutputKeyClass(Text.class); // the values are adjacent vertexes (Writable) jobConf.setOutputValueClass(AdjVertex.class); FileInputFormat.setInputPaths(jobConf, srcPath); FileOutputFormat.setOutputPath(jobConf, destPath); jobConf.setNumMapTasks(mapperNum);//from w w w . ja va 2s. c om jobConf.setNumReduceTasks(reducerNum); try { this.runningJob = JobClient.runJob(jobConf); } catch (IOException e) { throw new ProcessorExecutionException(e); } }