Example usage for org.apache.hadoop.mapred JobConf setNumMapTasks

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf setNumMapTasks.

Prototype

public void setNumMapTasks(int n)

Source Link

Document

Set the number of map tasks for this job.

Usage

From source file:org.sf.xrime.algorithms.transform.vertex.OutAdjVertex2StrongLabeledSWLTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    // Create a JobConf with default settings.
    JobConf jobConf = new JobConf(conf, OutAdjVertex2StrongLabeledSWLTransformer.class);
    jobConf.setJobName("OutAdjVertex2StrongLabeledSWLTransformer");

    jobConf.setMapOutputKeyClass(Text.class);
    jobConf.setMapOutputValueClass(LabeledAdjSetVertex.class);

    jobConf.setOutputKeyClass(Text.class);
    jobConf.setOutputValueClass(LabeledAdjSetVertexWithTwoHopLabel.class);

    jobConf.setMapperClass(MapClass.class);
    jobConf.setReducerClass(ReduceClass.class);

    // makes the file format suitable for machine processing.
    jobConf.setInputFormat(SequenceFileInputFormat.class);
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    // Enable compression.
    jobConf.setCompressMapOutput(true);//from w ww  .  j  a va2  s. co  m
    jobConf.setMapOutputCompressorClass(GzipCodec.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);
    jobConf.setNumReduceTasks(reducerNum);

    try {
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.algorithms.transform.vertex.OutAdjVertex2StrongSetWithLabelTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    // Create a JobConf with default settings.
    JobConf jobConf = new JobConf(conf, OutAdjVertex2StrongSetWithLabelTransformer.class);
    jobConf.setJobName("OutAdjVertex2StrongSetWithLabelTransformer");

    jobConf.setMapOutputKeyClass(Text.class);
    jobConf.setMapOutputValueClass(LabeledAdjSetVertex.class);

    jobConf.setOutputKeyClass(Text.class);
    jobConf.setOutputValueClass(AdjSetVertexWithTwoHopLabel.class);

    jobConf.setMapperClass(MapClass.class);
    jobConf.setReducerClass(ReduceClass.class);

    // makes the file format suitable for machine processing.
    jobConf.setInputFormat(SequenceFileInputFormat.class);
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    // Enable compression.
    jobConf.setCompressMapOutput(true);//from   w  w  w.  j  av  a 2s.  c  o m
    jobConf.setMapOutputCompressorClass(GzipCodec.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);
    jobConf.setNumReduceTasks(reducerNum);

    try {
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.postprocessing.SequenceFileToTextFileTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf convertor = new JobConf(conf, SequenceFileToTextFileTransformer.class);
    convertor.setJobName("SequenceFileToTextFileTransformer");

    convertor.setInputFormat(SequenceFileInputFormat.class);
    convertor.setOutputFormat(TextOutputFormat.class);

    convertor.setMapperClass(SequenceFileToTextFileMapper.class);
    convertor.setMapOutputValueClass(Text.class);
    convertor.setOutputKeyClass(Text.class);
    convertor.setOutputValueClass(Text.class);

    // ONLY mapper, no combiner, no reducer.
    convertor.setNumMapTasks(getMapperNum());
    convertor.setNumReduceTasks(0);// w w w. j  a  v  a2  s. com

    FileInputFormat.setInputPaths(convertor, srcPath);
    FileOutputFormat.setOutputPath(convertor, destPath);
    try {
        this.runningJob = JobClient.runJob(convertor);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.preprocessing.pajek.Pajek2AdjVertexTransformer.java

License:Apache License

/**
 * Merge the sublists of incoming adjacent vertexes of a vertex into one.
 *///w  w w  . j a v  a 2  s . c  o  m
/*
public static class ReduceClass extends MapReduceBase implements Reducer<Text, AdjVertex, Text, AdjVertex> {
        
   public void reduce(Text key, Iterator<AdjVertex> values,
    OutputCollector<Text, AdjVertex> output, 
    Reporter reporter) throws IOException {
        
 HashSet<String> dest = new HashSet<String>();
        
        
 while (values.hasNext()) {
    AdjVertex neighbour = values.next();
    List<Edge> edges = neighbour.getEdges(); 
    if(edges!=null){
       for(Iterator<Edge> iterator = edges.iterator(); iterator.hasNext();){
          dest.add(iterator.next().getTo());                                   //get the dest of edge
       }
    }
 }
        
 // Create the AdjVertex instance.
 AdjVertex adj = new AdjVertex();
 String source = key.toString();
 // Set the vertex.
 adj.setId(source);
        
        
 Iterator<String> iterator;
 for(iterator = dest.iterator(); iterator.hasNext();){
    String destvert= iterator.next();
    Edge edge = new Edge();
    edge.setFrom(source);
    edge.setTo(destvert);
    adj.addEdge(edge);
 }
 output.collect(key, adj);
   }
}*/

public void execute() throws ProcessorExecutionException {
    JobConf jobConf = new JobConf(conf, Pajek2AdjVertexTransformer.class);
    jobConf.setJobName("tansfer_pajek2Adjvert");

    jobConf.setMapperClass(MapClass.class);
    jobConf.setReducerClass(SmthReducer.class);

    // makes the file format suitable for machine processing.
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    // the keys are author names (strings)
    jobConf.setOutputKeyClass(Text.class);
    // the values are adjacent vertexes (Writable)
    jobConf.setOutputValueClass(AdjVertex.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);
    jobConf.setNumReduceTasks(reducerNum);

    try {
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.preprocessing.smth.Raw2InAdjVertexTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf jobConf = new JobConf(conf, Raw2InAdjVertexTransformer.class);
    jobConf.setJobName("Smth-InAdjVertexesList");

    jobConf.setMapperClass(MapClass.class);
    jobConf.setCombinerClass(ReduceClass.class);
    jobConf.setReducerClass(ReduceClass.class);

    // makes the file format suitable for machine processing.
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    // the keys are author names (strings)
    jobConf.setOutputKeyClass(Text.class);
    // the values are adjacent vertexes (Writable)
    jobConf.setOutputValueClass(AdjVertex.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);
    jobConf.setNumReduceTasks(reducerNum);

    try {/*from w ww.  ja  v  a  2  s.  c o m*/
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.preprocessing.smth.Raw2OutAdjVertexTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf jobConf = new JobConf(conf, Raw2OutAdjVertexTransformer.class);
    jobConf.setJobName("Smth - OutAdjVertexesList");

    jobConf.setMapperClass(MapClass.class);
    jobConf.setCombinerClass(ReduceClass.class);
    jobConf.setReducerClass(ReduceClass.class);

    // the keys are author names (strings)
    jobConf.setOutputKeyClass(Text.class);
    // the values are adjacent vertexes (Writable)
    jobConf.setOutputValueClass(AdjVertex.class);

    // makes the file format suitable for machine processing.
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);
    jobConf.setNumReduceTasks(reducerNum);

    try {/*  w  w  w .  j a v  a  2 s  .c om*/
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.preprocessing.smth.Raw2SortedInAdjVertexTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf jobConf = new JobConf(conf, Raw2SortedInAdjVertexTransformer.class);
    jobConf.setJobName("Smth - SortedInAdjVertexesList");

    // the keys are author names (strings)
    jobConf.setOutputKeyClass(Text.class);
    // the values are adjacent vertexes (Writable)
    jobConf.setOutputValueClass(AdjVertex.class);

    jobConf.setMapperClass(MapClass.class);
    jobConf.setCombinerClass(ReduceClass.class);
    jobConf.setReducerClass(ReduceClass.class);

    // makes the file format suitable for machine processing.
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);
    jobConf.setNumReduceTasks(reducerNum);

    try {/*from  ww w  .j av  a  2s  . c  o m*/
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.preprocessing.smth.Raw2SortedOutAdjVertexTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf jobConf = new JobConf(conf, Raw2SortedOutAdjVertexTransformer.class);
    jobConf.setJobName("Smth - SortedOutAdjVertexesList");

    // the keys are author names (strings)
    jobConf.setOutputKeyClass(Text.class);
    // the values are adjacent vertexes (Writable)
    jobConf.setOutputValueClass(AdjVertex.class);

    jobConf.setMapperClass(MapClass.class);
    jobConf.setCombinerClass(ReduceClass.class);
    jobConf.setReducerClass(ReduceClass.class);

    // makes the file format suitable for machine processing.
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);
    jobConf.setNumReduceTasks(reducerNum);

    try {/*  w w w.  ja va  2 s  .c  o m*/
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        e.printStackTrace();
    }
}

From source file:org.sf.xrime.preprocessing.smth.SmthTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf smthPreprocess = new JobConf(conf, SmthTransformer.class);
    smthPreprocess.setJobName("SmthPreprocess");

    smthPreprocess.setOutputFormat(SequenceFileOutputFormat.class);

    smthPreprocess.setMapperClass(SmthMapper.class);
    smthPreprocess.setReducerClass(SmthReducer.class);
    smthPreprocess.setNumMapTasks(mapperNum);
    smthPreprocess.setNumReduceTasks(reducerNum);

    smthPreprocess.setMapOutputValueClass(AdjVertex.class);
    smthPreprocess.setOutputKeyClass(Text.class);
    smthPreprocess.setOutputValueClass(AdjVertex.class);

    FileInputFormat.setInputPaths(smthPreprocess, srcPath);
    FileOutputFormat.setOutputPath(smthPreprocess, destPath);

    try {/*w w  w .j  a va  2  s . c  o  m*/
        this.runningJob = JobClient.runJob(smthPreprocess);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }

}

From source file:org.sf.xrime.preprocessing.textadj.TextAdjTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf textAdjPreprocess = new JobConf(conf, TextAdjTransformer.class);
    textAdjPreprocess.setJobName("SmthPreprocess");

    textAdjPreprocess.setOutputFormat(SequenceFileOutputFormat.class);

    textAdjPreprocess.setMapperClass(TextAdjMapper.class);
    textAdjPreprocess.setReducerClass(SmthReducer.class);
    textAdjPreprocess.setNumMapTasks(mapperNum);
    textAdjPreprocess.setNumReduceTasks(reducerNum);

    textAdjPreprocess.setMapOutputValueClass(AdjVertex.class);
    textAdjPreprocess.setOutputKeyClass(Text.class);
    textAdjPreprocess.setOutputValueClass(AdjVertex.class);

    FileInputFormat.setInputPaths(textAdjPreprocess, srcPath);
    FileOutputFormat.setOutputPath(textAdjPreprocess, destPath);

    try {/*from   ww w. jav a  2  s.c o m*/
        this.runningJob = JobClient.runJob(textAdjPreprocess);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}