Example usage for org.apache.hadoop.mapreduce.lib.output MultipleOutputs MultipleOutputs

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.output MultipleOutputs MultipleOutputs.

Prototype

public MultipleOutputs(TaskInputOutputContext<?, ?, KEYOUT, VALUEOUT> context) 

Source Link

Document

Creates and initializes multiple outputs support, it should be instantiated in the Mapper/Reducer setup method.

Usage

From source file:org.apache.mahout.fpm.bigfim.ComputeTidListReducer.java

License:Apache License

@Override
public void setup(Context context) {
    Configuration conf = context.getConfiguration();

    minSup = conf.getInt(MIN_SUP_KEY, 1);
    subDbSize = conf.getInt(SUBDB_SIZE, 1);
    int numberOfMappers = conf.getInt(NUMBER_OF_MAPPERS_KEY, 1);
    bucketSizes = new ArrayList<AtomicInteger>(numberOfMappers);
    for (int i = 0; i < numberOfMappers; i++) {
        bucketSizes.add(new AtomicInteger());
    }/*from  w  w w.  jav a  2  s  .c  o m*/

    mos = new MultipleOutputs<IntArrayWritable, IntArrayWritable>(context);

}

---

From source file:org.apache.mahout.fpm.disteclat.ItemReaderReducer.java

License:Apache License

@Override
public void setup(Context context) {
    Configuration conf = context.getConfiguration();

    mos = new MultipleOutputs<Text, Writable>(context);
    numberOfMappers = parseInt(conf.get(NUMBER_OF_MAPPERS_KEY, "1"));
}

---

From source file:org.apache.mahout.fpm.disteclat.PrefixComputerReducer.java

License:Apache License

@Override
public void setup(Context context) {
    Configuration conf = context.getConfiguration();

    minSup = conf.getInt(MIN_SUP_KEY, 1);
    int numberOfMappers = conf.getInt(NUMBER_OF_MAPPERS_KEY, 1);
    bucketSizes = new ArrayList<AtomicInteger>(numberOfMappers);
    for (int i = 0; i < numberOfMappers; i++) {
        bucketSizes.add(new AtomicInteger());
    }//ww  w .j a  v a 2 s  .c o  m

    mos = new MultipleOutputs<IntArrayWritable, IntArrayWritable>(context);

}

---

From source file:org.apache.pirk.responder.wideskies.mapreduce.ColumnMultReducer.java

License:Apache License

@Override
public void setup(Context ctx) throws IOException, InterruptedException {
    super.setup(ctx);

    outputValue = new Text();
    mos = new MultipleOutputs<>(ctx);

    FileSystem fs = FileSystem.newInstance(ctx.getConfiguration());
    String queryDir = ctx.getConfiguration().get("pirMR.queryInputDir");
    query = new HadoopFileSystemStore(fs).recall(queryDir, Query.class);
}

---

From source file:org.apache.pirk.responder.wideskies.mapreduce.ExpTableReducer.java

License:Apache License

@Override
public void setup(Context ctx) throws IOException, InterruptedException {
    super.setup(ctx);
    mos = new MultipleOutputs<>(ctx);
    reducerID = String.format("%05d", ctx.getTaskAttemptID().getTaskID().getId());
    logger.info("reducerID = " + reducerID);
}

---

From source file:org.apache.pirk.responder.wideskies.mapreduce.FinalResponseReducer.java

License:Apache License

@Override
public void setup(Context ctx) throws IOException, InterruptedException {
    super.setup(ctx);

    mos = new MultipleOutputs<>(ctx);

    FileSystem fs = FileSystem.newInstance(ctx.getConfiguration());
    storage = new HadoopFileSystemStore(fs);
    String queryDir = ctx.getConfiguration().get("pirMR.queryInputDir");
    Query query = storage.recall(queryDir, Query.class);
    QueryInfo queryInfo = query.getQueryInfo();

    outputFile = ctx.getConfiguration().get("pirMR.outputFile");

    response = new Response(queryInfo);
}

---

From source file:org.apache.pirk.responder.wideskies.mapreduce.RowCalcReducer.java

License:Apache License

@Override
public void setup(Context ctx) throws IOException, InterruptedException {
    super.setup(ctx);

    keyOut = new LongWritable();
    valueOut = new Text();
    mos = new MultipleOutputs<>(ctx);

    fs = FileSystem.newInstance(ctx.getConfiguration());
    String queryDir = ctx.getConfiguration().get("pirMR.queryInputDir");
    query = new HadoopFileSystemStore(fs).recall(queryDir, Query.class);
    queryInfo = query.getQueryInfo();//from   www.ja v a  2s.  co m

    try {
        SystemConfiguration.setProperty("data.schemas", ctx.getConfiguration().get("data.schemas"));
        SystemConfiguration.setProperty("query.schemas", ctx.getConfiguration().get("query.schemas"));
        SystemConfiguration.setProperty("pir.stopListFile", ctx.getConfiguration().get("pirMR.stopListFile"));

        DataSchemaLoader.initialize(true, fs);
        QuerySchemaLoader.initialize(true, fs);

    } catch (Exception e) {
        e.printStackTrace();
    }

    if (ctx.getConfiguration().get("pirWL.useLocalCache").equals("true")) {
        useLocalCache = true;
    }
    if (ctx.getConfiguration().get("pirWL.limitHitsPerSelector").equals("true")) {
        limitHitsPerSelector = true;
    }
    maxHitsPerSelector = Integer.parseInt(ctx.getConfiguration().get("pirWL.maxHitsPerSelector"));

    logger.info("RowCalcReducer -- useLocalCache = " + useLocalCache + " limitHitsPerSelector =  "
            + limitHitsPerSelector + " maxHitsPerSelector = " + maxHitsPerSelector);
}

---

From source file:org.apache.rya.accumulo.mr.merge.reducers.MultipleFileReducer.java

License:Apache License

@Override
protected void setup(final Context context) {
    mos = new MultipleOutputs<>(context);
}

---

From source file:org.apache.sqoop.mapreduce.mainframe.MainframeDatasetImportMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    inputSplit = (MainframeDatasetInputSplit) context.getInputSplit();
    mos = new MultipleOutputs<Text, NullWritable>(context);
    numberOfRecords = 0;//from   ww  w  . j  a  v a  2 s. c om
    outkey = new Text();
}

---

From source file:org.bgi.flexlab.gaea.tools.mapreduce.annotator.AnnotationReducer.java

License:Open Source License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    long setupStart = System.currentTimeMillis();
    conf = context.getConfiguration();/*from  ww  w  .j  a v  a 2s  .  c  o m*/

    long start = System.currentTimeMillis();
    ReferenceShare genomeShare = new ReferenceShare();
    genomeShare.loadChromosomeList();

    System.err.println("genomeShare" + (System.currentTimeMillis() - start) + "");

    Config userConfig = new Config(conf, genomeShare);
    userConfig.setVerbose(conf.getBoolean("verbose", false));
    userConfig.setDebug(conf.getBoolean("debug", false));

    start = System.currentTimeMillis();
    AnnotatorBuild annoBuild = new AnnotatorBuild(userConfig);
    userConfig.setSnpEffectPredictor(annoBuild.createSnpEffPredictor());
    annoBuild.buildForest();
    System.err.println("build SnpEffectPredictor" + (System.currentTimeMillis() - start) + "");
    sampleNames = new ArrayList<>();
    vcfCodecs = new HashMap<>();
    Path inputPath = new Path(conf.get("inputFilePath"));
    FileSystem fs = inputPath.getFileSystem(conf);
    FileStatus[] files = fs.listStatus(inputPath);

    for (FileStatus file : files) {
        System.out.println(file.getPath());
        if (file.isFile()) {
            SingleVCFHeader singleVcfHeader = new SingleVCFHeader();
            singleVcfHeader.readHeaderFrom(file.getPath(), fs);
            VCFHeader vcfHeader = singleVcfHeader.getHeader();
            VCFHeaderVersion vcfVersion = singleVcfHeader.getVCFVersion(vcfHeader);
            VCFCodec vcfcodec = new VCFCodec();
            vcfcodec.setVCFHeader(vcfHeader, vcfVersion);
            vcfCodecs.put(file.getPath().getName(), vcfcodec);
            System.out.println("getname: " + file.getPath().getName());

            sampleNames.addAll(vcfHeader.getSampleNamesInOrder());
            System.out.println(sampleNames.toString());
        }

    }

    multipleOutputs = new MultipleOutputs(context);
    System.err.println("getVCFHeader" + (System.currentTimeMillis() - start) + "");

    vcfAnnotator = new VcfAnnotator(userConfig);

    start = System.currentTimeMillis();
    //??
    dbAnnotator = new DBAnnotator(userConfig);
    try {
        dbAnnotator.connection();
    } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
        e.printStackTrace();
    }
    System.err.println("dbAnnotator.connection" + (System.currentTimeMillis() - start) + "");

    // header?
    resultValue.set(userConfig.getHeader());
    for (int i = 0; i < sampleNames.size(); i++) {
        multipleOutputs.write(SampleNameModifier.modify(sampleNames.get(i)), NullWritable.get(), resultValue,
                sampleNames.get(i) + "/part");
    }
    System.err.println("mapper.setup" + (System.currentTimeMillis() - setupStart) + "");
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »