Example usage for org.apache.hadoop.mapred JobConf JobConf

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf JobConf.

Prototype

public JobConf()

Source Link

Document

Construct a map/reduce job configuration.

Usage

From source file:com.hotels.plunger.TapDataWriter.java

License:Apache License

private void writeToHadoopTap(Tap<?, ?, ?> tap) throws IOException {
    @SuppressWarnings("unchecked")
    Tap<JobConf, ?, ?> hadoopTap = (Tap<JobConf, ?, ?>) tap;
    JobConf conf = new JobConf();

    HadoopFlowProcess flowProcess = new HadoopFlowProcess(conf);
    hadoopTap.sinkConfInit(flowProcess, conf);
    TupleEntryCollector collector = hadoopTap.openForWrite(flowProcess);
    for (TupleEntry tuple : data.asTupleEntryList()) {
        collector.add(tuple);/*from   ww w .j av  a  2  s  .c  om*/
    }
    collector.close();
}

From source file:com.hotels.plunger.TapDataWriter.java

License:Apache License

private void writeToHadoopPartitionTap(Tap<?, ?, ?> tap) throws IOException {
    @SuppressWarnings("unchecked")
    BasePartitionTap<JobConf, ?, ?> hadoopTap = (BasePartitionTap<JobConf, ?, ?>) tap;
    JobConf conf = new JobConf();

    // Avoids deletion of results when using a partition tap (close() will delete the _temporary before the copy has
    // been done if not in a flow)
    HadoopUtil.setIsInflow(conf);/* w w  w .  ja  v  a  2 s . co m*/

    HadoopFlowProcess flowProcess = new HadoopFlowProcess(conf);
    hadoopTap.sinkConfInit(flowProcess, conf);
    TupleEntryCollector collector = hadoopTap.openForWrite(flowProcess);
    for (TupleEntry tuple : data.asTupleEntryList()) {
        collector.add(tuple);
    }
    collector.close();

    // We need to clean up the '_temporary' folder
    BasePartitionTap<JobConf, ?, ?> partitionTap = hadoopTap;
    @SuppressWarnings("unchecked")
    String basePath = partitionTap.getParent().getFullIdentifier(flowProcess);
    deleteTemporaryPath(new Path(basePath), FileSystem.get(conf));
}