Example usage for org.apache.hadoop.mapred JobConf getClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf getClass.

Prototype

public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface)

Source Link

Document

Get the value of the name property as a Class implementing the interface specified by xface.

Usage

From source file:org.apache.avro.mapred.HadoopReducer.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
protected AvroReducer<K, V, OUT> getReducer(JobConf conf) {
    return ReflectionUtils.newInstance(conf.getClass(AvroJob.REDUCER, AvroReducer.class, AvroReducer.class),
            conf);/*from   w  w  w  .java  2s . c  om*/
}

From source file:org.archive.hadoop.PerMapOutputFormat.java

License:Apache License

private OutputFormat<K, V> getOutputFormat(JobConf job) {
    return ReflectionUtils.newInstance(
            job.getClass("permap.output.format.class", SequenceFileOutputFormat.class, OutputFormat.class),
            job);/*from  w w  w .j  ava2s.c om*/
}

From source file:org.commoncrawl.hadoop.io.ARCInputFormat.java

License:Open Source License

/**
 * @inheritDoc//from   www . jav  a 2  s .  c  o m
 */
public void configure(JobConf job) {

    blockSize = job.getInt(P_IO_BLOCK_SIZE, 32 * 1024);
    int bufferSize = job.getInt(P_IO_BUFFER_SIZE, 10 * 1024 * 1024);
    int queueSize = Math.max(1, bufferSize / blockSize);
    int timeout = job.getInt(P_IO_TIMEOUT, 60 * 1000);

    ArcFileReader.setBlockSize(blockSize);
    ArcFileReader.setBufferQueueSize(queueSize);
    ArcFileReader.setIOTimeoutValue(timeout);

    LOG.info("Block Size: " + blockSize);
    LOG.info("Queue Size: " + queueSize);
    LOG.info("IO Timeout: " + timeout);

    Class archiveSourceClass = job.getClass(P_ARC_SOURCE, JetS3tARCSource.class, ARCSource.class);
    arcSource = (ARCSource) ReflectionUtils.newInstance(archiveSourceClass, job);
}

From source file:org.sf.xrime.algorithms.transform.vertex.AdjVertex2AdjSetVertexMapper.java

License:Apache License

public void configure(JobConf job) {
    super.configure(job);

    Class<? extends AdjVertex2AdjSetVertexTransformer.EdgeFilter> labelAdderClass = job.getClass(
            AdjVertex2AdjSetVertexTransformer.edgeFilterKey,
            AdjVertex2AdjSetVertexTransformer.EverythingEdgeFilter.class,
            AdjVertex2AdjSetVertexTransformer.EdgeFilter.class);
    try {/*w w w  . jav  a  2  s.c o m*/
        filter = labelAdderClass.newInstance();
    } catch (InstantiationException e) {
        filter = new AdjVertex2AdjSetVertexTransformer.EverythingEdgeFilter();
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        filter = new AdjVertex2AdjSetVertexTransformer.EverythingEdgeFilter();
        e.printStackTrace();
    }
}

From source file:org.sf.xrime.algorithms.transform.vertex.Vertex2LabeledMapper.java

License:Apache License

public void configure(JobConf job) {
    super.configure(job);

    conf = job;/*from  www . j a  va 2  s.  c  o  m*/

    Class<? extends LabelAdder> labelAdderClass = job.getClass(Vertex2LabeledTransformer.labelFactoryKey, null,
            LabelAdder.class);
    try {
        if (labelAdderClass == null) {
            // Allow null label adder, i.e., allow Labeled objects with empty initial label set.
            theLabelAdder = null;
        } else {
            theLabelAdder = labelAdderClass.newInstance();
            theLabelAdder.configure(job);
        }
    } catch (InstantiationException e) {
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    }
}

From source file:tap.core.CombinerBridge.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
protected TapReducerInterface<V, V> getReducer(JobConf conf) {
    return ReflectionUtils
            .newInstance(conf.getClass(Phase.COMBINER, TapReducer.class, TapReducerInterface.class), conf);
}

From source file:tap.core.MapperBridge.java

License:Apache License

@SuppressWarnings("unchecked")
@Override// w  w  w  .  j a v a 2s  .c o m
public void configure(JobConf conf) {
    this.mapper = ReflectionUtils.newInstance(conf.getClass(Phase.MAPPER, TapMapper.class, TapMapper.class),
            conf);
    this.isMapOnly = conf.getNumReduceTasks() == 0;
    try {
        determineInputFormat(conf);
        determineOutputFormat(conf);
        this.groupBy = conf.get(Phase.GROUP_BY);
        this.sortBy = conf.get(Phase.SORT_BY);
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        throw new RuntimeException(e);
    }

    mapper.setConf(conf);
    mapper.init(conf.get("map.input.file"));
}

From source file:tap.core.MapperBridge.java

License:Apache License

/**
 * @param conf//  w  w  w  . j  av  a  2  s . c  om
 * @throws IOException
 * @throws FileNotFoundException
 */
private void determineInputFormat(JobConf conf) throws FileNotFoundException, IOException {

    /**
     * Compare mapper input file signature with Hadoop configured class
     */
    FileFormat ff = sniffMapInFormat(conf);
    if (!ff.isCompatible(conf.getInputFormat())) {
        throw new IllegalArgumentException("Map input format not compatible with file format.");
    }

    //otherwise assume it is avro?
    if (conf.getInputFormat() instanceof TextInputFormat) {
        Class<?> inClass = conf.getClass(Phase.MAP_IN_CLASS, Object.class, Object.class);
        if (inClass == String.class) {
            isStringInput = true;
        } else if (inClass == Text.class) {
            isTextInput = true;
        } else {
            isJsonInput = true;
            inSchema = ReflectUtils.getSchema((IN) ReflectionUtils.newInstance(inClass, conf));
        }
    }
    isProtoInput = conf.getInputFormat() instanceof TapfileInputFormat;
}

From source file:tap.core.MapperBridge.java

License:Apache License

/**
 * @param conf// w  w  w . j a  v  a  2s .c om
 */
@SuppressWarnings("unchecked")
private void determineOutputFormat(JobConf conf) throws Exception {

    Class<?> outClass = conf.getClass(Phase.MAP_OUT_CLASS, Object.class, Object.class);
    OUT out;
    out = (OUT) ObjectFactory.newInstance(outClass);

    /*OUT out = (OUT) ReflectionUtils.newInstance(
        conf.getClass(Phase.MAP_OUT_CLASS, Object.class, Object.class),
        conf);
       */
    outPipe = new Pipe<OUT>(out);
    schema = ReflectUtils.getSchema(out);
}

From source file:tap.core.ReducerBridge.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
protected TapReducerInterface<V, OUT> getReducer(JobConf conf) {
    Class<? extends TapReducerInterface> theClass = conf.getClass(Phase.REDUCER, TapReducer.class,
            TapReducerInterface.class);
    return ReflectionUtils.newInstance(theClass, conf);
}