Example usage for org.apache.hadoop.mapred JobConf getClass

List of usage examples for org.apache.hadoop.mapred JobConf getClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. getClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf getClass.

Prototype

public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface)

Source Link

Document

Get the value of the name property as a Class implementing the interface specified by xface.

Usage

From source file:com.moz.fiji.express.flow.framework.MapredInputFormatWrapper.java

License:Apache License

@SuppressWarnings("unchecked")
private void initInputFormat(JobConf conf) {
    if (realInputFormat == null) {
        realInputFormat = ReflectionUtils.newInstance(conf.getClass(CLASS_CONF_KEY, null, InputFormat.class),
                conf);/*from www  .j a v  a  2 s .  c  o  m*/
    }
}

From source file:com.ricemap.spateDB.mapred.IndexedPrism.java

License:Apache License

@SuppressWarnings("unchecked")
@Override//from   w w  w .j  ava 2  s. c  om
public InputSplit[] getSplits(final JobConf job, int numSplits) throws IOException {
    // Get a list of all input files. There should be exactly two files.
    final Path[] inputFiles = getInputPaths(job);
    GlobalIndex<Partition> gIndexes[] = new GlobalIndex[inputFiles.length];

    BlockFilter blockFilter = null;
    try {
        Class<? extends BlockFilter> blockFilterClass = job.getClass(SpatialSite.FilterClass, null,
                BlockFilter.class);
        if (blockFilterClass != null) {
            // Get all blocks the user wants to process
            blockFilter = blockFilterClass.newInstance();
            blockFilter.configure(job);
        }
    } catch (InstantiationException e1) {
        e1.printStackTrace();
    } catch (IllegalAccessException e1) {
        e1.printStackTrace();
    }

    if (blockFilter != null) {
        // Extract global indexes from input files

        for (int i_file = 0; i_file < inputFiles.length; i_file++) {
            FileSystem fs = inputFiles[i_file].getFileSystem(job);
            gIndexes[i_file] = SpatialSite.getGlobalIndex(fs, inputFiles[i_file]);
        }
    }

    final Vector<CombineFileSplit> matchedSplits = new Vector<CombineFileSplit>();
    if (gIndexes[0] == null || gIndexes[1] == null) {
        // Join every possible pair (Cartesian product)
        BlockLocation[][] fileBlockLocations = new BlockLocation[inputFiles.length][];
        for (int i_file = 0; i_file < inputFiles.length; i_file++) {
            FileSystem fs = inputFiles[i_file].getFileSystem(job);
            FileStatus fileStatus = fs.getFileStatus(inputFiles[i_file]);
            fileBlockLocations[i_file] = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
        }
        LOG.info("Doing a Cartesian product of blocks: " + fileBlockLocations[0].length + "x"
                + fileBlockLocations[1].length);
        for (BlockLocation block1 : fileBlockLocations[0]) {
            for (BlockLocation block2 : fileBlockLocations[1]) {
                FileSplit fsplit1 = new FileSplit(inputFiles[0], block1.getOffset(), block1.getLength(),
                        block1.getHosts());
                FileSplit fsplit2 = new FileSplit(inputFiles[1], block2.getOffset(), block2.getLength(),
                        block2.getHosts());
                CombineFileSplit combinedSplit = (CombineFileSplit) FileSplitUtil.combineFileSplits(job,
                        fsplit1, fsplit2);
                matchedSplits.add(combinedSplit);
            }
        }
    } else {
        // Filter block pairs by the BlockFilter
        blockFilter.selectCellPairs(gIndexes[0], gIndexes[1], new ResultCollector2<Partition, Partition>() {
            @Override
            public void collect(Partition p1, Partition p2) {
                try {
                    List<FileSplit> splits1 = new ArrayList<FileSplit>();
                    Path path1 = new Path(inputFiles[0], p1.filename);
                    splitFile(job, path1, splits1);

                    List<FileSplit> splits2 = new ArrayList<FileSplit>();
                    Path path2 = new Path(inputFiles[1], p2.filename);
                    splitFile(job, path2, splits2);

                    for (FileSplit split1 : splits1) {
                        for (FileSplit split2 : splits2) {
                            matchedSplits.add(
                                    (CombineFileSplit) FileSplitUtil.combineFileSplits(job, split1, split2));
                        }
                    }

                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });
    }

    LOG.info("Matched " + matchedSplits.size() + " combine splits");

    // Return all matched splits
    return matchedSplits.toArray(new InputSplit[matchedSplits.size()]);
}

From source file:com.ricemap.spateDB.mapred.SpatialInputFormat.java

License:Apache License

@Override
protected FileStatus[] listStatus(JobConf job) throws IOException {
    try {/*w ww.jav a  2s  .c om*/
        if (compressionCodecs == null)
            compressionCodecs = new CompressionCodecFactory(job);
        Class<? extends BlockFilter> blockFilterClass = job.getClass(SpatialSite.FilterClass, null,
                BlockFilter.class);
        if (blockFilterClass == null) {
            LOG.info("No block filter specified");
            // No block filter specified by user
            return super.listStatus(job);
        }
        // Get all blocks the user wants to process
        BlockFilter blockFilter;
        blockFilter = blockFilterClass.newInstance();
        blockFilter.configure(job);

        // Filter files based on user specified filter function
        List<FileStatus> result = new ArrayList<FileStatus>();
        Path[] dirs = getInputPaths(job);

        for (Path dir : dirs) {
            FileSystem fs = dir.getFileSystem(job);
            listStatus(fs, dir, result, blockFilter);
        }

        LOG.info("Spatial filter function matched with " + result.size() + " cells");

        return result.toArray(new FileStatus[result.size()]);
    } catch (InstantiationException e) {
        LOG.warn(e);
        return super.listStatus(job);
    } catch (IllegalAccessException e) {
        LOG.warn(e);
        return super.listStatus(job);
    }
}

From source file:edu.umn.cs.spatialHadoop.mapred.IndexedRectangle.java

License:Open Source License

@SuppressWarnings("unchecked")
@Override/*w  ww  . j  a v  a 2 s .c  o  m*/
public InputSplit[] getSplits(final JobConf job, int numSplits) throws IOException {
    // Get a list of all input files. There should be exactly two files.
    final Path[] inputFiles = getInputPaths(job);
    GlobalIndex<Partition> gIndexes[] = new GlobalIndex[inputFiles.length];

    BlockFilter blockFilter = null;
    try {
        Class<? extends BlockFilter> blockFilterClass = job.getClass(SpatialSite.FilterClass, null,
                BlockFilter.class);
        if (blockFilterClass != null) {
            // Get all blocks the user wants to process
            blockFilter = blockFilterClass.newInstance();
            blockFilter.configure(job);
        }
    } catch (InstantiationException e1) {
        e1.printStackTrace();
    } catch (IllegalAccessException e1) {
        e1.printStackTrace();
    }

    if (blockFilter != null) {
        // Extract global indexes from input files

        for (int i_file = 0; i_file < inputFiles.length; i_file++) {
            FileSystem fs = inputFiles[i_file].getFileSystem(job);
            gIndexes[i_file] = SpatialSite.getGlobalIndex(fs, inputFiles[i_file]);
        }
    }

    final Vector<CombineFileSplit> matchedSplits = new Vector<CombineFileSplit>();
    if (gIndexes[0] == null || gIndexes[1] == null) {
        // Join every possible pair (Cartesian product)
        InputSplit[][] inputSplits = new InputSplit[inputFiles.length][];

        for (int i_file = 0; i_file < inputFiles.length; i_file++) {
            JobConf temp = new JobConf(job);
            setInputPaths(temp, inputFiles[i_file]);
            inputSplits[i_file] = super.getSplits(temp, 1);
        }
        LOG.info("Doing a Cartesian product of blocks: " + inputSplits[0].length + "x" + inputSplits[1].length);
        for (InputSplit split1 : inputSplits[0]) {
            for (InputSplit split2 : inputSplits[1]) {
                CombineFileSplit combinedSplit = (CombineFileSplit) FileSplitUtil.combineFileSplits(job,
                        (FileSplit) split1, (FileSplit) split2);
                matchedSplits.add(combinedSplit);
            }
        }
    } else {
        // Filter block pairs by the BlockFilter
        blockFilter.selectCellPairs(gIndexes[0], gIndexes[1], new ResultCollector2<Partition, Partition>() {
            @Override
            public void collect(Partition p1, Partition p2) {
                try {
                    List<FileSplit> splits1 = new ArrayList<FileSplit>();
                    Path path1 = new Path(inputFiles[0], p1.filename);
                    splitFile(job, path1, splits1);

                    List<FileSplit> splits2 = new ArrayList<FileSplit>();
                    Path path2 = new Path(inputFiles[1], p2.filename);
                    splitFile(job, path2, splits2);

                    for (FileSplit split1 : splits1) {
                        for (FileSplit split2 : splits2) {
                            matchedSplits.add(
                                    (CombineFileSplit) FileSplitUtil.combineFileSplits(job, split1, split2));
                        }
                    }

                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });
    }

    LOG.info("Matched " + matchedSplits.size() + " combine splits");

    // Return all matched splits
    return matchedSplits.toArray(new InputSplit[matchedSplits.size()]);
}

From source file:edu.umn.cs.spatialHadoop.mapred.SpatialInputFormat.java

License:Open Source License

@Override
protected FileStatus[] listStatus(JobConf job) throws IOException {
    try {/* w ww  . jav  a2s  .  c  o  m*/
        // Create the compressionCodecs to be used later by isSplitable
        if (compressionCodecs == null)
            compressionCodecs = new CompressionCodecFactory(job);

        // Retrieve the BlockFilter set by the developers in the JobConf
        Class<? extends BlockFilter> blockFilterClass = job.getClass(SpatialSite.FilterClass, null,
                BlockFilter.class);
        if (blockFilterClass == null) {
            LOG.info("No block filter specified");
            // No block filter specified by user
            return super.listStatus(job);
        }
        // Get all blocks the user wants to process
        BlockFilter blockFilter;
        blockFilter = blockFilterClass.newInstance();
        blockFilter.configure(job);

        // Filter files based on user specified filter function
        List<FileStatus> result = new ArrayList<FileStatus>();
        Path[] inputDirs = getInputPaths(job);

        for (Path dir : inputDirs) {
            FileSystem fs = dir.getFileSystem(job);
            listStatus(fs, dir, result, blockFilter);
        }

        LOG.info("Spatial filter function matched with " + result.size() + " cells");

        return result.toArray(new FileStatus[result.size()]);
    } catch (InstantiationException e) {
        LOG.warn(e);
        return super.listStatus(job);
    } catch (IllegalAccessException e) {
        LOG.warn(e);
        return super.listStatus(job);
    }
}

From source file:it.crs4.pydoop.pipes.PipesNonJavaInputFormat.java

License:Apache License

public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
    // Delegate the generation of input splits to the 'original' InputFormat
    return ReflectionUtils
            .newInstance(job.getClass(Submitter.INPUT_FORMAT, TextInputFormat.class, InputFormat.class), job)
            .getSplits(job, numSplits);//  www.  j a  v a2  s .c  om
}

From source file:it.crs4.pydoop.pipes.Submitter.java

License:Apache License

/**
 * Get the user's original partitioner./*from   w ww.j  a  v  a 2s . co  m*/
 * @param conf the configuration to look in
 * @return the class that the user submitted
 */
static Class<? extends Partitioner> getJavaPartitioner(JobConf conf) {
    return conf.getClass(Submitter.PARTITIONER, HashPartitioner.class, Partitioner.class);
}

From source file:org.apache.avro.mapred.AvroMultipleOutputs.java

License:Apache License

/**
 * Returns the named output OutputFormat.
 *
 * @param conf        job conf//from  w  w w. j  av a  2 s.  c  o  m
 * @param namedOutput named output
 * @return namedOutput OutputFormat
 */
public static Class<? extends OutputFormat> getNamedOutputFormatClass(JobConf conf, String namedOutput) {
    checkNamedOutput(conf, namedOutput, false);
    return conf.getClass(MO_PREFIX + namedOutput + FORMAT, null, OutputFormat.class);
}

From source file:org.apache.avro.mapred.HadoopCombiner.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
protected AvroReducer<K, V, Pair<K, V>> getReducer(JobConf conf) {
    return ReflectionUtils.newInstance(conf.getClass(AvroJob.COMBINER, AvroReducer.class, AvroReducer.class),
            conf);//from www. j a va 2s  .  c  om
}

From source file:org.apache.avro.mapred.HadoopMapper.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
public void configure(JobConf conf) {
    this.mapper = ReflectionUtils.newInstance(conf.getClass(AvroJob.MAPPER, AvroMapper.class, AvroMapper.class),
            conf);//from  w ww . j a va 2s  .c om
    this.isMapOnly = conf.getNumReduceTasks() == 0;
}