Example usage for org.apache.hadoop.mapreduce.lib.input CombineFileInputFormat subclass-usage

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.input CombineFileInputFormat subclass-usage.

Usage

From source file com.alexholmes.hadooputils.combine.common.mapreduce.SplitMetricsCombineInputFormat.java

/**
 * An input format that writes split details to a sink, the default sink being a logger.
 *
 * @param <K> The key type.
 * @param <V> The value type.
 */

From source file com.blackberry.logdriver.mapreduce.avro.AvroBlockInputFormat.java

public class AvroBlockInputFormat extends CombineFileInputFormat<AvroFileHeader, BytesWritable> {
    private static final Logger LOG = LoggerFactory.getLogger(AvroBlockInputFormat.class);
    private static final long MAX_SPLIT_LOCATIONS = 100000;

    /**
     * Avro files are always splitable.

From source file com.blackberry.logdriver.mapreduce.boom.BoomInputFormat.java

public class BoomInputFormat extends CombineFileInputFormat<LogLineData, Text> {
    private static final Logger LOG = LoggerFactory.getLogger(BoomInputFormat.class);

    private static final long MAX_SPLIT_LOCATIONS = 100000;

    @Override

From source file com.bonc.mr_roamRecognition_hjpt.comm.FirstJobCombineTextInputFormat.java

/**
 * Input format that is a <code>CombineFileInputFormat</code>-equivalent for
 * <code>TextInputFormat</code>.
 *
 * @see CombineFileInputFormat
 */

From source file com.bonc.mr_roamRecognition_hjpt.comm.PathCombineTextInputFormat.java

/**
 * Input format that is a <code>CombineFileInputFormat</code>-equivalent for
 * <code>TextInputFormat</code>.
 *
 * @see CombineFileInputFormat
 */

From source file com.cloudera.science.avro2parquet.CombinedAvroKeyInputFormat.java

/**
 * A combined input format for reading Avro data.
 * 
 * @author "Matthew Hayes"
 *
 * @param <T> Type of data to be read

From source file com.conversantmedia.mapreduce.input.CombineTextFileInputFormat.java

/**
 * Simple implementation of a combinefile input format.
 *
 * Concept for FileLineWritable borrowed from:
 * http://www.idryman.org/blog/2013/09/22/process-small-files-on-hadoop-using-combinefileinputformat-1/
 * Needed to rewrite to work properly in new API

From source file com.conversantmedia.mapreduce.io.avro.CombineAvroKeyFileInputFormat.java

/**
 * Custom implementation/extension of the combine file input format designed to
 * combine small Avro files and skip any empty files or bad records.
 *
 * @param <T>   The Avro record type
 */

From source file com.ikanow.aleph2.analytics.spark.assets.BeFileInputFormat_Pure.java

/** The file input format specific to batch enrichment modules - uses the default Hadoop CombineFileInputFormat
 *  for cases when the Hadoop version of google.commons.base.Stopwatch is first on the classpath!
 *  OTHERWISE IDENTICAL TO BeFileInputForm in hadoop_analytic_services
 * @author jfreydank
 */
public class BeFileInputFormat_Pure extends CombineFileInputFormat<String, Tuple2<Long, IBatchRecord>> {

From source file com.ikanow.infinit.e.data_model.custom.InfiniteFileInputFormat.java

public class InfiniteFileInputFormat extends CombineFileInputFormat<Object, BSONObject> {

    @Override
    protected boolean isSplitable(JobContext context, Path filename) {
        return false;
    }