Example usage for org.apache.hadoop.mapreduce.lib.input FileInputFormat PATHFILTER_CLASS

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.input FileInputFormat PATHFILTER_CLASS.

Prototype

String PATHFILTER_CLASS

To view the source code for org.apache.hadoop.mapreduce.lib.input FileInputFormat PATHFILTER_CLASS.

Click Source Link

Usage

From source file:kina.config.GenericMongoKinaConfig.java

License:Apache License

private void initBSONDumpConfig() {
    inputFormatClass = KinaBSONFileInputFormat.class;

    Path path = new Path(bsonFile);
    try {/*  w  w w . j a va2s.com*/
        path = path.getFileSystem(configHadoop).makeQualified(path);
        if (!path.getFileSystem(configHadoop).exists(path)) {
            throw new IOException(new FileNotFoundException(path.getName()));
        }

        String dirStr = org.apache.hadoop.util.StringUtils.escapeString(path.toString());
        String dirs = configHadoop.get(FileInputFormat.INPUT_DIR);
        configHadoop.set(FileInputFormat.INPUT_DIR, dirs == null ? dirStr : dirs + "," + dirStr);
        configHadoop.set(FileInputFormat.INPUT_DIR_RECURSIVE, recursiveBsonFileDiscovery.toString());

        configHadoop.setClass(FileInputFormat.PATHFILTER_CLASS, KinaMongoPathFilter.class, PathFilter.class);

        if (bsonFilesExcludePatterns != null) {
            configHadoop.setStrings(KinaMongoPathFilter.PATH_FILTER_CONF, bsonFilesExcludePatterns);
        }

    } catch (IOException e) {
        throw new kina.exceptions.IOException(e);
    }
}

---

From source file:org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper.java

License:Apache License

private static void populateMRSettingsToRetain() {

    // FileInputFormat
    mrSettingsToRetain.add(FileInputFormat.INPUT_DIR);
    mrSettingsToRetain.add(FileInputFormat.SPLIT_MAXSIZE);
    mrSettingsToRetain.add(FileInputFormat.SPLIT_MINSIZE);
    mrSettingsToRetain.add(FileInputFormat.PATHFILTER_CLASS);
    mrSettingsToRetain.add(FileInputFormat.NUM_INPUT_FILES);
    mrSettingsToRetain.add(FileInputFormat.INPUT_DIR_RECURSIVE);

    // FileOutputFormat
    mrSettingsToRetain.add(MRConfiguration.OUTPUT_BASENAME);
    mrSettingsToRetain.add(FileOutputFormat.COMPRESS);
    mrSettingsToRetain.add(FileOutputFormat.COMPRESS_CODEC);
    mrSettingsToRetain.add(FileOutputFormat.COMPRESS_TYPE);
    mrSettingsToRetain.add(FileOutputFormat.OUTDIR);
    mrSettingsToRetain.add(FileOutputCommitter.SUCCESSFUL_JOB_OUTPUT_DIR_MARKER);
}

---