Example usage for org.apache.hadoop.mapreduce.lib.input MultipleInputs DIR_FORMATS

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.input MultipleInputs DIR_FORMATS.

Prototype

String DIR_FORMATS

To view the source code for org.apache.hadoop.mapreduce.lib.input MultipleInputs DIR_FORMATS.

Click Source Link

Usage

From source file:org.apache.druid.indexer.path.StaticPathSpec.java

License:Apache License

private static void addInputPath(Job job, Iterable<String> pathStrings,
        Class<? extends InputFormat> inputFormatClass) {
    Configuration conf = job.getConfiguration();
    StringBuilder inputFormats = new StringBuilder(
            StringUtils.nullToEmptyNonDruidDataString(conf.get(MultipleInputs.DIR_FORMATS)));

    String[] paths = Iterables.toArray(pathStrings, String.class);
    for (int i = 0; i < paths.length - 1; i++) {
        if (inputFormats.length() > 0) {
            inputFormats.append(',');
        }/*from w w w . ja v a  2 s.  co  m*/
        inputFormats.append(paths[i]).append(';').append(inputFormatClass.getName());
    }
    if (inputFormats.length() > 0) {
        conf.set(MultipleInputs.DIR_FORMATS, inputFormats.toString());
    }
    // add last one separately for possible initialization in MultipleInputs
    MultipleInputs.addInputPath(job, new Path(paths[paths.length - 1]), inputFormatClass);
}

---

From source file:org.apache.druid.indexer.path.StaticPathSpecTest.java

License:Apache License

@Test
public void testAddingPaths() throws Exception {
    Job job = new Job();
    StaticPathSpec pathSpec = new StaticPathSpec("/a/c,/a/b/{c,d}", null);

    DataSchema schema = new DataSchema("ds", null, new AggregatorFactory[0], null, null, jsonMapper);
    HadoopIOConfig io = new HadoopIOConfig(null, null, null);
    pathSpec.addInputPaths(new HadoopDruidIndexerConfig(new HadoopIngestionSpec(schema, io, null)), job);

    String paths = job.getConfiguration().get(MultipleInputs.DIR_FORMATS);
    String formatter = TextInputFormat.class.getName();
    String[] expected = { "/a/c;" + formatter, "/a/b/c;" + formatter, "/a/b/d;" + formatter };
    Assert.assertArrayEquals(expected, paths.split(","));
}

---