Example usage for org.apache.mahout.text PrefixAdditionFilter PrefixAdditionFilter

Introduction

In this page you can find the example usage for org.apache.mahout.text PrefixAdditionFilter PrefixAdditionFilter.

Prototype

public PrefixAdditionFilter(Configuration conf, String keyPrefix, Map<String, String> options,
            ChunkedWriter writer, Charset charset, FileSystem fs) 

Source Link

Usage

From source file:com.netease.news.text.SequenceFilesFromDirectory.java

License:Apache License

private int runSequential(Configuration conf, Path input, Path output, Map<String, String> options)
        throws IOException, InterruptedException, NoSuchMethodException {
    // Running sequentially
    Charset charset = Charset.forName(getOption(CHARSET_OPTION[0]));
    String keyPrefix = getOption(KEY_PREFIX_OPTION[0]);
    FileSystem fs = FileSystem.get(input.toUri(), conf);
    ChunkedWriter writer = new ChunkedWriter(conf, Integer.parseInt(options.get(CHUNK_SIZE_OPTION[0])), output);

    try {/*from  w ww .  j  a v  a 2  s. co m*/
        SequenceFilesFromDirectoryFilter pathFilter;
        String fileFilterClassName = options.get(FILE_FILTER_CLASS_OPTION[0]);
        if (PrefixAdditionFilter.class.getName().equals(fileFilterClassName)) {
            pathFilter = new PrefixAdditionFilter(conf, keyPrefix, options, writer, charset, fs);
        } else {
            pathFilter = ClassUtils.instantiateAs(fileFilterClassName, SequenceFilesFromDirectoryFilter.class,
                    new Class[] { Configuration.class, String.class, Map.class, ChunkedWriter.class,
                            Charset.class, FileSystem.class },
                    new Object[] { conf, keyPrefix, options, writer, charset, fs });
        }
        fs.listStatus(input, pathFilter);
    } finally {
        Closeables.close(writer, false);
    }
    return 0;
}

---