Example usage for org.apache.hadoop.fs PathFilter accept

Introduction

In this page you can find the example usage for org.apache.hadoop.fs PathFilter accept.

Prototype

boolean accept(Path path);

Source Link

Document

Tests whether or not the specified abstract pathname should be included in a pathname list.

Usage

From source file:org.apache.nifi.processors.hadoop.inotify.TestNotificationEventPathFilter.java

License:Apache License

@Test
public void acceptPathShouldProperlyMatchWhenWatchDirectoryMatchesPath() throws Exception {
    PathFilter filter = new NotificationEventPathFilter(Pattern.compile("/root(/.*)?"), false);
    assertTrue(filter.accept(new Path("/root")));
}

From source file:sifti.mongo.bson.BSONFileInputFormatGeneric.java

License:Apache License

@Override
public List<InputSplit> getSplits(final JobContext context) throws IOException {
    Configuration config = context.getConfiguration();
    PathFilter pf = getInputPathFilter(context);
    ArrayList<InputSplit> splits = new ArrayList<InputSplit>();
    List<FileStatus> inputFiles = listStatus(context);
    for (FileStatus file : inputFiles) {
        if (pf != null && !pf.accept(file.getPath())) {
            LOG.info("skipping file " + file.getPath() + " not matched path filter.");
            continue;
        } else {/* w ww  .j ava  2 s.  c  om*/
            LOG.info("processing file " + file.getPath());
        }

        BSONSplitter splitter = new BSONSplitter();
        splitter.setConf(config);
        splitter.setInputPath(file.getPath());
        Path splitFilePath = new Path(file.getPath().getParent(), "." + file.getPath().getName() + ".splits");
        try {
            splitter.loadSplitsFromSplitFile(file, splitFilePath);
        } catch (BSONSplitter.NoSplitFileException nsfe) {
            LOG.info("No split file for " + file + "; building split file");
            splitter.readSplitsForFile(file);
        }
        LOG.info("BSONSplitter found " + splitter.getAllSplits().size() + " splits.");
        splits.addAll(splitter.getAllSplits());
    }
    LOG.info("Total of " + splits.size() + " found.");
    return splits;
}

From source file:voldemort.store.readonly.mr.utils.HadoopUtils.java

License:Apache License

/**
 * Given a directory path, return the paths of all directories in that tree
 * that have no sub-directories./*  w w w  . j  ava  2s.co  m*/
 * 
 * @param directory
 * @return
 */
public static List<String> getLowestLevelDirectories(FileSystem fs, Path directory, PathFilter pathFilter)
        throws IOException {
    List<String> lowestLevelDirectories = new ArrayList<String>();

    if (hasSubDirectories(fs, directory)) {
        // recurse on each of this directory's sub-directories, ignoring any
        // files in the
        // directory
        FileStatus[] statuses = fs.listStatus(directory);
        for (FileStatus status : statuses) {
            if (status.isDir()) {
                lowestLevelDirectories.addAll(getLowestLevelDirectories(fs, status.getPath(), pathFilter));
            }
        }
    } else if (pathFilter == null || pathFilter.accept(directory)) {
        // this directory has no sub-directories, and either there is no
        // filter or it passes the
        // filter, so add it and return
        lowestLevelDirectories.add(directory.toString());
    }

    return lowestLevelDirectories;
}