List of usage examples for org.apache.hadoop.fs GlobFilter GlobFilter
public GlobFilter(String filePattern) throws IOException
From source file:com.streamsets.pipeline.stage.origin.hdfs.spooler.HdfsFileSystem.java
License:Apache License
public HdfsFileSystem(String filePattern, PathMatcherMode mode, boolean processSubdirectories, FileSystem fs) { this.filePattern = filePattern; this.processSubdirectories = processSubdirectories; this.fs = fs; try {//from w w w. j av a 2s. com if (mode == GLOB) { filter = new GlobFilter(filePattern); } else if (mode == REGEX) { Pattern pattern = Pattern.compile(filePattern); filter = path -> pattern.matcher(path.toString()).matches(); } else { throw new IllegalArgumentException("Unrecognized Path Matcher Mode: " + mode.getLabel()); } } catch (IOException e) { throw new IllegalArgumentException("Can't create filter pattern: " + e.toString(), e); } }
From source file:org.apache.hoya.avro.RoleHistoryWriter.java
License:Apache License
/** * Find all history entries in a dir. The dir is created if it is * not already defined.// w ww .ja v a 2 s .c o m * * The scan uses the match pattern {@link HoyaKeys#HISTORY_FILENAME_MATCH_PATTERN} * while dropping empty files and directories which match the pattern. * The list is then sorted with a comparator that sorts on filename, * relying on the filename of newer created files being later than the old ones. * * * * @param fs filesystem * @param dir dir to scan * @param includeEmptyFiles should empty files be included in the result? * @return a possibly empty list * @throws IOException IO problems * @throws FileNotFoundException if the target dir is actually a path */ public List<Path> findAllHistoryEntries(FileSystem fs, Path dir, boolean includeEmptyFiles) throws IOException { assert fs != null; assert dir != null; if (!fs.exists(dir)) { fs.mkdirs(dir); } else if (!fs.isDirectory(dir)) { throw new FileNotFoundException("Not a directory " + dir.toString()); } PathFilter filter = new GlobFilter(HoyaKeys.HISTORY_FILENAME_GLOB_PATTERN); FileStatus[] stats = fs.listStatus(dir, filter); List<Path> paths = new ArrayList<Path>(stats.length); for (FileStatus stat : stats) { log.debug("Possible entry: {}", stat.toString()); if (stat.isFile() && (includeEmptyFiles || stat.getLen() > 0)) { paths.add(stat.getPath()); } } sortHistoryPaths(paths); return paths; }
From source file:org.apache.slider.common.tools.SliderUtils.java
License:Apache License
/** * Copy a directory to a new FS -both paths must be qualified. If * a directory needs to be created, supplied permissions can override * the default values. Existing directories are not touched * @param conf conf file/*w w w .j a va2 s . c o m*/ * @param srcDirPath src dir * @param destDirPath dest dir * @param permission permission for the dest directory; null means "default" * @return # of files copies */ public static int copyDirectory(Configuration conf, Path srcDirPath, Path destDirPath, FsPermission permission) throws IOException, BadClusterStateException { FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf); FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf); //list all paths in the src. if (!srcFS.exists(srcDirPath)) { throw new FileNotFoundException("Source dir not found " + srcDirPath); } if (!srcFS.isDirectory(srcDirPath)) { throw new FileNotFoundException("Source dir not a directory " + srcDirPath); } GlobFilter dotFilter = new GlobFilter("[!.]*"); FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter); int srcFileCount = entries.length; if (srcFileCount == 0) { return 0; } if (permission == null) { permission = FsPermission.getDirDefault(); } if (!destFS.exists(destDirPath)) { new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath, permission); } Path[] sourcePaths = new Path[srcFileCount]; for (int i = 0; i < srcFileCount; i++) { FileStatus e = entries[i]; Path srcFile = e.getPath(); if (srcFS.isDirectory(srcFile)) { String msg = "Configuration dir " + srcDirPath + " contains a directory " + srcFile; log.warn(msg); throw new IOException(msg); } log.debug("copying src conf file {}", srcFile); sourcePaths[i] = srcFile; } log.debug("Copying {} files from {} to dest {}", srcFileCount, srcDirPath, destDirPath); FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf); return srcFileCount; }
From source file:org.apache.slider.server.avro.RoleHistoryWriter.java
License:Apache License
/** * Find all history entries in a dir. The dir is created if it is * not already defined.//from w w w . jav a 2 s . c om * * The scan uses the match pattern {@link SliderKeys#HISTORY_FILENAME_MATCH_PATTERN} * while dropping empty files and directories which match the pattern. * The list is then sorted with a comparator that sorts on filename, * relying on the filename of newer created files being later than the old ones. * * * * @param fs filesystem * @param dir dir to scan * @param includeEmptyFiles should empty files be included in the result? * @return a possibly empty list * @throws IOException IO problems * @throws FileNotFoundException if the target dir is actually a path */ public List<Path> findAllHistoryEntries(FileSystem fs, Path dir, boolean includeEmptyFiles) throws IOException { assert fs != null; assert dir != null; if (!fs.exists(dir)) { fs.mkdirs(dir); } else if (!fs.isDirectory(dir)) { throw new FileNotFoundException("Not a directory " + dir.toString()); } PathFilter filter = new GlobFilter(SliderKeys.HISTORY_FILENAME_GLOB_PATTERN); FileStatus[] stats = fs.listStatus(dir, filter); List<Path> paths = new ArrayList<Path>(stats.length); for (FileStatus stat : stats) { log.debug("Possible entry: {}", stat.toString()); if (stat.isFile() && (includeEmptyFiles || stat.getLen() > 0)) { paths.add(stat.getPath()); } } sortHistoryPaths(paths); return paths; }