List of usage examples for org.apache.hadoop.fs PathFilter PathFilter
PathFilter
From source file:com.streamsets.pipeline.stage.origin.hdfs.spooler.HdfsFileSystem.java
License:Apache License
public void addDirectory(WrappedFile dirPath, List<WrappedFile> directories) throws Exception { PathFilter pathFilter = new PathFilter() { @Override//from w w w .j a v a 2 s.c o m public boolean accept(Path entry) { try { FileStatus fileStatus = fs.getFileStatus(entry); if (fileStatus.isDirectory()) { if (processSubdirectories) { directories.add(new HdfsFile(fs, entry)); } return false; } } catch (IOException ex) { LOG.error("Failed to open file {}", entry.toString(), ex); } return false; } }; fs.globStatus(new Path(dirPath.getAbsolutePath(), "*"), pathFilter); }
From source file:com.streamsets.pipeline.stage.origin.hdfs.spooler.HdfsFileSystem.java
License:Apache License
public void handleOldFiles(WrappedFile dirpath, WrappedFile startingFile, boolean useLastModified, List<WrappedFile> toProcess) throws IOException { PathFilter pathFilter = new PathFilter() { @Override//from w ww . j a v a 2s . com public boolean accept(Path entry) { if (!patternMatches(entry.getName())) { LOG.debug("Ignoring old file '{}' that do not match the file name pattern '{}'", entry.getName(), filePattern); return false; } if (startingFile == null) { return false; } if (compare(new HdfsFile(fs, entry), startingFile, useLastModified) < 0) { toProcess.add(new HdfsFile(fs, entry)); } return false; } }; Path path = new Path(dirpath.getAbsolutePath(), "*"); fs.globStatus(path, pathFilter); if (processSubdirectories) { fs.globStatus(new Path(path, "*"), pathFilter); } }
From source file:com.twitter.algebra.matrix.format.MapDir.java
License:Apache License
private static long dirSize(FileStatus dirStatus, FileSystem fs) throws FileNotFoundException, IOException { FileStatus[] files = fs.listStatus(dirStatus.getPath(), new PathFilter() { @Override/*from www. ja v a 2 s . c o m*/ public boolean accept(final Path file) { return true; } }); long size = 0; for (FileStatus file : files) size += file.getLen(); return size; }
From source file:com.uber.hoodie.common.model.HoodieTableMetadata.java
License:Apache License
/** * Scan the commit times (only choosing commit file with the given suffix) *///from w w w .ja va 2s . co m private List<String> scanCommits(final String commitFileSuffix) throws IOException { log.info("Attempting to load the commits under " + metadataFolder + " with suffix " + commitFileSuffix); final List<String> commitFiles = new ArrayList<>(); fs.listStatus(metadataFolder, new PathFilter() { @Override public boolean accept(Path path) { if (path.getName().endsWith(commitFileSuffix)) { commitFiles.add(path.getName().split("\\.")[0]); return true; } return false; } }); return commitFiles; }
From source file:datafu.hourglass.jobs.AbstractPartitionPreservingIncrementalJob.java
License:Apache License
/** * Moves files from the staging path to the final output path. * /*from w w w. ja v a 2 s .co m*/ * @param report report to update with output paths * @param sourcePath source of data to move * @throws IOException */ private void moveStagedFiles(Report report, Path sourcePath) throws IOException { _log.info("Following files produced in staging path:"); for (FileStatus stat : getFileSystem().globStatus(new Path(sourcePath, "*.avro"))) { _log.info(String.format("* %s (%d bytes)", stat.getPath(), stat.getLen())); } FileStatus[] incrementalParts = getFileSystem().globStatus(new Path(sourcePath, "*"), new PathFilter() { @Override public boolean accept(Path path) { String[] pathParts = path.getName().split("-"); try { Long.parseLong(pathParts[0]); return true; } catch (NumberFormatException e) { return false; } } }); // collect the new incremental data from the temp folder and move to subfolders Map<String, Path> incrementalTargetPaths = new HashMap<String, Path>(); for (FileStatus stat : incrementalParts) { String[] pathParts = stat.getPath().getName().split("-"); try { String timestamp = pathParts[0]; if (!incrementalTargetPaths.containsKey(timestamp)) { Path parent = new Path(sourcePath, timestamp); if (!getFileSystem().exists(parent)) { getFileSystem().mkdirs(parent); } else { throw new RuntimeException("already exists: " + parent.toString()); } incrementalTargetPaths.put(timestamp, parent); } Path parent = incrementalTargetPaths.get(timestamp); _log.info(String.format("Moving %s to %s", stat.getPath().getName(), parent.toString())); getFileSystem().rename(stat.getPath(), new Path(parent, stat.getPath().getName())); } catch (NumberFormatException e) { throw new RuntimeException(e); } } for (Path src : incrementalTargetPaths.values()) { Date srcDate; try { srcDate = PathUtils.datedPathFormat.parse(src.getName()); } catch (ParseException e) { throw new RuntimeException(e); } Path target = new Path(getOutputPath(), PathUtils.nestedDatedPathFormat.format(srcDate)); _log.info(String.format("Moving %s to %s", src.getName(), target)); getFileSystem().mkdirs(target.getParent()); if (!getFileSystem().rename(src, target)) { throw new RuntimeException("Failed to rename " + src + " to " + target); } report.outputFiles.add(new DatePath(srcDate, target)); } }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FileStatus[] globStatus(Path pathPattern, PathFilter filter) throws IOException { PathFilter wrappedFilter = new PathFilter() { @Override//from w w w . j a va 2s . c o m public boolean accept(Path path) { return filter.accept(unwrapPath(path)); } }; UnwrappedPath unwrappedPathPattern = unwrapPath(pathPattern); FileStatus[] fileStatuses = this.wrappedFS.globStatus(unwrappedPathPattern, wrappedFilter); if (fileStatuses == null) { return null; } if (unwrappedPathPattern.isUnwrapped()) { for (FileStatus fileStatus : fileStatuses) { fileStatus .setPath(setAuthority(wrapPath(fileStatus.getPath()), pathPattern.toUri().getAuthority())); } } return fileStatuses; }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FileStatus[] listStatus(Path f, PathFilter filter) throws FileNotFoundException, IOException { long startTime = System.nanoTime(); UnwrappedPath unwrappedPath = unwrapPath(f); PathFilter wrappedFilter = new PathFilter() { @Override/*from ww w . j a v a 2s . c om*/ public boolean accept(Path path) { return filter.accept(unwrapPath(path)); } }; FileStatus[] fileStatuses = this.wrappedFS.listStatus(unwrappedPath, wrappedFilter); if (unwrappedPath.isUnwrapped()) { for (FileStatus fileStatus : fileStatuses) { fileStatus.setPath(setAuthority(wrapPath(fileStatus.getPath()), f.toUri().getAuthority())); } } if (!this.skipOther) { int fd = LiveOperationStatisticsAggregator.instance.registerFileDescriptor(f.toString()); LiveOperationStatisticsAggregator.instance.aggregateOperationStatistics(OperationSource.SFS, OperationCategory.OTHER, startTime, System.nanoTime(), fd); } return fileStatuses; }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FileStatus[] listStatus(Path[] path, PathFilter filter) throws FileNotFoundException, IOException { UnwrappedPath[] unwrappedPaths = new UnwrappedPath[path.length]; for (int i = 0; i < path.length; ++i) { unwrappedPaths[i] = unwrapPath(path[i]); }// w w w. ja va2 s . c om PathFilter wrappedFilter = new PathFilter() { @Override public boolean accept(Path p) { return filter.accept(unwrapPath(p)); } }; FileStatus[] fileStatuses = this.wrappedFS.listStatus(unwrappedPaths, wrappedFilter); for (int i = 0; i < fileStatuses.length; ++i) { if (unwrappedPaths[i].isUnwrapped()) { fileStatuses[i] .setPath(setAuthority(wrapPath(fileStatuses[i].getPath()), path[i].toUri().getAuthority())); } } return fileStatuses; }
From source file:dz.lab.hdfs.LsWithPathFilter.java
/** * @param args/*from w w w .ja v a2 s .co m*/ * @throws IOException */ public static void main(String[] args) throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path path = new Path("/"); // restrict result of listStatus() by supplying PathFilter FileStatus[] files = fs.listStatus(path, new PathFilter() { @Override public boolean accept(Path path) { // do not show path whose name equals to user if (path.getName().equals("user")) { return false; } return true; } }); for (FileStatus file : files) { System.out.println(file.getPath().getName()); } }
From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java
License:BSD License
public static void removeReducerFiles(final String fileName) throws IOException { PathFilter filter = new PathFilter() { @Override/*from w ww .j a v a 2 s .c o m*/ public boolean accept(Path arg0) { if (arg0.getName().contains(fileName)) return true; return false; } }; FileSystem fs = FileSystem.get(new Configuration()); Path path = new Path(fs.getHomeDirectory() + "/relationships"); FileStatus[] status = fs.listStatus(path, filter); for (FileStatus fileStatus : status) { fs.delete(new Path(fs.getHomeDirectory() + "/relationships/" + fileStatus.getPath().getName()), true); } }