List of usage examples for org.apache.hadoop.fs FileStatus setPath
public void setPath(final Path p)
From source file:com.ibm.stocator.fs.common.ObjectStoreGlobber.java
License:Open Source License
public FileStatus[] glob() throws IOException { // First we get the scheme and authority of the pattern that was passed // in./*from w ww . java 2s. c om*/ LOG.debug("Welcome to glob : " + pathPattern.toString()); String scheme = schemeFromPath(pathPattern); String authority = authorityFromPath(pathPattern); // Next we strip off everything except the pathname itself, and expand all // globs. Expansion is a process which turns "grouping" clauses, // expressed as brackets, into separate path patterns. String pathPatternString = pathPattern.toUri().getPath(); List<String> flattenedPatterns = ObjectStoreGlobExpander.expand(pathPatternString); LOG.debug("expanded : " + pathPatternString); // Now loop over all flattened patterns. In every case, we'll be trying to // match them to entries in the filesystem. ArrayList<FileStatus> results = new ArrayList<FileStatus>(flattenedPatterns.size()); boolean sawWildcard = false; for (String flatPattern : flattenedPatterns) { LOG.debug("pattern from list: " + flatPattern); Path absPattern = new Path(flatPattern.isEmpty() ? Path.CUR_DIR : flatPattern); List<String> components = getPathComponents(absPattern.toUri().getPath()); ArrayList<FileStatus> candidates = new ArrayList<FileStatus>(1); FileStatus rootPlaceholder = new FileStatus(0, true, 0, 0, 0, new Path(scheme, authority, Path.SEPARATOR)); LOG.debug("Going to add candidate: " + rootPlaceholder.getPath().toString()); candidates.add(rootPlaceholder); String cmpCombined = ""; ObjectStoreGlobFilter globFilter = null; for (int componentIdx = 0; componentIdx < components.size() && !sawWildcard; componentIdx++) { globFilter = new ObjectStoreGlobFilter(components.get(componentIdx)); if (globFilter.hasPattern()) { sawWildcard = true; } else { cmpCombined = cmpCombined + "/" + components.get(componentIdx); } } String component = unescapePathComponent(cmpCombined); if (component != null && component.length() > 0) { for (FileStatus candidate : candidates) { candidate.setPath(new Path(candidate.getPath(), component)); } } else { globFilter = new ObjectStoreGlobFilter(components.get(0)); } ArrayList<FileStatus> newCandidates = new ArrayList<FileStatus>(candidates.size()); for (FileStatus candidate : candidates) { if (globFilter.hasPattern()) { FileStatus[] children = listStatus(candidate.getPath()); if (children.length == 1) { if (!getFileStatus(candidate.getPath()).isDirectory()) { continue; } } for (FileStatus child : children) { if (globFilter.accept(child.getPath())) { newCandidates.add(child); } } } else { FileStatus childStatus = null; childStatus = getFileStatus(new Path(candidate.getPath(), component)); if (childStatus != null) { newCandidates.add(childStatus); } } } candidates = newCandidates; for (FileStatus status : candidates) { if (status == rootPlaceholder) { status = getFileStatus(rootPlaceholder.getPath()); if (status == null) { continue; } } if (filter.accept(status.getPath())) { results.add(status); } } } if (!sawWildcard && results.isEmpty() && (flattenedPatterns.size() <= 1)) { return null; } return results.toArray(new FileStatus[0]); }
From source file:com.uber.hoodie.common.table.view.IncrementalTimelineSyncFileSystemView.java
License:Apache License
private void removeFileSlicesForPartition(HoodieTimeline timeline, HoodieInstant instant, String partition, List<String> paths) { if (isPartitionAvailableInStore(partition)) { log.info("Removing file slices for partition (" + partition + ") for instant (" + instant + ")"); FileStatus[] statuses = paths.stream().map(p -> { FileStatus status = new FileStatus(); status.setPath(new Path(p)); return status; }).toArray(FileStatus[]::new); List<HoodieFileGroup> fileGroups = buildFileGroups(statuses, timeline.filterCompletedAndCompactionInstants(), false); applyDeltaFileSlicesToPartitionView(partition, fileGroups, DeltaApplyMode.REMOVE); } else {/*from w ww. j a va 2 s . c o m*/ log.warn("Skipping partition (" + partition + ") when syncing instant (" + instant + ") as it is not loaded"); } }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { long startTime = System.nanoTime(); Path path = file.getPath();// ww w.j av a 2 s .c o m file.setPath(unwrapPath(path)); BlockLocation[] blockLocations = this.wrappedFS.getFileBlockLocations(file, start, len); file.setPath(path); if (!this.skipOther) { int fd = LiveOperationStatisticsAggregator.instance.registerFileDescriptor(file.getPath().toString()); LiveOperationStatisticsAggregator.instance.aggregateOperationStatistics(OperationSource.SFS, OperationCategory.OTHER, startTime, System.nanoTime(), fd); } return blockLocations; }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FileStatus getFileLinkStatus(Path f) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { UnwrappedPath unwrappedPath = unwrapPath(f); FileStatus fileStatus = this.wrappedFS.getFileLinkStatus(unwrappedPath); if (unwrappedPath.isUnwrapped()) { fileStatus.setPath(setAuthority(wrapPath(fileStatus.getPath()), f.toUri().getAuthority())); }/*from w ww.j ava2 s. c o m*/ return fileStatus; }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FileStatus getFileStatus(Path f) throws IOException { long startTime = System.nanoTime(); UnwrappedPath unwrappedPath = unwrapPath(f); FileStatus fileStatus = this.wrappedFS.getFileStatus(unwrappedPath); if (unwrappedPath.isUnwrapped()) { fileStatus.setPath(setAuthority(wrapPath(fileStatus.getPath()), f.toUri().getAuthority())); }/*from www. j a v a 2s. c om*/ if (!this.skipOther) { int fd = LiveOperationStatisticsAggregator.instance.registerFileDescriptor(f.toString()); LiveOperationStatisticsAggregator.instance.aggregateOperationStatistics(OperationSource.SFS, OperationCategory.OTHER, startTime, System.nanoTime(), fd); } return fileStatus; }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FileStatus[] globStatus(Path pathPattern) throws IOException { UnwrappedPath unwrappedPathPattern = unwrapPath(pathPattern); FileStatus[] fileStatuses = this.wrappedFS.globStatus(unwrappedPathPattern); if (fileStatuses == null) { return null; }/* w w w. j av a 2 s.co m*/ if (unwrappedPathPattern.isUnwrapped()) { for (FileStatus fileStatus : fileStatuses) { fileStatus .setPath(setAuthority(wrapPath(fileStatus.getPath()), pathPattern.toUri().getAuthority())); } } return fileStatuses; }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FileStatus[] globStatus(Path pathPattern, PathFilter filter) throws IOException { PathFilter wrappedFilter = new PathFilter() { @Override// w ww.j a va2 s .c o m public boolean accept(Path path) { return filter.accept(unwrapPath(path)); } }; UnwrappedPath unwrappedPathPattern = unwrapPath(pathPattern); FileStatus[] fileStatuses = this.wrappedFS.globStatus(unwrappedPathPattern, wrappedFilter); if (fileStatuses == null) { return null; } if (unwrappedPathPattern.isUnwrapped()) { for (FileStatus fileStatus : fileStatuses) { fileStatus .setPath(setAuthority(wrapPath(fileStatus.getPath()), pathPattern.toUri().getAuthority())); } } return fileStatuses; }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException { long startTime = System.nanoTime(); UnwrappedPath unwrappedPath = unwrapPath(f); FileStatus[] fileStatuses = this.wrappedFS.listStatus(unwrappedPath); if (unwrappedPath.isUnwrapped()) { for (FileStatus fileStatus : fileStatuses) { fileStatus.setPath(setAuthority(wrapPath(fileStatus.getPath()), f.toUri().getAuthority())); }/* ww w. j a v a2s . c o m*/ } if (!this.skipOther) { int fd = LiveOperationStatisticsAggregator.instance.registerFileDescriptor(f.toString()); LiveOperationStatisticsAggregator.instance.aggregateOperationStatistics(OperationSource.SFS, OperationCategory.OTHER, startTime, System.nanoTime(), fd); } return fileStatuses; }
From source file:de.zib.sfs.StatisticsFileSystem.java
License:BSD License
@Override public FileStatus[] listStatus(Path f, PathFilter filter) throws FileNotFoundException, IOException { long startTime = System.nanoTime(); UnwrappedPath unwrappedPath = unwrapPath(f); PathFilter wrappedFilter = new PathFilter() { @Override/*from w w w . j av a 2 s. c o m*/ public boolean accept(Path path) { return filter.accept(unwrapPath(path)); } }; FileStatus[] fileStatuses = this.wrappedFS.listStatus(unwrappedPath, wrappedFilter); if (unwrappedPath.isUnwrapped()) { for (FileStatus fileStatus : fileStatuses) { fileStatus.setPath(setAuthority(wrapPath(fileStatus.getPath()), f.toUri().getAuthority())); } } if (!this.skipOther) { int fd = LiveOperationStatisticsAggregator.instance.registerFileDescriptor(f.toString()); LiveOperationStatisticsAggregator.instance.aggregateOperationStatistics(OperationSource.SFS, OperationCategory.OTHER, startTime, System.nanoTime(), fd); } return fileStatuses; }