List of usage examples for org.apache.hadoop.fs PathFilter PathFilter
PathFilter
From source file:org.apache.blur.mapreduce.lib.BlurInputFormatTest.java
License:Apache License
private void walkOutput(Path output, Configuration conf, ResultReader resultReader) throws IOException { FileSystem fileSystem = output.getFileSystem(conf); FileStatus fileStatus = fileSystem.getFileStatus(output); if (fileStatus.isDir()) { FileStatus[] listStatus = fileSystem.listStatus(output, new PathFilter() { @Override/*from w w w . j av a2 s.co m*/ public boolean accept(Path path) { return !path.getName().startsWith("_"); } }); for (FileStatus fs : listStatus) { walkOutput(fs.getPath(), conf, resultReader); } } else { Reader reader = new SequenceFile.Reader(fileSystem, output, conf); Text rowId = new Text(); TableBlurRecord tableBlurRecord = new TableBlurRecord(); while (reader.next(rowId, tableBlurRecord)) { resultReader.read(rowId, tableBlurRecord); } reader.close(); } }
From source file:org.apache.blur.mapreduce.lib.BlurOutputCommitter.java
License:Apache License
private void commitOrAbortJob(JobContext jobContext, Path shardPath, boolean commit) throws IOException { LOG.info("CommitOrAbort [{0}] path [{1}]", commit, shardPath); FileSystem fileSystem = shardPath.getFileSystem(jobContext.getConfiguration()); FileStatus[] listStatus = fileSystem.listStatus(shardPath, new PathFilter() { @Override//w ww. j a v a2 s .c o m public boolean accept(Path path) { LOG.info("Checking path [{0}]", path); if (path.getName().endsWith(".task_complete")) { return true; } return false; } }); for (FileStatus fileStatus : listStatus) { Path path = fileStatus.getPath(); LOG.info("Trying to commitOrAbort [{0}]", path); String name = path.getName(); boolean taskComplete = name.endsWith(".task_complete"); if (fileStatus.isDir()) { String taskAttemptName = getTaskAttemptName(name); if (taskAttemptName == null) { LOG.info("Dir name [{0}] not task attempt", name); continue; } TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptName); if (taskAttemptID.getJobID().equals(jobContext.getJobID())) { if (commit) { if (taskComplete) { fileSystem.rename(path, new Path(shardPath, taskAttemptName + ".commit")); LOG.info("Committing [{0}] in path [{1}]", taskAttemptID, path); } else { fileSystem.delete(path, true); LOG.info("Deleting tmp dir [{0}] in path [{1}]", taskAttemptID, path); } } else { fileSystem.delete(path, true); LOG.info("Deleting aborted job dir [{0}] in path [{1}]", taskAttemptID, path); } } else { LOG.warn("TaskAttempt JobID [{0}] does not match JobContext JobId [{1}]", taskAttemptID.getJobID(), jobContext.getJobID()); } } } }
From source file:org.apache.blur.store.hdfs.BlurLockFactory.java
License:Apache License
private static FileStatus[] getFileStatusForLock(FileSystem fileSystem, Path dir, final String lockName) throws FileNotFoundException, IOException { FileStatus[] listStatus = fileSystem.listStatus(dir, new PathFilter() { @Override//from ww w . j a v a 2s . c o m public boolean accept(Path path) { return path.getName().startsWith(lockName + "."); } }); return listStatus; }
From source file:org.apache.blur.store.hdfs.HdfsDirectory.java
License:Apache License
@Override public String[] listAll() throws IOException { LOG.debug("listAll [{0}]", getPath()); if (_useCache) { Set<String> names = new HashSet<String>(_fileStatusMap.keySet()); return names.toArray(new String[names.size()]); }//from w w w . j av a 2 s .com Tracer trace = Trace.trace("filesystem - list", Trace.param("path", getPath())); try { FileStatus[] files = _fileSystem.listStatus(getPath(), new PathFilter() { @Override public boolean accept(Path path) { try { return _fileSystem.isFile(path); } catch (IOException e) { throw new RuntimeException(e); } } }); SortedSet<String> result = new TreeSet<String>(); for (int i = 0; i < files.length; i++) { String name = files[i].getPath().getName(); if (name.endsWith(LNK)) { result.add(getRealFileName(name)); } else { result.add(name); } } return result.toArray(new String[result.size()]); } finally { trace.done(); } }
From source file:org.apache.blur.thrift.TableAdmin.java
License:Apache License
private void loadShard(Path newLoadShardPath, FileSystem fileSystem, Path tablePath) throws IOException { Path shardPath = new Path(tablePath, newLoadShardPath.getName()); FileStatus[] listStatus = fileSystem.listStatus(newLoadShardPath, new PathFilter() { @Override/*from w ww. j a v a2 s . co m*/ public boolean accept(Path path) { return path.getName().endsWith(".commit"); } }); for (FileStatus fileStatus : listStatus) { Path src = fileStatus.getPath(); Path dst = new Path(shardPath, src.getName()); if (fileSystem.rename(src, dst)) { LOG.info("Successfully moved [{0}] to [{1}].", src, dst); } else { LOG.info("Could not move [{0}] to [{1}].", src, dst); throw new IOException("Could not move [" + src + "] to [" + dst + "]."); } } }
From source file:org.apache.blur.thrift.TableAdmin.java
License:Apache License
private void validateIndexesExist(Path shardPath, FileSystem fileSystem, Configuration configuration) throws IOException { FileStatus[] listStatus = fileSystem.listStatus(shardPath, new PathFilter() { @Override/*from www . ja v a 2s.c o m*/ public boolean accept(Path path) { return path.getName().endsWith(".commit"); } }); for (FileStatus fileStatus : listStatus) { Path path = fileStatus.getPath(); HdfsDirectory directory = new HdfsDirectory(configuration, path); try { if (!DirectoryReader.indexExists(directory)) { throw new IOException("Path [" + path + "] is not a valid index."); } } finally { directory.close(); } } }
From source file:org.apache.carbondata.core.util.BlockletDataMapUtil.java
License:Apache License
/** * This method will create file name to block Meta Info Mapping. This method will reduce the * number of namenode calls and using this method one namenode will fetch 1000 entries * * @param segmentFilePath/* w w w . ja va 2 s . c om*/ * @return * @throws IOException */ public static Map<String, BlockMetaInfo> createCarbonDataFileBlockMetaInfoMapping(String segmentFilePath, Configuration configuration) throws IOException { Map<String, BlockMetaInfo> fileNameToMetaInfoMapping = new TreeMap(); CarbonFile carbonFile = FileFactory.getCarbonFile(segmentFilePath, configuration); if (carbonFile instanceof AbstractDFSCarbonFile) { PathFilter pathFilter = new PathFilter() { @Override public boolean accept(Path path) { return CarbonTablePath.isCarbonDataFile(path.getName()); } }; CarbonFile[] carbonFiles = carbonFile.locationAwareListFiles(pathFilter); for (CarbonFile file : carbonFiles) { String[] location = file.getLocations(); long len = file.getSize(); BlockMetaInfo blockMetaInfo = new BlockMetaInfo(location, len); fileNameToMetaInfoMapping.put(file.getPath(), blockMetaInfo); } } return fileNameToMetaInfoMapping; }
From source file:org.apache.carbondata.datamap.examples.MinMaxIndexDataMap.java
License:Apache License
@Override public void init(DataMapModel model) throws MemoryException, IOException { Path indexPath = FileFactory.getPath(model.getFilePath()); FileSystem fs = FileFactory.getFileSystem(indexPath); if (!fs.exists(indexPath)) { throw new IOException(String.format("Path %s for MinMax index dataMap does not exist", indexPath)); }/* w ww. j a va 2 s. com*/ if (!fs.isDirectory(indexPath)) { throw new IOException(String.format("Path %s for MinMax index dataMap must be a directory", indexPath)); } FileStatus[] indexFileStatus = fs.listStatus(indexPath, new PathFilter() { @Override public boolean accept(Path path) { return path.getName().endsWith(".minmaxindex"); } }); this.indexFilePath = new String[indexFileStatus.length]; this.readMinMaxDataMap = new MinMaxIndexBlockDetails[indexFileStatus.length][]; for (int i = 0; i < indexFileStatus.length; i++) { this.indexFilePath[i] = indexFileStatus[i].getPath().toString(); this.readMinMaxDataMap[i] = readJson(this.indexFilePath[i]); } }
From source file:org.apache.eagle.jpm.mr.history.crawler.JobHistoryDAOImpl.java
License:Apache License
@SuppressWarnings("deprecation") @Override/*from ww w .j a v a 2 s.co m*/ public List<Pair<Long, String>> readFileNames(int year, int month, int day, int serialNumber) throws Exception { LOG.info("crawl file names under one serial number : " + year + "/" + month + "/" + day + ":" + serialNumber); List<Pair<Long, String>> jobFileNames = new ArrayList<>(); String serialPath = buildWholePathToSerialNumber(year, month, day, serialNumber); try { Path hdfsFile = new Path(serialPath); // filter those files which is job configuration file in xml format FileStatus[] files = hdfs.listStatus(hdfsFile, new PathFilter() { @Override public boolean accept(Path path) { if (path.getName().endsWith(".xml")) { return false; } return true; } }); for (FileStatus fs : files) { if (!fs.isDir()) { jobFileNames.add(Pair.of(fs.getModificationTime(), fs.getPath().getName())); } } if (LOG.isDebugEnabled()) { StringBuilder sb = new StringBuilder(); for (Pair<Long, String> sn : jobFileNames) { sb.append(sn.getRight()); sb.append(","); } LOG.debug("crawled: " + sb); } } catch (Exception ex) { LOG.error("fail reading job history file names under serial number " + serialPath, ex); throw ex; } return jobFileNames; }
From source file:org.apache.falcon.converter.OozieProcessMapper.java
License:Apache License
private void addArchiveForCustomJars(Cluster cluster, Workflow processWorkflow, List<String> archiveList, Path libPath) throws FalconException { if (libPath == null) { return;/* ww w . j a v a2s .c om*/ } try { final FileSystem fs = libPath.getFileSystem(ClusterHelper.getConfiguration(cluster)); if (fs.isFile(libPath)) { // File, not a Dir archiveList.add(libPath.toString()); return; } // lib path is a directory, add each file under the lib dir to archive final FileStatus[] fileStatuses = fs.listStatus(libPath, new PathFilter() { @Override public boolean accept(Path path) { try { return fs.isFile(path) && path.getName().endsWith(".jar"); } catch (IOException ignore) { return false; } } }); for (FileStatus fileStatus : fileStatuses) { archiveList.add(fileStatus.getPath().toString()); } } catch (IOException e) { throw new FalconException("Error adding archive for custom jars under: " + libPath, e); } }