List of usage examples for org.apache.hadoop.fs FileStatus isFile
public boolean isFile()
From source file:com.cloudera.impala.common.FileSystemUtil.java
License:Apache License
/** * Performs a non-recursive delete of all visible (non-hidden) files in a given * directory. Returns the number of files deleted as part of this operation. *///from w ww.j a v a 2 s . c om public static int deleteAllVisibleFiles(Path directory) throws IOException { FileSystem fs = directory.getFileSystem(CONF); Preconditions.checkState(fs.getFileStatus(directory).isDirectory()); int numFilesDeleted = 0; for (FileStatus fStatus : fs.listStatus(directory)) { // Only delete files that are not hidden. if (fStatus.isFile() && !isHiddenFile(fStatus.getPath().getName())) { LOG.debug("Removing: " + fStatus.getPath()); fs.delete(fStatus.getPath(), false); ++numFilesDeleted; } } return numFilesDeleted; }
From source file:com.cloudera.impala.common.FileSystemUtil.java
License:Apache License
/** * Returns the total number of visible (non-hidden) files in a directory. *//* www.j a va 2s .co m*/ public static int getTotalNumVisibleFiles(Path directory) throws IOException { FileSystem fs = directory.getFileSystem(CONF); Preconditions.checkState(fs.getFileStatus(directory).isDirectory()); int numFiles = 0; for (FileStatus fStatus : fs.listStatus(directory)) { // Only delete files that are not hidden. if (fStatus.isFile() && !isHiddenFile(fStatus.getPath().getName())) { ++numFiles; } } return numFiles; }
From source file:com.cloudera.oryx.common.servcomp.Store.java
License:Open Source License
/** * Gets the total size of all files in all subdirectories of a path. * * @param key path to compute size of/*from ww w . j a v a2s . com*/ * @return total number of bytes at the requested path in bytes */ public long getSizeRecursive(String key) throws IOException { Preconditions.checkNotNull(key); Path path = Namespaces.toPath(key); if (!fs.exists(path)) { return 0L; } long size = 0L; RemoteIterator<? extends FileStatus> it = fs.listFiles(path, true); while (it.hasNext()) { FileStatus status = it.next(); // listFiles() should only show files Preconditions.checkState(status.isFile()); size += status.getLen(); } return size; }
From source file:com.cloudera.oryx.common.servcomp.Store.java
License:Open Source License
/** * @param dirKey location of directory whose contents will be downloaded * @param dir local {@link File} to store files/directories under * @throws IOException if an error occurs while downloading */// w w w . ja v a 2s . c o m public void downloadDirectory(String dirKey, File dir) throws IOException { Preconditions.checkNotNull(dirKey); Preconditions.checkNotNull(dir); Preconditions.checkArgument(dir.exists() && dir.isDirectory(), "Not a directory: %s", dir); Path dirPath = Namespaces.toPath(dirKey); if (!fs.exists(dirPath)) { return; } Preconditions.checkArgument(fs.getFileStatus(dirPath).isDirectory(), "Not a directory: %s", dirPath); boolean complete = false; try { for (FileStatus status : fs.listStatus(dirPath)) { String name = status.getPath().getName(); String fromKey = dirKey + '/' + name; File toLocal = new File(dir, name); if (status.isFile()) { download(fromKey, toLocal); } else { boolean success = toLocal.mkdir(); if (!success && !toLocal.exists()) { throw new IOException("Can't make " + toLocal); } downloadDirectory(fromKey, toLocal); } } complete = true; } finally { if (!complete) { log.warn("Failed to download {} so deleting {}", dirKey, dir); IOUtils.deleteRecursively(dir); } } }
From source file:com.continuent.tungsten.common.file.HdfsFileIO.java
License:Open Source License
/** * {@inheritDoc}/*from w ww . ja v a 2s . c o m*/ * * @see com.continuent.tungsten.common.file.FileIO#isFile(com.continuent.tungsten.common.file.FilePath) */ @Override public boolean isFile(FilePath path) { FileStatus fileStatus = getStatus(path); return fileStatus.isFile(); }
From source file:com.datamoin.tajo.tpcds.TpcDSTestUtil.java
License:Apache License
public static void createTables(String database, TajoClient client) throws Exception { String dataDir = getDataDir(); if (dataDir == null || dataDir.isEmpty()) { throw new IOException("No TPCDS_DATA_DIR property. Use -DTPCDS_DATA_DIR=<data dir>"); }/*from ww w. j a v a 2 s. c om*/ if (dataDir.startsWith("hdfs://")) { Path path = new Path(dataDir); FileSystem fs = path.getFileSystem(new Configuration()); for (String eachTable : tableNames) { Path tableDataDir = new Path(path, eachTable); if (!fs.exists(tableDataDir)) { throw new IOException(eachTable + " data dir [" + tableDataDir + "] not exists."); } } } else { File dataDirFile = new File(dataDir); if (!dataDirFile.exists()) { throw new IOException("TPCDS_DATA_DIR [" + dataDir + "] not exists."); } if (dataDirFile.isFile()) { throw new IOException("TPCDS_DATA_DIR [" + dataDir + "] is not a directory."); } for (String eachTable : tableNames) { File tableDataDir = new File(dataDirFile, eachTable); if (!tableDataDir.exists()) { throw new IOException(eachTable + " data dir [" + tableDataDir + "] not exists."); } } } KeyValueSet opt = new KeyValueSet(); opt.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); LOG.info("Create database: " + database); client.executeQuery("create database if not exists " + database); Path tpcdsResourceURL = new Path(ClassLoader.getSystemResource("tpcds").toString()); Path ddlPath = new Path(tpcdsResourceURL, "ddl"); FileSystem localFs = FileSystem.getLocal(new Configuration()); FileStatus[] files = localFs.listStatus(ddlPath); String dataDirWithPrefix = dataDir; if (dataDir.indexOf("://") < 0) { dataDirWithPrefix = "file://" + dataDir; } for (FileStatus eachFile : files) { if (eachFile.isFile()) { String tableName = eachFile.getPath().getName().split("\\.")[0]; String query = FileUtil.readTextFile(new File(eachFile.getPath().toUri())); query = query.replace("${DB}", database); query = query.replace("${DATA_LOCATION}", dataDirWithPrefix + "/" + tableName); LOG.info("Create table:" + tableName + "," + query); client.executeQuery(query); } } }
From source file:com.facebook.presto.hadoop.HadoopFileStatus.java
License:Apache License
public static boolean isFile(FileStatus status) { return status.isFile(); }
From source file:com.flipkart.fdp.migration.distcp.codec.GenericHadoopCodec.java
License:Apache License
public List<FileTuple> getInputPaths(Collection<String> paths, Collection<String> excludeList) throws Exception { System.out.println("A total of " + paths.size() + " paths to scan..."); List<FileTuple> fileList = new ArrayList<FileTuple>(); List<String> inputPaths = new ArrayList<String>(); // Process regular expression based paths for (String path : paths) { System.out.println("Processing path: " + path); FileStatus[] stats = fs.globStatus(new Path(path)); if (stats == null || stats.length <= 0) continue; for (FileStatus fstat : stats) { if (fstat.isFile()) { fileList.add(new FileTuple(MirrorUtils.getSimplePath(fstat.getPath()), fstat.getLen(), fstat.getModificationTime())); } else { inputPaths.add(MirrorUtils.getSimplePath(fstat.getPath())); }//w w w . ja v a 2 s . c o m } } if (inputPaths.size() > 0) { for (String path : inputPaths) { List<FileTuple> fstat = getFileStatusRecursive(new Path(path), excludeList); fileList.addAll(fstat); } } return fileList; }
From source file:com.flipkart.fdp.migration.distcp.codec.GenericHadoopCodec.java
License:Apache License
public List<FileTuple> getFileStatusRecursive(Path path, Collection<String> excludeList) throws IOException, AuthenticationException { List<FileTuple> response = new ArrayList<FileTuple>(); FileStatus file = fs.getFileStatus(path); //TODO excludeList to be checked if file (not folder) is mentioned in excludeList. if (file != null && file.isFile()) { response.add(new FileTuple(MirrorUtils.getSimplePath(file.getPath()), file.getLen(), file.getModificationTime())); return response; }//from w ww. java2 s .c om FileStatus[] fstats = fs.listStatus(path); if (fstats != null && fstats.length > 0) { for (FileStatus fstat : fstats) { if (fstat.isDirectory() && !excludeList.contains(MirrorUtils.getSimplePath(fstat.getPath()))) { response.addAll(getFileStatusRecursive(fstat.getPath(), excludeList)); } else { //TODO excludeList to be checked if file (not folder) is mentioned in excludeList. response.add(new FileTuple(MirrorUtils.getSimplePath(fstat.getPath()), fstat.getLen(), fstat.getModificationTime())); } } } return response; }
From source file:com.flipkart.fdp.migration.distcp.state.HDFSStateManager.java
License:Apache License
private List<TransferStatus> getAllStats(Path path) throws IOException { Gson gson = new Gson(); List<TransferStatus> status = new ArrayList<TransferStatus>(); FileStatus fstats[] = null;/*from w w w . jav a2 s . c o m*/ if (fs.isDirectory(path)) { fstats = fs.listStatus(path); } else { try { fstats = new FileStatus[1]; fstats[0] = fs.getFileStatus(path); } catch (Exception e) { return status; } } if (fstats == null || fstats.length <= 0) return status; for (FileStatus fstat : fstats) { if (fstat.isFile()) { try { BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(fstat.getPath()))); String line = null; while (null != (line = reader.readLine())) { if (line.trim().length() <= 1) continue; try { TransferStatus tstat = gson.fromJson(line, TransferStatus.class); if (tstat != null) status.add(tstat); } catch (Exception ein) { System.out.println("Exception Reading from location: " + fstat.getPath() + ", Message: " + ein.getMessage()); } } reader.close(); } catch (Exception e) { System.out.println("Exception reading previous state: " + e.getMessage()); } } } return status; }