List of usage examples for org.apache.hadoop.fs FileStatus isFile
public boolean isFile()
From source file:HiveKeyIgnoringBAMOutputFormat.java
License:Open Source License
private void setSAMHeaderFrom(JobConf job) throws IOException { if (wrappedOutputFormat.getSAMHeader() != null) return;/*from w ww . jav a 2 s .c o m*/ // XXX: We're not told where to take the SAM header from so we just merge // them all. There should probably be a better way of doing this. final List<SAMFileHeader> headers = new ArrayList<SAMFileHeader>(); // The "best" sort order among the headers: unsorted if they're sorted // differently, otherwise their common sort order. SAMFileHeader.SortOrder sortOrder = null; // XXX: it seems that FileInputFormat.getInputPaths(job) will point to // the directories of the input tables in the query. I'm not sure if this // is always the case. for (final Path table : FileInputFormat.getInputPaths(job)) { final FileSystem fs = table.getFileSystem(job); for (final FileStatus stat : fs.listStatus(table)) { if (!stat.isFile()) throw new IOException("Unexpected directory '" + stat.getPath() + "', expected only files"); final SAMFileReader r = new SAMFileReader(fs.open(stat.getPath())); final SAMFileHeader h = r.getFileHeader(); r.close(); headers.add(h); if (sortOrder == null) { sortOrder = h.getSortOrder(); continue; } if (sortOrder == SAMFileHeader.SortOrder.unsorted) continue; if (sortOrder != h.getSortOrder()) sortOrder = SAMFileHeader.SortOrder.unsorted; } } wrappedOutputFormat.setSAMHeader(new SamFileHeaderMerger(sortOrder, headers, true).getMergedHeader()); }
From source file:ch.cern.db.hdfs.Main.java
License:GNU General Public License
private void printFileStatus(FileStatus status) { System.out.println();//from w w w. ja v a2s .c o m System.out.println("Showing metadata for: " + status.getPath()); System.out.println(" isDirectory: " + status.isDirectory()); System.out.println(" isFile: " + status.isFile()); System.out.println(" isSymlink: " + status.isSymlink()); System.out.println(" encrypted: " + status.isEncrypted()); System.out.println(" length: " + status.getLen()); System.out.println(" replication: " + status.getReplication()); System.out.println(" blocksize: " + status.getBlockSize()); System.out.println(" modification_time: " + new Date(status.getModificationTime())); System.out.println(" access_time: " + new Date(status.getAccessTime())); System.out.println(" owner: " + status.getOwner()); System.out.println(" group: " + status.getGroup()); System.out.println(" permission: " + status.getPermission()); System.out.println(); }
From source file:cn.ict.magicube.fs.shell.FixedLs.java
License:Apache License
@Override protected void processPath(PathData item) throws IOException { FileStatus stat = item.stat; String line = String.format(lineFormat, (stat.isDirectory() ? "d" : "-"), stat.getPermission(), (stat.isFile() ? stat.getReplication() : "-"), formatSize(stat.getLen()), dateFormat.format(new Date(stat.getModificationTime())), item); out.println(line);//from w ww .ja va 2s . c o m }
From source file:cn.lhfei.hadoop.ch03.ShowFileStatusTest.java
License:Apache License
@Test public void fileStatusForFile() throws IOException { Path path = new Path("/dir/file"); FileStatus stat = fs.getFileStatus(path); Assert.assertEquals(stat.getPath().toUri().getPath(), "/dir/file"); Assert.assertTrue(stat.isFile()); stat.getBlockSize();//from w w w . j a v a 2 s .com }
From source file:co.cask.cdap.common.logging.LogFileReader.java
License:Apache License
/** * Recursive method to tail the log. Reads from the current log file * instance (i), and if that does not have sufficient size, recurses to the * next older instance (i+1). If the caller knows the size of the current * file (i), the caller can pass it via the fileSize parameter. * * @param lines A list of log lines to append read lines to * @param i The current log file instance to start reading from * @param size number of bytes to read at most * @param sizeHint if known, the caller should pass in the length of the * current log file instance. This helps to seek to the end * of a file that has not been closed yet (and hence file * status does not reflect its correct size). Only needed * at instance 0. Otherwise (for recursive calls) this is * -1, and the file size will be obatained via file status. * @return The list of lines read// w w w. ja va 2 s.c om * @throws IOException if reading goes badly wrong */ private List<String> tail(ArrayList<String> lines, int i, long size, long sizeHint) throws IOException { // get the path of the current log file instance (xxx.log[.i]) Path path = new Path(config.getLogFilePath(), makeFileName(i)); // check for its existence, if it does not exist, return empty list if (!fileSystem.exists(path)) { return lines; } FileStatus status = fileSystem.getFileStatus(path); if (!status.isFile()) { return lines; } long fileSize; if (sizeHint >= 0) { fileSize = sizeHint; } else if (i > 0) { fileSize = status.getLen(); } else { fileSize = determineTrueFileSize(path, status); } long seekPos = 0; long bytesToRead = size; if (fileSize >= size) { // if size of currentFile is sufficient, we need to seek to the // position that is size bytes from the end of the file. seekPos = fileSize - size; } else { // if size of current file is less than limit, make a recursive // call to tail for previous file tail(lines, i + 1, size - fileSize, -1); bytesToRead = fileSize; } // open current file for reading byte[] bytes = new byte[(int) bytesToRead]; try (FSDataInputStream input = fileSystem.open(path)) { // seek into latest file if (seekPos > 0) { input.seek(seekPos); } // read to the end of current file input.readFully(bytes); } int pos = 0; if (seekPos > 0) { // if we seeked into the file, then we are likely in the middle of the // line, and we want to skip up to the first new line while (pos < bytesToRead && bytes[pos] != '\n') { pos++; } pos++; // now we are just after the first new line } // read lines until the end of the buffer while (pos < bytesToRead) { int start = pos; while (pos < bytesToRead && bytes[pos] != '\n') { pos++; } // now we are at end of file or at the new line if (pos != start) { // ignore empty lines String line = new String(bytes, start, pos - start, LogFileWriter.CHARSET_UTF8); lines.add(line); } pos++; // skip the new line character } return lines; }
From source file:co.cask.cdap.internal.app.runtime.batch.dataset.partitioned.DynamicPartitioningOutputCommitter.java
License:Apache License
/** * Merge two paths together. Anything in from will be moved into to, if there * are any name conflicts while merging the files or directories in from win. * @param fs the File System to use//w ww . jav a2 s .c om * @param from the path data is coming from. * @param to the path data is going to. * @throws IOException on any error */ private void mergePaths(FileSystem fs, final FileStatus from, final Path to) throws IOException { if (from.isFile()) { if (fs.exists(to)) { if (!fs.delete(to, true)) { throw new IOException("Failed to delete " + to); } } if (!fs.rename(from.getPath(), to)) { throw new IOException("Failed to rename " + from + " to " + to); } } else if (from.isDirectory()) { if (fs.exists(to)) { FileStatus toStat = fs.getFileStatus(to); if (!toStat.isDirectory()) { if (!fs.delete(to, true)) { throw new IOException("Failed to delete " + to); } if (!fs.rename(from.getPath(), to)) { throw new IOException("Failed to rename " + from + " to " + to); } } else { //It is a directory so merge everything in the directories for (FileStatus subFrom : fs.listStatus(from.getPath())) { Path subTo = new Path(to, subFrom.getPath().getName()); mergePaths(fs, subFrom, subTo); } } } else { //it does not exist just rename if (!fs.rename(from.getPath(), to)) { throw new IOException("Failed to rename " + from + " to " + to); } } } }
From source file:co.cask.tigon.logging.LogFileReader.java
License:Apache License
/** * Recursive method to tail the log. Reads from the current log file * instance (i), and if that does not have sufficient size, recurses to the * next older instance (i+1). If the caller knows the size of the current * file (i), he can pass it via the fileSize parameter. * @param lines A list of log lines to append read lines to * @param i The current log file instance to start reading from * @param size number of bytes to read at most * @param sizeHint if known, the caller should pass in the length of the * current log file instance. This helps to seek to the end * of a file that has not been closed yet (and hence file * status does not reflect its correct size). Only needed * at instance 0. Otherwise (for recursive calls) this is * -1, and the file size will be obatained via file status. * @return The list of lines read/* w w w . ja va 2 s.co m*/ * @throws java.io.IOException if reading goes badly wrong */ private List<String> tail(ArrayList<String> lines, int i, long size, long sizeHint) throws IOException { // get the path of the current log file instance (xxx.log[.i]) Path path = new Path(config.getLogFilePath(), makeFileName(i)); // check for its existence, if it does not exist, return empty list if (!fileSystem.exists(path)) { return lines; } FileStatus status = fileSystem.getFileStatus(path); if (!status.isFile()) { return lines; } long fileSize; if (sizeHint >= 0) { fileSize = sizeHint; } else if (i > 0) { fileSize = status.getLen(); } else { fileSize = determineTrueFileSize(path, status); } long seekPos = 0; long bytesToRead = size; if (fileSize >= size) { // if size of currentFile is sufficient, we need to seek to the // position that is size bytes from the end of the file. seekPos = fileSize - size; } else { // if size of current file is less than limit, make a recursive // call to tail for previous file tail(lines, i + 1, size - fileSize, -1); bytesToRead = fileSize; } // open current file for reading byte[] bytes = new byte[(int) bytesToRead]; FSDataInputStream input = fileSystem.open(path); try { // seek into latest file if (seekPos > 0) { input.seek(seekPos); } // read to the end of current file input.readFully(bytes); } finally { input.close(); } int pos = 0; if (seekPos > 0) { // if we seeked into the file, then we are likely in the middle of the // line, and we want to skip up to the first new line while (pos < bytesToRead && bytes[pos] != '\n') { pos++; } pos++; // now we are just after the first new line } // read lines until the end of the buffer while (pos < bytesToRead) { int start = pos; while (pos < bytesToRead && bytes[pos] != '\n') { pos++; } // now we are at end of file or at the new line if (pos != start) { // ignore empty lines String line = new String(bytes, start, pos - start, LogFileWriter.CHARSET_UTF8); lines.add(line); } pos++; // skip the new line character } return lines; }
From source file:com.aliyun.fs.oss.nat.NativeOssFileSystem.java
License:Apache License
/** * <p>/*ww w. j av a2 s. c o m*/ * If <code>f</code> is a file, this method will make a single call to Oss. * If <code>f</code> is a directory, this method will make a maximum of * (<i>n</i> / 1000) + 2 calls to Oss, where <i>n</i> is the total number of * files and directories contained directly in <code>f</code>. * </p> */ @Override public FileStatus[] listStatus(Path f) throws IOException { Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); if (key.length() > 0) { final FileStatus fileStatus = getFileStatus(f); if (fileStatus.isFile()) { return new FileStatus[] { fileStatus }; } } URI pathUri = absolutePath.toUri(); Set<FileStatus> status = new TreeSet<FileStatus>(); String priorLastKey = null; do { PartialListing listing = store.list(key, OSS_MAX_LISTING_LENGTH, priorLastKey, false); for (FileMetadata fileMetadata : listing.getFiles()) { Path subPath = keyToPath(fileMetadata.getKey()); String relativePath = pathUri.relativize(subPath.toUri()).getPath(); if (fileMetadata.getKey().equals(key + "/")) { // this is just the directory we have been asked to list } else if (relativePath.endsWith(FOLDER_SUFFIX)) { status.add(newDirectory( new Path("/" + relativePath.substring(0, relativePath.indexOf(FOLDER_SUFFIX))))); } else { // Here, we need to convert "file/path" to "/file/path". // Otherwise, Path.makeQualified will throw `URISyntaxException`. Path modifiedPath = new Path("/" + subPath.toString()); status.add(newFile(fileMetadata, modifiedPath)); } } for (String commonPrefix : listing.getCommonPrefixes()) { Path subPath = keyToPath(commonPrefix); String relativePath = pathUri.relativize(subPath.toUri()).getPath(); status.add(newDirectory(new Path("/" + relativePath))); } priorLastKey = listing.getPriorLastKey(); } while (priorLastKey != null); if (status.isEmpty()) { return new FileStatus[0]; } return status.toArray(new FileStatus[status.size()]); }
From source file:com.bigstep.datalake.DLFileSystem.java
License:Apache License
@Override public FileStatus[] listStatus(final Path f) throws IOException { statistics.incrementReadOps(1);//from ww w . ja v a 2 s .c o m //this forces a new request for file status for the current //file/directory but it is the only way to tell if the //current path is a file or directory. FileStatus prefixStatus = this.getFileStatus(f); if (prefixStatus.isFile()) return new FileStatus[] { prefixStatus }; final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS; return new FsPathResponseRunner<FileStatus[]>(op, f) { @Override FileStatus[] decodeResponse(Map<?, ?> json) { final Map<?, ?> rootmap = (Map<?, ?>) json.get(FileStatus.class.getSimpleName() + "es"); final List<?> array = JsonUtil.getList(rootmap, FileStatus.class.getSimpleName()); //convert FileStatus final FileStatus[] statuses = new FileStatus[array.size()]; int i = 0; for (Object object : array) { final Map<?, ?> m = (Map<?, ?>) object; statuses[i++] = makeQualified(JsonUtil.toFileStatus(m, false), f); } return statuses; } }.run(); }
From source file:com.cloudera.cdk.data.filesystem.PathIterator.java
License:Apache License
private boolean advance() { while (true) { if (directories.hasNext()) { final Path directory = directories.next(); try { final FileStatus[] stats = fs.listStatus(directory, PathFilters.notHidden()); final List<Path> nextFileSet = Lists.newArrayListWithCapacity(stats.length); for (FileStatus stat : stats) { if (stat.isFile()) { nextFileSet.add(stat.getPath()); }// w w w .j a v a2s . c o m } if (nextFileSet.size() > 0) { this.files = nextFileSet.iterator(); return true; } } catch (IOException ex) { throw new DatasetException("Cannot list files in " + directory, ex); } } else { return false; } } }