Example usage for org.apache.hadoop.fs FileStatus isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isDirectory.

Prototype

public boolean isDirectory() 

Source Link

Document

Is this a directory?

Usage

From source file:org.apache.gobblin.util.FileListUtils.java

License:Apache License

/**
 * Given a path to copy, list all files rooted at the given path to copy
 *
 * @param fs the file system of the path
 * @param path root path to copy/*from ww w .j  a  va2s .c  o m*/
 * @param fileFilter a filter only applied to root
 * @param applyFilterToDirectories a control to decide whether to apply filter to directories
 * @param includeEmptyDirectories a control to include empty directories for copy
 */
public static List<FileStatus> listFilesToCopyAtPath(FileSystem fs, Path path, PathFilter fileFilter,
        boolean applyFilterToDirectories, boolean includeEmptyDirectories) throws IOException {
    List<FileStatus> files = Lists.newArrayList();
    FileStatus rootFile = fs.getFileStatus(path);

    listFilesRecursivelyHelper(fs, files, rootFile, fileFilter, applyFilterToDirectories,
            includeEmptyDirectories);

    // Copy the empty root directory
    if (files.size() == 0 && rootFile.isDirectory() && includeEmptyDirectories) {
        files.add(rootFile);
    }
    return files;
}

---

From source file:org.apache.gobblin.util.FileListUtils.java

License:Apache License

private static List<FileStatus> listFilesRecursivelyHelper(FileSystem fs, List<FileStatus> files,
        FileStatus fileStatus, PathFilter fileFilter, boolean applyFilterToDirectories,
        boolean includeEmptyDirectories) throws FileNotFoundException, IOException {
    if (fileStatus.isDirectory()) {
        for (FileStatus status : fs.listStatus(fileStatus.getPath(),
                applyFilterToDirectories ? fileFilter : NO_OP_PATH_FILTER)) {
            if (status.isDirectory()) {
                // Number of files collected before diving into the directory
                int numFilesBefore = files.size();

                listFilesRecursivelyHelper(fs, files, status, fileFilter, applyFilterToDirectories,
                        includeEmptyDirectories);

                // Number of files collected after diving into the directory
                int numFilesAfter = files.size();
                if (numFilesAfter == numFilesBefore && includeEmptyDirectories) {
                    /*/* w  ww  . ja  v  a  2  s  .co  m*/
                     * This is effectively an empty directory, which needs explicit copying. Has there any data file
                     * in the directory, the directory would be created as a side-effect of copying the data file
                     */
                    files.add(status);
                }
            } else {
                listFilesRecursivelyHelper(fs, files, status, fileFilter, applyFilterToDirectories,
                        includeEmptyDirectories);
            }
        }
    } else if (fileFilter.accept(fileStatus.getPath())) {
        files.add(fileStatus);
    }
    return files;
}

---

From source file:org.apache.gobblin.util.FileListUtils.java

License:Apache License

/**
 * Get any data file, which is not hidden or a directory, from the given path
 *//*from w w w  .ja va2 s  .  c om*/
public static FileStatus getAnyNonHiddenFile(FileSystem fs, Path path) throws IOException {
    HiddenFilter hiddenFilter = new HiddenFilter();

    FileStatus root = fs.getFileStatus(path);
    if (!root.isDirectory()) {
        return hiddenFilter.accept(path) ? root : null;
    }

    // DFS to get the first data file
    Stack<FileStatus> folders = new Stack<>();
    folders.push(root);
    while (!folders.empty()) {
        FileStatus curFolder = folders.pop();
        try {
            for (FileStatus status : fs.listStatus(curFolder.getPath(), hiddenFilter)) {
                if (status.isDirectory()) {
                    folders.push(status);
                } else {
                    return status;
                }
            }
        } catch (FileNotFoundException exc) {
            // continue
        }
    }

    return null;
}

---

From source file:org.apache.gobblin.util.PullFileLoader.java

License:Apache License

private List<PathWithTimeStamp> fetchJobFilePathsRecursivelyHelper(Path path) {
    List<PathWithTimeStamp> paths = Lists.newArrayList();
    try {//w w  w.  j  a  va2  s .  co  m
        FileStatus[] statuses = this.fs.listStatus(path);
        if (statuses == null) {
            log.error("Path does not exist: " + path);
            return paths;
        }

        for (FileStatus status : statuses) {
            if (status.isDirectory()) {
                paths.addAll(fetchJobFilePathsRecursivelyHelper(status.getPath()));
            } else if (this.javaPropsPullFileFilter.accept(status.getPath())) {
                log.debug("modification time of {} is {}", status.getPath(), status.getModificationTime());
                paths.add(new PathWithTimeStamp(status.getModificationTime(), status.getPath()));
            } else if (this.hoconPullFileFilter.accept(status.getPath())) {
                log.debug("modification time of {} is {}", status.getPath(), status.getModificationTime());
                paths.add(new PathWithTimeStamp(status.getModificationTime(), status.getPath()));
            }
        }

        return paths;
    } catch (IOException ioe) {
        log.error("Could not load properties at path: " + path, ioe);
        return Lists.newArrayList();
    }
}

---

From source file:org.apache.hadoop.examples.terasort.TeraOutputFormat.java

License:Apache License

@Override
public void checkOutputSpecs(JobContext job) throws InvalidJobConfException, IOException {
    // Ensure that the output directory is set
    Path outDir = getOutputPath(job);
    if (outDir == null) {
        throw new InvalidJobConfException("Output directory not set in JobConf.");
    }//from  w w w  .  ja  va 2s.c o m

    final Configuration jobConf = job.getConfiguration();

    // get delegation token for outDir's file system
    TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, jobConf);

    final FileSystem fs = outDir.getFileSystem(jobConf);

    try {
        // existing output dir is considered empty iff its only content is the
        // partition file.
        //
        final FileStatus[] outDirKids = fs.listStatus(outDir);
        boolean empty = false;
        if (outDirKids != null && outDirKids.length == 1) {
            final FileStatus st = outDirKids[0];
            final String fname = st.getPath().getName();
            empty = !st.isDirectory() && TeraInputFormat.PARTITION_FILENAME.equals(fname);
        }
        if (TeraSort.getUseSimplePartitioner(job) || !empty) {
            throw new FileAlreadyExistsException("Output directory " + outDir + " already exists");
        }
    } catch (FileNotFoundException ignored) {
    }
}

---

From source file:org.apache.hive.common.util.MockFileSystem.java

License:Apache License

public void touch(MockFile file) {
    if (fileStatusMap.containsKey(file)) {
        FileStatus fileStatus = fileStatusMap.get(file);
        FileStatus fileStatusNew = new FileStatus(fileStatus.getLen(), fileStatus.isDirectory(),
                fileStatus.getReplication(), fileStatus.getBlockSize(), fileStatus.getModificationTime() + 1,
                fileStatus.getAccessTime(), fileStatus.getPermission(), fileStatus.getOwner(),
                fileStatus.getGroup(), fileStatus.getPath());
        fileStatusMap.put(file, fileStatusNew);
    }/*ww  w  .  j  a  v a  2  s .c om*/
}

---

From source file:org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem.java

License:Apache License

/** {@inheritDoc} */
@Override//  ww w . java 2  s .  co  m
public Collection<IgfsFile> listFiles(IgfsPath path) {
    try {
        FileStatus[] statuses = fileSys.listStatus(convert(path));

        if (statuses == null)
            throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);

        Collection<IgfsFile> res = new ArrayList<>(statuses.length);

        for (FileStatus status : statuses) {
            IgfsFileInfo fsInfo = status.isDirectory() ? new IgfsFileInfo(true, properties(status))
                    : new IgfsFileInfo((int) status.getBlockSize(), status.getLen(), null, null, false,
                            properties(status));

            res.add(new IgfsFileImpl(new IgfsPath(path, status.getPath().getName()), fsInfo, 1));
        }

        return res;
    } catch (FileNotFoundException ignored) {
        throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
    } catch (IOException e) {
        throw handleSecondaryFsError(e,
                "Failed to list statuses due to secondary file system exception: " + path);
    }
}

---

From source file:org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem.java

License:Apache License

/** {@inheritDoc} */
@Override/*ww w  . j  a  v a  2 s . c  o m*/
public IgfsFile info(final IgfsPath path) {
    try {
        final FileStatus status = fileSys.getFileStatus(convert(path));

        if (status == null)
            return null;

        final Map<String, String> props = properties(status);

        return new IgfsFile() {
            @Override
            public IgfsPath path() {
                return path;
            }

            @Override
            public boolean isFile() {
                return status.isFile();
            }

            @Override
            public boolean isDirectory() {
                return status.isDirectory();
            }

            @Override
            public int blockSize() {
                // By convention directory has blockSize == 0, while file has blockSize > 0:
                return isDirectory() ? 0 : (int) status.getBlockSize();
            }

            @Override
            public long groupBlockSize() {
                return status.getBlockSize();
            }

            @Override
            public long accessTime() {
                return status.getAccessTime();
            }

            @Override
            public long modificationTime() {
                return status.getModificationTime();
            }

            @Override
            public String property(String name) throws IllegalArgumentException {
                String val = props.get(name);

                if (val == null)
                    throw new IllegalArgumentException(
                            "File property not found [path=" + path + ", name=" + name + ']');

                return val;
            }

            @Nullable
            @Override
            public String property(String name, @Nullable String dfltVal) {
                String val = props.get(name);

                return val == null ? dfltVal : val;
            }

            @Override
            public long length() {
                return status.getLen();
            }

            /** {@inheritDoc} */
            @Override
            public Map<String, String> properties() {
                return props;
            }
        };
    } catch (FileNotFoundException ignore) {
        return null;
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
    }
}

---

From source file:org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.java

License:Apache License

/**
 * Convert a file status obtained from the secondary file system to a status of the primary file system.
 *
 * @param status Secondary file system status.
 * @return Primary file system status./*from w w w. ja  v a2 s .c  o  m*/
 */
private FileStatus toPrimary(FileStatus status) {
    return status != null
            ? new FileStatus(status.getLen(), status.isDirectory(), status.getReplication(),
                    status.getBlockSize(), status.getModificationTime(), status.getAccessTime(),
                    status.getPermission(), status.getOwner(), status.getGroup(), toPrimary(status.getPath()))
            : null;
}

---

From source file:org.apache.ignite.igfs.HadoopIgfs20FileSystemAbstractSelfTest.java

License:Apache License

/** @throws Exception If failed. */
public void testCreateBase() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
    Path file = new Path(dir, "someFile");

    assertPathDoesNotExist(fs, file);/*ww w  .j  a va2 s .  co  m*/

    FsPermission fsPerm = new FsPermission((short) 644);

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class), Options.CreateOpts.perms(fsPerm));

    // Try to write something in file.
    os.write("abc".getBytes());

    os.close();

    // Check file status.
    FileStatus fileStatus = fs.getFileStatus(file);

    assertFalse(fileStatus.isDirectory());
    assertEquals(file, fileStatus.getPath());
    assertEquals(fsPerm, fileStatus.getPermission());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• »