Example usage for org.apache.hadoop.fs FileStatus isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isDirectory.

Prototype

public boolean isDirectory() 

Source Link

Document

Is this a directory?

Usage

From source file:org.apache.ignite.internal.igfs.hadoop.IgfsHadoopFileSystemWrapper.java

License:Apache License

/** {@inheritDoc} */
@Override/*from w  w w  .  ja va2s  .  com*/
public Collection<IgfsFile> listFiles(IgfsPath path) {
    try {
        FileStatus[] statuses = fileSys.listStatus(convert(path));

        if (statuses == null)
            throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);

        Collection<IgfsFile> res = new ArrayList<>(statuses.length);

        for (FileStatus status : statuses) {
            IgfsFileInfo fsInfo = status.isDirectory() ? new IgfsFileInfo(true, properties(status))
                    : new IgfsFileInfo((int) status.getBlockSize(), status.getLen(), null, null, false,
                            properties(status));

            res.add(new IgfsFileImpl(new IgfsPath(path, status.getPath().getName()), fsInfo, 1));
        }

        return res;
    } catch (FileNotFoundException ignored) {
        throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
    } catch (IOException e) {
        throw handleSecondaryFsError(e,
                "Failed to list statuses due to secondary file system exception: " + path);
    }
}

---

From source file:org.apache.ignite.internal.igfs.hadoop.IgfsHadoopFileSystemWrapper.java

License:Apache License

/** {@inheritDoc} */
@Override//from  www.  j  a  v  a  2  s  .c  o  m
public IgfsFile info(final IgfsPath path) {
    try {
        final FileStatus status = fileSys.getFileStatus(convert(path));

        if (status == null)
            return null;

        final Map<String, String> props = properties(status);

        return new IgfsFile() {
            @Override
            public IgfsPath path() {
                return path;
            }

            @Override
            public boolean isFile() {
                return status.isFile();
            }

            @Override
            public boolean isDirectory() {
                return status.isDirectory();
            }

            @Override
            public int blockSize() {
                return (int) status.getBlockSize();
            }

            @Override
            public long groupBlockSize() {
                return status.getBlockSize();
            }

            @Override
            public long accessTime() {
                return status.getAccessTime();
            }

            @Override
            public long modificationTime() {
                return status.getModificationTime();
            }

            @Override
            public String property(String name) throws IllegalArgumentException {
                String val = props.get(name);

                if (val == null)
                    throw new IllegalArgumentException(
                            "File property not found [path=" + path + ", name=" + name + ']');

                return val;
            }

            @Nullable
            @Override
            public String property(String name, @Nullable String dfltVal) {
                String val = props.get(name);

                return val == null ? dfltVal : val;
            }

            @Override
            public long length() {
                return status.getLen();
            }

            /** {@inheritDoc} */
            @Override
            public Map<String, String> properties() {
                return props;
            }
        };

    } catch (FileNotFoundException ignore) {
        return null;
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.delegate.HadoopIgfsSecondaryFileSystemDelegateImpl.java

License:Apache License

/** {@inheritDoc} */
@Override/*from   w w  w  .j a v  a  2  s . c o m*/
public Collection<IgfsFile> listFiles(IgfsPath path) {
    try {
        FileStatus[] statuses = fileSystemForUser().listStatus(convert(path));

        if (statuses == null)
            throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);

        Collection<IgfsFile> res = new ArrayList<>(statuses.length);

        for (FileStatus s : statuses) {
            IgfsEntryInfo fsInfo = s.isDirectory()
                    ? IgfsUtils.createDirectory(IgniteUuid.randomUuid(), null, properties(s), s.getAccessTime(),
                            s.getModificationTime())
                    : IgfsUtils.createFile(IgniteUuid.randomUuid(), (int) s.getBlockSize(), s.getLen(), null,
                            null, false, properties(s), s.getAccessTime(), s.getModificationTime());

            res.add(new IgfsFileImpl(new IgfsPath(path, s.getPath().getName()), fsInfo, 1));
        }

        return res;
    } catch (FileNotFoundException ignored) {
        throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
    } catch (IOException e) {
        throw handleSecondaryFsError(e,
                "Failed to list statuses due to secondary file system exception: " + path);
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.delegate.HadoopIgfsSecondaryFileSystemDelegateImpl.java

License:Apache License

/** {@inheritDoc} */
@Override//from  w ww  .j  a  va2s  . c o  m
public IgfsFile info(final IgfsPath path) {
    try {
        final FileStatus status = fileSystemForUser().getFileStatus(convert(path));

        if (status == null)
            return null;

        final Map<String, String> props = properties(status);

        return new IgfsFileImpl(new IgfsFile() {
            @Override
            public IgfsPath path() {
                return path;
            }

            @Override
            public boolean isFile() {
                return status.isFile();
            }

            @Override
            public boolean isDirectory() {
                return status.isDirectory();
            }

            @Override
            public int blockSize() {
                // By convention directory has blockSize == 0, while file has blockSize > 0:
                return isDirectory() ? 0 : (int) status.getBlockSize();
            }

            @Override
            public long groupBlockSize() {
                return status.getBlockSize();
            }

            @Override
            public long accessTime() {
                return status.getAccessTime();
            }

            @Override
            public long modificationTime() {
                return status.getModificationTime();
            }

            @Override
            public String property(String name) throws IllegalArgumentException {
                String val = props.get(name);

                if (val == null)
                    throw new IllegalArgumentException(
                            "File property not found [path=" + path + ", name=" + name + ']');

                return val;
            }

            @Nullable
            @Override
            public String property(String name, @Nullable String dfltVal) {
                String val = props.get(name);

                return val == null ? dfltVal : val;
            }

            @Override
            public long length() {
                return status.getLen();
            }

            /** {@inheritDoc} */
            @Override
            public Map<String, String> properties() {
                return props;
            }
        }, 0);
    } catch (FileNotFoundException ignore) {
        return null;
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
    }
}

---

From source file:org.apache.impala.catalog.HdfsTable.java

License:Apache License

/**
 * Returns all partitions which match the partition keys directory structure and pass
 * the type compatibility check.//from www .j  a va  2 s  .  c  om
 *
 * path e.g. c1=1/c2=2/c3=3
 * partitionKeys The ordered partition keys. e.g.("c1", "c2", "c3")
 * depth The start position in partitionKeys to match the path name.
 * partitionValues The partition values used to create a partition.
 * partitionExprs The list of LiteralExprs which is used to avoid duplicate partitions.
 * E.g. Having /c1=0001 and /c1=01, we should make sure only one partition
 * will be added.
 * existingPartitions All partitions which exist in Hive Metastore or newly added.
 * partitionsNotInHms Contains all the recovered partitions.
 */
private void getAllPartitionsNotInHms(Path path, List<String> partitionKeys, int depth, FileSystem fs,
        List<String> partitionValues, List<LiteralExpr> partitionExprs,
        HashSet<List<LiteralExpr>> existingPartitions, List<List<String>> partitionsNotInHms)
        throws IOException {
    if (depth == partitionKeys.size()) {
        if (existingPartitions.contains(partitionExprs)) {
            if (LOG.isTraceEnabled()) {
                LOG.trace(
                        String.format("Skip recovery of path '%s' because it already " + "exists in metastore",
                                path.toString()));
            }
        } else {
            partitionsNotInHms.add(partitionValues);
            existingPartitions.add(partitionExprs);
        }
        return;
    }

    FileStatus[] statuses = fs.listStatus(path);
    for (FileStatus status : statuses) {
        if (!status.isDirectory())
            continue;
        Pair<String, LiteralExpr> keyValues = getTypeCompatibleValue(status.getPath(),
                partitionKeys.get(depth));
        if (keyValues == null)
            continue;

        List<String> currentPartitionValues = Lists.newArrayList(partitionValues);
        List<LiteralExpr> currentPartitionExprs = Lists.newArrayList(partitionExprs);
        currentPartitionValues.add(keyValues.first);
        currentPartitionExprs.add(keyValues.second);
        getAllPartitionsNotInHms(status.getPath(), partitionKeys, depth + 1, fs, currentPartitionValues,
                currentPartitionExprs, existingPartitions, partitionsNotInHms);
    }
}

---

From source file:org.apache.impala.common.FileSystemUtil.java

License:Apache License

/**
 * Relocates all visible (non-hidden) files from a source directory to a destination
 * directory. Files are moved (renamed) to the new location unless the source and
 * destination directories are in different encryption zones, in which case the files
 * are copied so that they are decrypted and/or encrypted. Naming conflicts are
 * resolved by appending a UUID to the base file name. Any sub-directories within the
 * source directory are skipped. Returns the number of files relocated as part of this
 * operation./*from   ww  w . j a va2  s  .com*/
 */
public static int relocateAllVisibleFiles(Path sourceDir, Path destDir) throws IOException {
    FileSystem destFs = destDir.getFileSystem(CONF);
    FileSystem sourceFs = sourceDir.getFileSystem(CONF);
    Preconditions.checkState(destFs.isDirectory(destDir));
    Preconditions.checkState(sourceFs.isDirectory(sourceDir));

    // Use the same UUID to resolve all file name conflicts. This helps mitigate problems
    // that might happen if there is a conflict moving a set of files that have
    // dependent file names. For example, foo.lzo and foo.lzo_index.
    UUID uuid = UUID.randomUUID();

    // Enumerate all the files in the source
    int numFilesMoved = 0;
    for (FileStatus fStatus : sourceFs.listStatus(sourceDir)) {
        if (fStatus.isDirectory()) {
            if (LOG.isTraceEnabled()) {
                LOG.trace("Skipping copy of directory: " + fStatus.getPath());
            }
            continue;
        } else if (isHiddenFile(fStatus.getPath().getName())) {
            continue;
        }

        Path destFile = new Path(destDir, fStatus.getPath().getName());
        if (destFs.exists(destFile)) {
            destFile = new Path(destDir, appendToBaseFileName(destFile.getName(), uuid.toString()));
        }
        FileSystemUtil.relocateFile(fStatus.getPath(), destFile, false);
        ++numFilesMoved;
    }
    return numFilesMoved;
}

---

From source file:org.apache.impala.common.FileSystemUtil.java

License:Apache License

/**
 * Returns true if the given Path contains any visible sub directories, otherwise false.
 *//*from   ww  w.  j  a  va  2s . c o m*/
public static boolean containsVisibleSubdirectory(Path directory) throws FileNotFoundException, IOException {
    FileSystem fs = directory.getFileSystem(CONF);
    // Enumerate all the files in the source
    for (FileStatus fStatus : fs.listStatus(directory)) {
        String pathName = fStatus.getPath().getName();
        if (fStatus.isDirectory() && !isHiddenFile(pathName)) {
            return true;
        }
    }
    return false;
}

---

From source file:org.apache.impala.common.FileSystemUtil.java

License:Apache License

/**
 * Returns true if the file corresponding to 'fileStatus' is a valid data file as
 * per Impala's partitioning rules. A fileStatus is considered invalid if its a
 * directory/hidden file/LZO index file. LZO index files are skipped because they are
 * read by the scanner directly. Currently Impala doesn't allow subdirectories in the
 * partition paths./*from  w w w .j a  v a 2  s .c o m*/
 */
public static boolean isValidDataFile(FileStatus fileStatus) {
    String fileName = fileStatus.getPath().getName();
    return !(fileStatus.isDirectory() || FileSystemUtil.isHiddenFile(fileName)
            || HdfsCompression.fromFileName(fileName) == HdfsCompression.LZO_INDEX);
}

---

From source file:org.apache.kylin.dict.AppendTrieDictionaryChecker.java

License:Apache License

public void listDictSlicePath(FileSystem fs, FileStatus path, List<Path> list) throws IOException {
    if (path.isDirectory()) {
        for (FileStatus status : fs.listStatus(path.getPath())) {
            listDictSlicePath(fs, status, list);
        }//from w  w w .  j  a v a2s .c o  m
    } else {
        if (path.getPath().getName().startsWith(CachedTreeMap.CACHED_PREFIX)) {
            list.add(path.getPath());
        }
    }
}

---

From source file:org.apache.kylin.dict.global.AppendTrieDictionaryChecker.java

License:Apache License

public void listDictSlicePath(FileSystem fs, FileStatus path, List<Path> list) throws IOException {
    if (path.isDirectory()) {
        for (FileStatus status : fs.listStatus(path.getPath())) {
            listDictSlicePath(fs, status, list);
        }/*  w  w w  .j  a va 2s.  c o m*/
    } else {
        if (path.getPath().getName().startsWith(GlobalDictHDFSStore.IndexFormatV1.SLICE_PREFIX)) {
            list.add(path.getPath());
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• »