Example usage for org.apache.hadoop.fs FileStatus isFile

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus isFile.

Prototype

public boolean isFile() 

Source Link

Document

Is this a file?

Usage

From source file:org.apache.hoya.tools.CoreFileSystem.java

License:Apache License

/**
 * Verify that a path exists//from  w  w  w  . jav a 2 s .  c  o  m
 * @param path path to check
 * @throws FileNotFoundException file not found or is not a file
 * @throws IOException  trouble with FS
 */
public void verifyFileExists(Path path) throws IOException {
    FileStatus status = fileSystem.getFileStatus(path);

    if (!status.isFile()) {
        throw new FileNotFoundException("Not a file: " + path.toString());
    }
}

---

From source file:org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem.java

License:Apache License

/** {@inheritDoc} */
@Override/*from ww w.j  av  a 2  s  .  c o m*/
public IgfsFile info(final IgfsPath path) {
    try {
        final FileStatus status = fileSys.getFileStatus(convert(path));

        if (status == null)
            return null;

        final Map<String, String> props = properties(status);

        return new IgfsFile() {
            @Override
            public IgfsPath path() {
                return path;
            }

            @Override
            public boolean isFile() {
                return status.isFile();
            }

            @Override
            public boolean isDirectory() {
                return status.isDirectory();
            }

            @Override
            public int blockSize() {
                // By convention directory has blockSize == 0, while file has blockSize > 0:
                return isDirectory() ? 0 : (int) status.getBlockSize();
            }

            @Override
            public long groupBlockSize() {
                return status.getBlockSize();
            }

            @Override
            public long accessTime() {
                return status.getAccessTime();
            }

            @Override
            public long modificationTime() {
                return status.getModificationTime();
            }

            @Override
            public String property(String name) throws IllegalArgumentException {
                String val = props.get(name);

                if (val == null)
                    throw new IllegalArgumentException(
                            "File property not found [path=" + path + ", name=" + name + ']');

                return val;
            }

            @Nullable
            @Override
            public String property(String name, @Nullable String dfltVal) {
                String val = props.get(name);

                return val == null ? dfltVal : val;
            }

            @Override
            public long length() {
                return status.getLen();
            }

            /** {@inheritDoc} */
            @Override
            public Map<String, String> properties() {
                return props;
            }
        };
    } catch (FileNotFoundException ignore) {
        return null;
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
    }
}

---

From source file:org.apache.ignite.internal.igfs.hadoop.IgfsHadoopFileSystemWrapper.java

License:Apache License

/** {@inheritDoc} */
@Override//from w  w  w  .  j  a  va2s.  c om
public IgfsFile info(final IgfsPath path) {
    try {
        final FileStatus status = fileSys.getFileStatus(convert(path));

        if (status == null)
            return null;

        final Map<String, String> props = properties(status);

        return new IgfsFile() {
            @Override
            public IgfsPath path() {
                return path;
            }

            @Override
            public boolean isFile() {
                return status.isFile();
            }

            @Override
            public boolean isDirectory() {
                return status.isDirectory();
            }

            @Override
            public int blockSize() {
                return (int) status.getBlockSize();
            }

            @Override
            public long groupBlockSize() {
                return status.getBlockSize();
            }

            @Override
            public long accessTime() {
                return status.getAccessTime();
            }

            @Override
            public long modificationTime() {
                return status.getModificationTime();
            }

            @Override
            public String property(String name) throws IllegalArgumentException {
                String val = props.get(name);

                if (val == null)
                    throw new IllegalArgumentException(
                            "File property not found [path=" + path + ", name=" + name + ']');

                return val;
            }

            @Nullable
            @Override
            public String property(String name, @Nullable String dfltVal) {
                String val = props.get(name);

                return val == null ? dfltVal : val;
            }

            @Override
            public long length() {
                return status.getLen();
            }

            /** {@inheritDoc} */
            @Override
            public Map<String, String> properties() {
                return props;
            }
        };

    } catch (FileNotFoundException ignore) {
        return null;
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.delegate.HadoopIgfsSecondaryFileSystemDelegateImpl.java

License:Apache License

/** {@inheritDoc} */
@Override/*from   w w  w .j  a  v  a2 s . c  o  m*/
public IgfsFile info(final IgfsPath path) {
    try {
        final FileStatus status = fileSystemForUser().getFileStatus(convert(path));

        if (status == null)
            return null;

        final Map<String, String> props = properties(status);

        return new IgfsFileImpl(new IgfsFile() {
            @Override
            public IgfsPath path() {
                return path;
            }

            @Override
            public boolean isFile() {
                return status.isFile();
            }

            @Override
            public boolean isDirectory() {
                return status.isDirectory();
            }

            @Override
            public int blockSize() {
                // By convention directory has blockSize == 0, while file has blockSize > 0:
                return isDirectory() ? 0 : (int) status.getBlockSize();
            }

            @Override
            public long groupBlockSize() {
                return status.getBlockSize();
            }

            @Override
            public long accessTime() {
                return status.getAccessTime();
            }

            @Override
            public long modificationTime() {
                return status.getModificationTime();
            }

            @Override
            public String property(String name) throws IllegalArgumentException {
                String val = props.get(name);

                if (val == null)
                    throw new IllegalArgumentException(
                            "File property not found [path=" + path + ", name=" + name + ']');

                return val;
            }

            @Nullable
            @Override
            public String property(String name, @Nullable String dfltVal) {
                String val = props.get(name);

                return val == null ? dfltVal : val;
            }

            @Override
            public long length() {
                return status.getLen();
            }

            /** {@inheritDoc} */
            @Override
            public Map<String, String> properties() {
                return props;
            }
        }, 0);
    } catch (FileNotFoundException ignore) {
        return null;
    } catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
    }
}

---

From source file:org.apache.impala.common.FileSystemUtil.java

License:Apache License

/**
 * Performs a non-recursive delete of all visible (non-hidden) files in a given
 * directory. Returns the number of files deleted as part of this operation.
 *///  w  ww  . ja  v  a2 s.  c om
public static int deleteAllVisibleFiles(Path directory) throws IOException {
    FileSystem fs = directory.getFileSystem(CONF);
    Preconditions.checkState(fs.getFileStatus(directory).isDirectory());
    int numFilesDeleted = 0;
    for (FileStatus fStatus : fs.listStatus(directory)) {
        // Only delete files that are not hidden.
        if (fStatus.isFile() && !isHiddenFile(fStatus.getPath().getName())) {
            if (LOG.isTraceEnabled())
                LOG.trace("Removing: " + fStatus.getPath());
            fs.delete(fStatus.getPath(), false);
            ++numFilesDeleted;
        }
    }
    return numFilesDeleted;
}

---

From source file:org.apache.kylin.engine.mr.DFSFileTable.java

License:Apache License

public static Pair<Long, Long> getSizeAndLastModified(String path) throws IOException {
    FileSystem fs = HadoopUtil.getFileSystem(path);

    // get all contained files if path is directory
    ArrayList<FileStatus> allFiles = new ArrayList<>();
    FileStatus status = fs.getFileStatus(new Path(path));
    if (status.isFile()) {
        allFiles.add(status);/*from  ww w  . j  av  a 2s . c  om*/
    } else {
        FileStatus[] listStatus = fs.listStatus(new Path(path));
        allFiles.addAll(Arrays.asList(listStatus));
    }

    long size = 0;
    long lastModified = 0;
    for (FileStatus file : allFiles) {
        size += file.getLen();
        lastModified = Math.max(lastModified, file.getModificationTime());
    }

    return Pair.newPair(size, lastModified);
}

---

From source file:org.apache.kylin.engine.mr.DFSFileTableReader.java

License:Apache License

public DFSFileTableReader(String filePath, String delim, int expectedColumnNumber) throws IOException {
    filePath = HadoopUtil.fixWindowsPath(filePath);
    this.filePath = filePath;
    this.delim = delim;
    this.expectedColumnNumber = expectedColumnNumber;
    this.readerList = new ArrayList<RowReader>();

    FileSystem fs = HadoopUtil.getFileSystem(filePath);

    ArrayList<FileStatus> allFiles = new ArrayList<>();
    FileStatus status = fs.getFileStatus(new Path(filePath));
    if (status.isFile()) {
        allFiles.add(status);//  ww w . ja  v  a  2  s .  co m
    } else {
        FileStatus[] listStatus = fs.listStatus(new Path(filePath));
        allFiles.addAll(Arrays.asList(listStatus));
    }

    try {
        for (FileStatus f : allFiles) {
            RowReader rowReader = new SeqRowReader(HadoopUtil.getCurrentConfiguration(),
                    f.getPath().toString());
            this.readerList.add(rowReader);
        }
    } catch (IOException e) {
        if (isExceptionSayingNotSeqFile(e) == false)
            throw e;

        this.readerList = new ArrayList<RowReader>();
        for (FileStatus f : allFiles) {
            RowReader rowReader = new CsvRowReader(fs, f.getPath().toString());
            this.readerList.add(rowReader);
        }
    }
}

---

From source file:org.apache.kylin.engine.mr.SortedColumnDFSFile.java

License:Apache License

@Override
public TableReader getReader() throws IOException {
    final Comparator<String> comparator = getComparatorByType(dataType);

    ArrayList<TableReader> readers = new ArrayList<>();
    String filePath = HadoopUtil.fixWindowsPath(dfsPath);
    FileSystem fs = HadoopUtil.getFileSystem(filePath);
    ArrayList<FileStatus> allFiles = new ArrayList<>();
    FileStatus status = fs.getFileStatus(new Path(filePath));
    if (status.isFile()) {
        allFiles.add(status);/*from ww w.ja  v  a 2  s . c o  m*/
    } else {
        FileStatus[] listStatus = fs.listStatus(new Path(filePath));
        for (FileStatus f : listStatus) {
            if (f.isFile())
                allFiles.add(f);
        }
    }
    for (FileStatus f : allFiles) {
        DFSFileTableReader reader = new DFSFileTableReader(f.getPath().toString(), -1);
        readers.add(reader);
    }

    return new SortedColumnDFSFileReader(readers, comparator);
}

---

From source file:org.apache.lens.server.query.QueryResultPurger.java

License:Apache License

public void purgePaths(Path path, DateUtil.TimeDiff retention, boolean purgeDirectory) throws IOException {
    int counter = 0;
    FileSystem fs = path.getFileSystem(conf);
    FileStatus[] fileList = fs.listStatus(path);
    for (FileStatus f : fileList) {
        if ((f.isFile() || (f.isDirectory() && purgeDirectory)) && canBePurged(f, retention)) {
            try {
                if (fs.delete(f.getPath(), true)) {
                    counter++;//from w w w . ja  v a2  s  .c o m
                } else {
                    getMetrics().incrCounter(this.getClass(), QUERY_RESULT_PURGER_ERROR_COUNTER);
                }
            } catch (IOException e) {
                getMetrics().incrCounter(this.getClass(), QUERY_RESULT_PURGER_ERROR_COUNTER);
            }
        }
    }
    log.info("Purged {} files/directories in {}", counter, path.toString());
}

---

From source file:org.apache.nifi.processors.hadoop.MoveHDFS.java

License:Apache License

protected Set<Path> selectFiles(final FileSystem hdfs, final Path inputPath, Set<Path> filesVisited)
        throws IOException {
    if (null == filesVisited) {
        filesVisited = new HashSet<>();
    }/* ww w  .j ava  2s.c  om*/

    if (!hdfs.exists(inputPath)) {
        throw new IOException("Selection directory " + inputPath.toString() + " doesn't appear to exist!");
    }

    final Set<Path> files = new HashSet<>();

    FileStatus inputStatus = hdfs.getFileStatus(inputPath);

    if (inputStatus.isDirectory()) {
        for (final FileStatus file : hdfs.listStatus(inputPath)) {
            final Path canonicalFile = file.getPath();

            if (!filesVisited.add(canonicalFile)) { // skip files we've already seen (may be looping directory links)
                continue;
            }

            if (!file.isDirectory() && processorConfig.getPathFilter(inputPath).accept(canonicalFile)) {
                files.add(canonicalFile);

                if (getLogger().isDebugEnabled()) {
                    getLogger().debug(this + " selected file at path: " + canonicalFile.toString());
                }
            }
        }
    } else if (inputStatus.isFile()) {
        files.add(inputPath);
    }
    return files;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• »