Example usage for org.apache.hadoop.hdfs.protocol DirectoryListing getRemainingEntries

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs.protocol DirectoryListing getRemainingEntries.

Prototype

public int getRemainingEntries() 

Source Link

Document

Get the number of remaining entries that are left to be listed

Usage

From source file:com.mellanox.r4h.DistributedFileSystem.java

License:Apache License

private FileStatus[] listStatusInternal(Path p) throws IOException {
    String src = getPathName(p);/*from w  w w . j a  v a 2s.c  om*/

    // fetch the first batch of entries in the directory
    DirectoryListing thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME);

    if (thisListing == null) { // the directory does not exist
        throw new FileNotFoundException("File " + p + " does not exist.");
    }

    HdfsFileStatus[] partialListing = thisListing.getPartialListing();
    if (!thisListing.hasMore()) { // got all entries of the directory
        FileStatus[] stats = new FileStatus[partialListing.length];
        for (int i = 0; i < partialListing.length; i++) {
            stats[i] = partialListing[i].makeQualified(getUri(), p);
        }
        statistics.incrementReadOps(1);
        return stats;
    }

    // The directory size is too big that it needs to fetch more
    // estimate the total number of entries in the directory
    int totalNumEntries = partialListing.length + thisListing.getRemainingEntries();
    ArrayList<FileStatus> listing = new ArrayList<FileStatus>(totalNumEntries);
    // add the first batch of entries to the array list
    for (HdfsFileStatus fileStatus : partialListing) {
        listing.add(fileStatus.makeQualified(getUri(), p));
    }
    statistics.incrementLargeReadOps(1);

    // now fetch more entries
    do {
        thisListing = dfs.listPaths(src, thisListing.getLastName());

        if (thisListing == null) { // the directory is deleted
            throw new FileNotFoundException("File " + p + " does not exist.");
        }

        partialListing = thisListing.getPartialListing();
        for (HdfsFileStatus fileStatus : partialListing) {
            listing.add(fileStatus.makeQualified(getUri(), p));
        }
        statistics.incrementLargeReadOps(1);
    } while (thisListing.hasMore());

    return listing.toArray(new FileStatus[listing.size()]);
}

---