Example usage for org.apache.hadoop.hdfs DFSClient listPaths

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DFSClient listPaths.

Prototype

public DirectoryListing listPaths(String src, byte[] startAfter) throws IOException 

Source Link

Document

Get a partial listing of the indicated directory No block locations need to be fetched

Usage

From source file:com.pinterest.terrapin.controller.ControllerUtilTest.java

License:Apache License

public void testBuildIdealStateForHdfsDirHelper(boolean zkCompression, int numPartitions) throws Exception {
    String hdfsDir = Constants.HDFS_DATA_DIR + "/fileset";
    DFSClient dfsClient = mock(DFSClient.class);

    // Create three hosts in the clusters.
    List<BlockLocation> locations = ImmutableList
            .of(new BlockLocation(new String[] { "host1", "host2" }, new String[] { "host1", "host2" }, 0, 0));
    HdfsFileStatus[] fileStatuses = new HdfsFileStatus[numPartitions];
    for (int i = 0; i < numPartitions; ++i) {
        fileStatuses[i] = PowerMockito.mock(HdfsFileStatus.class);
        String localName = TerrapinUtil.formatPartitionName(i);
        when(fileStatuses[i].getLocalName()).thenReturn(localName);
        when(fileStatuses[i].getFullName(eq(hdfsDir))).thenReturn(hdfsDir + "/" + localName);
        when(fileStatuses[i].getLen()).thenReturn(1000L);
        BlockLocation[] locationArray = new BlockLocation[1];
        locations.subList(0, 1).toArray(locationArray);
        when(dfsClient.getBlockLocations(eq(fileStatuses[i].getFullName(hdfsDir)), anyLong(), anyLong()))
                .thenReturn(locationArray);
    }//from w  w w.  j a va2 s . c  o  m

    when(dfsClient.listPaths(eq(hdfsDir), any(byte[].class))).thenReturn(new DirectoryListing(fileStatuses, 0));

    IdealState is = ControllerUtil.buildIdealStateForHdfsDir(dfsClient, hdfsDir, "resource",
            PartitionerType.CASCADING, 2, zkCompression);

    assertEquals(numPartitions, is.getNumPartitions());
    assertEquals("resource", is.getResourceName());
    for (int i = 0; i < numPartitions; ++i) {
        String partition;
        if (numPartitions > 1000 && !zkCompression) {
            partition = "resource_" + i;
        } else {
            partition = "resource$" + i;
        }
        assertEquals(Sets.newHashSet("host1", "host2"), is.getInstanceSet(partition));
    }
    assertEquals("OnlineOffline", is.getStateModelDefRef());
    if (zkCompression) {
        assertTrue(is.getRecord().getBooleanField("enableCompression", false));
    }
    assertEquals(IdealState.RebalanceMode.CUSTOMIZED, is.getRebalanceMode());
}

---

From source file:com.pinterest.terrapin.TerrapinUtil.java

License:Apache License

/**
 * Retrieve list of files under @hdfsDir for @hdfsClient.
 *//*ww  w . j  a v a  2s. co m*/
public static List<HdfsFileStatus> getHdfsFileList(DFSClient hdfsClient, String hdfsDir) throws IOException {
    List<HdfsFileStatus> fileList = Lists.newArrayList();
    // Build a list of files.
    DirectoryListing listing = null;
    String continuation = "";
    while (true) {
        listing = hdfsClient.listPaths(hdfsDir, continuation.getBytes());
        for (HdfsFileStatus fileStatus : listing.getPartialListing()) {
            fileList.add(fileStatus);
        }
        // Go through the listing and paginate.
        if (!listing.hasMore()) {
            break;
        } else {
            continuation = new String(listing.getLastName());
        }
    }
    return fileList;
}

---