Example usage for org.apache.hadoop.hdfs.protocol DirectoryListing DirectoryListing

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs.protocol DirectoryListing DirectoryListing.

Prototype

public DirectoryListing(HdfsFileStatus[] partialListing, int remainingEntries) 

Source Link

Document

constructor

Usage

From source file:com.pinterest.terrapin.controller.ControllerUtilTest.java

License:Apache License

public void testBuildIdealStateForHdfsDirHelper(boolean zkCompression, int numPartitions) throws Exception {
    String hdfsDir = Constants.HDFS_DATA_DIR + "/fileset";
    DFSClient dfsClient = mock(DFSClient.class);

    // Create three hosts in the clusters.
    List<BlockLocation> locations = ImmutableList
            .of(new BlockLocation(new String[] { "host1", "host2" }, new String[] { "host1", "host2" }, 0, 0));
    HdfsFileStatus[] fileStatuses = new HdfsFileStatus[numPartitions];
    for (int i = 0; i < numPartitions; ++i) {
        fileStatuses[i] = PowerMockito.mock(HdfsFileStatus.class);
        String localName = TerrapinUtil.formatPartitionName(i);
        when(fileStatuses[i].getLocalName()).thenReturn(localName);
        when(fileStatuses[i].getFullName(eq(hdfsDir))).thenReturn(hdfsDir + "/" + localName);
        when(fileStatuses[i].getLen()).thenReturn(1000L);
        BlockLocation[] locationArray = new BlockLocation[1];
        locations.subList(0, 1).toArray(locationArray);
        when(dfsClient.getBlockLocations(eq(fileStatuses[i].getFullName(hdfsDir)), anyLong(), anyLong()))
                .thenReturn(locationArray);
    }// ww w .  j  a  v a  2s.c  o m

    when(dfsClient.listPaths(eq(hdfsDir), any(byte[].class))).thenReturn(new DirectoryListing(fileStatuses, 0));

    IdealState is = ControllerUtil.buildIdealStateForHdfsDir(dfsClient, hdfsDir, "resource",
            PartitionerType.CASCADING, 2, zkCompression);

    assertEquals(numPartitions, is.getNumPartitions());
    assertEquals("resource", is.getResourceName());
    for (int i = 0; i < numPartitions; ++i) {
        String partition;
        if (numPartitions > 1000 && !zkCompression) {
            partition = "resource_" + i;
        } else {
            partition = "resource$" + i;
        }
        assertEquals(Sets.newHashSet("host1", "host2"), is.getInstanceSet(partition));
    }
    assertEquals("OnlineOffline", is.getStateModelDefRef());
    if (zkCompression) {
        assertTrue(is.getRecord().getBooleanField("enableCompression", false));
    }
    assertEquals(IdealState.RebalanceMode.CUSTOMIZED, is.getRebalanceMode());
}

---

From source file:com.pinterest.terrapin.controller.HdfsManagerTest.java

License:Apache License

private void setupBaseDirListing(List<String> fileSets) throws IOException {
    HdfsFileStatus[] fsStatusList = new HdfsFileStatus[fileSets.size() + 1];
    fsStatusList[0] = buildHdfsStatus(Constants.HDFS_DATA_DIR + "/_distcp_XcndjkA", true, null);
    int i = 1;//w  w  w . j a  v a  2s  . co  m
    for (String fileSet : fileSets) {
        fsStatusList[i++] = buildHdfsStatus(Constants.HDFS_DATA_DIR + "/" + fileSet, true, null);
    }
    when(mockDfsClient.listPaths(eq(Constants.HDFS_DATA_DIR), any(byte[].class)))
            .thenReturn(new DirectoryListing(fsStatusList, 0));
}

---

From source file:com.pinterest.terrapin.controller.HdfsManagerTest.java

License:Apache License

private void setupListingForFileSet(String fileSet, List<String> resources, int partitions)
        throws NumberFormatException, IOException {
    HdfsFileStatus[] fsStatusList = new HdfsFileStatus[resources.size() + 1];
    fsStatusList[0] = buildHdfsStatus(Constants.HDFS_DATA_DIR + "/_distcp_XcnddseA", true, null);
    int i = 1;//from w  ww. ja  v  a2  s . com
    for (String resource : resources) {
        fsStatusList[i++] = buildHdfsStatus(TerrapinUtil.helixResourceToHdfsDir(resource), true,
                getTimestampFromResource(resource));
    }
    for (String resource : resources) {
        HdfsFileStatus[] resourceFsStatusList = new HdfsFileStatus[partitions + 1];
        resourceFsStatusList[0] = buildHdfsStatus(TerrapinUtil.helixResourceToHdfsDir(resource) + "/_SUCCESS",
                false, null);
        for (int j = 1; j <= partitions; ++j) {
            resourceFsStatusList[j] = buildHdfsStatus(TerrapinUtil.helixResourceToHdfsDir(resource)
                    + String.format("/%s", TerrapinUtil.formatPartitionName(j - 1)), false, null);
        }
        when(mockDfsClient.listPaths(eq(TerrapinUtil.helixResourceToHdfsDir(resource)), any(byte[].class)))
                .thenReturn(new DirectoryListing(resourceFsStatusList, 0));
    }
    when(mockDfsClient.listPaths(eq(Constants.HDFS_DATA_DIR + "/" + fileSet), any(byte[].class)))
            .thenReturn(new DirectoryListing(fsStatusList, 0));
}

---