Example usage for org.apache.hadoop.hdfs.protocol LocatedBlocks get

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs.protocol LocatedBlocks get.

Prototype

public LocatedBlock get(int index) 

Source Link

Document

Get located block.

Usage

From source file:com.mellanox.r4h.DFSClient.java

License:Apache License

/**
 * Get block location info about file//  www .jav a 2s . com
 * 
 * getBlockLocations() returns a list of hostnames that store
 * data for a specific file region. It returns a set of hostnames
 * for every block within the indicated region.
 * 
 * This function is very useful when writing code that considers
 * data-placement when performing operations. For example, the
 * MapReduce system tries to schedule tasks on the same machines
 * as the data-block the task processes.
 */
public BlockLocation[] getBlockLocations(String src, long start, long length)
        throws IOException, UnresolvedLinkException {
    TraceScope scope = getPathTraceScope("getBlockLocations", src);
    try {
        LocatedBlocks blocks = getLocatedBlocks(src, start, length);
        BlockLocation[] locations = DFSUtil.locatedBlocks2Locations(blocks);
        HdfsBlockLocation[] hdfsLocations = new HdfsBlockLocation[locations.length];
        for (int i = 0; i < locations.length; i++) {
            hdfsLocations[i] = new HdfsBlockLocation(locations[i], blocks.get(i));
        }
        return hdfsLocations;
    } finally {
        scope.close();
    }
}

---

From source file:com.splunk.shuttl.prototype.symlink.BucketBlockSymlinkPrototypeTest.java

License:Apache License

private void doSymlinkPathInDir(File fileInDir, LocatedBlocks blockLocations, List<LocatedBlock> locatedBlocks)
        throws IOException {
    assertEquals(1, locatedBlocks.size());
    LocatedBlock locatedBlock = blockLocations.get(0);
    assertEquals(1, locatedBlock.getLocations().length);

    DatanodeInfo datanodeInfo = locatedBlock.getLocations()[0];
    ClientDatanodeProtocol createClientDatanodeProtocolProxy = HadoopFileLocationPrototypeTest
            .createClientDatanodeProtocolProxy(datanodeInfo, hadoopFileSystem.getConf(), 1000);

    BlockLocalPathInfo blockLocalPathInfo = createClientDatanodeProtocolProxy
            .getBlockLocalPathInfo(locatedBlock.getBlock(), locatedBlock.getBlockToken());
    String absolutePathToBlock = blockLocalPathInfo.getBlockPath();
    assertTrue(new File(absolutePathToBlock).exists());
    FileUtil.symLink(absolutePathToBlock, fileInDir.getAbsolutePath());
}

---

From source file:com.splunk.shuttl.prototype.symlink.HadoopFileLocationPrototypeTest.java

License:Apache License

/**
 * Before running the test: <br/>/*from w  w  w .  j a v  a  2  s  . co m*/
 * <br/>
 * 1. run `ant hadoop-setup`<br/>
 * 2. run the following command in build-cache/hadoop: bin/hadoop fs -put
 * ../../test/resources/splunk-buckets/SPLUNK_BUCKET/
 * db_1336330530_1336330530_0 / <br/>
 * <br/>
 * Note: This will be automated soon!
 */
@Test(groups = { "prototype" })
public void printPathToABlockOnHadoop() throws IOException {
    // Connect to hdfs. Needs to be HDFS because we're casting to
    // org.apache.hadoop.hdfs.DistributedFileSystem
    URI uri = URI.create("hdfs://localhost:9000");
    fileSystem = (DistributedFileSystem) FileSystem.get(uri, new Configuration());
    namenode = fileSystem.getClient().namenode;

    // Get the path to the bucket that's been put to hadoop.
    Path bucketPath = new Path("/db_1336330530_1336330530_0");
    assertTrue(fileSystem.exists(bucketPath));

    // path to any file in the bucket. Chose .csv because it's
    // readable/verifiable.
    String filePath = "/db_1336330530_1336330530_0/bucket_info.csv";

    // Get location of the blocks for the file.
    LocatedBlocks blockLocations = namenode.getBlockLocations(filePath, 0, Long.MAX_VALUE);
    // There exists only one block because of how everything is set up.
    LocatedBlock locatedBlock = blockLocations.getLocatedBlocks().get(0);
    Block block = locatedBlock.getBlock();
    // There exists only one node.
    DatanodeInfo datanodeInfo = locatedBlock.getLocations()[0];

    // Get a proxy to the Datanode containing the block. (This took a while to
    // figure out)
    ClientDatanodeProtocol createClientDatanodeProtocolProxy = createClientDatanodeProtocolProxy(datanodeInfo,
            fileSystem.getConf(), 1000);

    // Get the local block path. Requires two settings on the server side of
    // hadoop.
    // 1. dfs.client.read.shortcircuit : 'true'
    // 2. dfs.block.local-path-access.user : '<user running the tests (ie.
    // periksson)>'
    BlockLocalPathInfo blockLocalPathInfo = createClientDatanodeProtocolProxy.getBlockLocalPathInfo(block,
            locatedBlock.getBlockToken());
    // Printing the local path to the block, so we can access it!!
    System.out.println("BLOCK PATH: " + blockLocalPathInfo.getBlockPath() + " !!!!!!!!!!!!!!!!!!");
}

---