Example usage for org.apache.hadoop.fs HdfsBlockLocation getLocatedBlock

Introduction

In this page you can find the example usage for org.apache.hadoop.fs HdfsBlockLocation getLocatedBlock.

Prototype

public LocatedBlock getLocatedBlock() 

Source Link

Usage

From source file:com.mellanox.r4h.DFSClient.java

License:Apache License

/**
 * Get block location information about a list of {@link HdfsBlockLocation}.
 * Used by {@link DistributedFileSystem#getFileBlockStorageLocations(List)} to
 * get {@link BlockStorageLocation}s for blocks returned by
 * {@link DistributedFileSystem#getFileBlockLocations(org.apache.hadoop.fs.FileStatus, long, long)} .
 * /*from  w w  w. ja  v a  2s. c  om*/
 * This is done by making a round of RPCs to the associated datanodes, asking
 * the volume of each block replica. The returned array of {@link BlockStorageLocation} expose this information as a {@link VolumeId}.
 * 
 * @param blockLocations
 *            target blocks on which to query volume location information
 * @return volumeBlockLocations original block array augmented with additional
 *         volume location information for each replica.
 */
public BlockStorageLocation[] getBlockStorageLocations(List<BlockLocation> blockLocations)
        throws IOException, UnsupportedOperationException, InvalidBlockTokenException {
    if (!getConf().isHdfsBlocksMetadataEnabled()) {
        throw new UnsupportedOperationException("Datanode-side support for "
                + "getVolumeBlockLocations() must also be enabled in the client " + "configuration.");
    }
    // Downcast blockLocations and fetch out required LocatedBlock(s)
    List<LocatedBlock> blocks = new ArrayList<LocatedBlock>();
    for (BlockLocation loc : blockLocations) {
        if (!(loc instanceof HdfsBlockLocation)) {
            throw new ClassCastException(
                    "DFSClient#getVolumeBlockLocations " + "expected to be passed HdfsBlockLocations");
        }
        HdfsBlockLocation hdfsLoc = (HdfsBlockLocation) loc;
        blocks.add(hdfsLoc.getLocatedBlock());
    }

    // Re-group the LocatedBlocks to be grouped by datanodes, with the values
    // a list of the LocatedBlocks on the datanode.
    Map<DatanodeInfo, List<LocatedBlock>> datanodeBlocks = new LinkedHashMap<DatanodeInfo, List<LocatedBlock>>();
    for (LocatedBlock b : blocks) {
        for (DatanodeInfo info : b.getLocations()) {
            if (!datanodeBlocks.containsKey(info)) {
                datanodeBlocks.put(info, new ArrayList<LocatedBlock>());
            }
            List<LocatedBlock> l = datanodeBlocks.get(info);
            l.add(b);
        }
    }

    // Make RPCs to the datanodes to get volume locations for its replicas
    TraceScope scope = Trace.startSpan("getBlockStorageLocations", traceSampler);
    Map<DatanodeInfo, HdfsBlocksMetadata> metadatas;
    try {
        metadatas = BlockStorageLocationUtilBridge.queryDatanodesForHdfsBlocksMetadata(conf, datanodeBlocks,
                getConf().getFileBlockStorageLocationsNumThreads(),
                getConf().getFileBlockStorageLocationsTimeoutMs(), getConf().getConnectToDnViaHostname());
        if (LOG.isTraceEnabled()) {
            LOG.trace("metadata returned: " + Joiner.on("\n").withKeyValueSeparator("=").join(metadatas));
        }
    } finally {
        scope.close();
    }

    // Regroup the returned VolumeId metadata to again be grouped by
    // LocatedBlock rather than by datanode
    Map<LocatedBlock, List<VolumeId>> blockVolumeIds = BlockStorageLocationUtilBridge
            .associateVolumeIdsWithBlocks(blocks, metadatas);

    // Combine original BlockLocations with new VolumeId information
    BlockStorageLocation[] volumeBlockLocations = BlockStorageLocationUtilBridge
            .convertToVolumeBlockLocations(blocks, blockVolumeIds);

    return volumeBlockLocations;
}

---