Example usage for org.apache.hadoop.net NetworkTopology add

Introduction

In this page you can find the example usage for org.apache.hadoop.net NetworkTopology add.

Prototype

public void add(Node node) 

Source Link

Document

Add a leaf node Update node counter & rack counter if necessary

Usage

From source file:com.bianfeng.bfas.hive.io.RealtimeInputFormat2.java

License:Apache License

/** 
 * This function identifies and returns the hosts that contribute 
 * most for a given split. For calculating the contribution, rack
 * locality is treated on par with host locality, so hosts from racks
 * that contribute the most are preferred over hosts on racks that 
 * contribute less//  www .  j  a va  2  s  . c om
 * @param blkLocations The list of block locations
 * @param offset 
 * @param splitSize 
 * @return array of hosts that contribute most to this split
 * @throws IOException
 */
protected String[] getSplitHosts(BlockLocation[] blkLocations, long offset, long splitSize,
        NetworkTopology clusterMap) throws IOException {

    int startIndex = getBlockIndex(blkLocations, offset);

    long bytesInThisBlock = blkLocations[startIndex].getOffset() + blkLocations[startIndex].getLength()
            - offset;

    //If this is the only block, just return
    if (bytesInThisBlock >= splitSize) {
        return blkLocations[startIndex].getHosts();
    }

    long bytesInFirstBlock = bytesInThisBlock;
    int index = startIndex + 1;
    splitSize -= bytesInThisBlock;

    while (splitSize > 0) {
        bytesInThisBlock = Math.min(splitSize, blkLocations[index++].getLength());
        splitSize -= bytesInThisBlock;
    }

    long bytesInLastBlock = bytesInThisBlock;
    int endIndex = index - 1;

    Map<Node, NodeInfo> hostsMap = new IdentityHashMap<Node, NodeInfo>();
    Map<Node, NodeInfo> racksMap = new IdentityHashMap<Node, NodeInfo>();
    String[] allTopos = new String[0];

    // Build the hierarchy and aggregate the contribution of 
    // bytes at each level. See TestGetSplitHosts.java 

    for (index = startIndex; index <= endIndex; index++) {

        // Establish the bytes in this block
        if (index == startIndex) {
            bytesInThisBlock = bytesInFirstBlock;
        } else if (index == endIndex) {
            bytesInThisBlock = bytesInLastBlock;
        } else {
            bytesInThisBlock = blkLocations[index].getLength();
        }

        allTopos = blkLocations[index].getTopologyPaths();

        // If no topology information is available, just
        // prefix a fakeRack
        if (allTopos.length == 0) {
            allTopos = fakeRacks(blkLocations, index);
        }

        // NOTE: This code currently works only for one level of
        // hierarchy (rack/host). However, it is relatively easy
        // to extend this to support aggregation at different
        // levels 

        for (String topo : allTopos) {

            Node node, parentNode;
            NodeInfo nodeInfo, parentNodeInfo;

            node = clusterMap.getNode(topo);

            if (node == null) {
                node = new NodeBase(topo);
                clusterMap.add(node);
            }

            nodeInfo = hostsMap.get(node);

            if (nodeInfo == null) {
                nodeInfo = new NodeInfo(node);
                hostsMap.put(node, nodeInfo);
                parentNode = node.getParent();
                parentNodeInfo = racksMap.get(parentNode);
                if (parentNodeInfo == null) {
                    parentNodeInfo = new NodeInfo(parentNode);
                    racksMap.put(parentNode, parentNodeInfo);
                }
                parentNodeInfo.addLeaf(nodeInfo);
            } else {
                nodeInfo = hostsMap.get(node);
                parentNode = node.getParent();
                parentNodeInfo = racksMap.get(parentNode);
            }

            nodeInfo.addValue(index, bytesInThisBlock);
            parentNodeInfo.addValue(index, bytesInThisBlock);

        } // for all topos

    } // for all indices

    return identifyHosts(allTopos.length, racksMap);
}

---

From source file:edu.ucsb.cs.hadoop.CustomFileInputFormat.java

License:Apache License

/**
 * This function identifies and returns the hosts that contribute most for a
 * given split. For calculating the contribution, rack locality is treated
 * on par with host locality, so hosts from racks that contribute the most
 * are preferred over hosts on racks that contribute less
 * @param blkLocations The list of block locations
 * @param offset//  w ww .jav  a  2 s.  c o m
 * @param splitSize
 * @return array of hosts that contribute most to this split
 * @throws IOException
 */
protected String[] getSplitHosts(BlockLocation[] blkLocations, long offset, long splitSize,
        NetworkTopology clusterMap) throws IOException {

    int startIndex = getBlockIndex(blkLocations, offset);

    long bytesInThisBlock = blkLocations[startIndex].getOffset() + blkLocations[startIndex].getLength()
            - offset;

    // If this is the only block, just return
    if (bytesInThisBlock >= splitSize) {
        return blkLocations[startIndex].getHosts();
    }

    long bytesInFirstBlock = bytesInThisBlock;
    int index = startIndex + 1;
    splitSize -= bytesInThisBlock;

    while (splitSize > 0) {
        bytesInThisBlock = Math.min(splitSize, blkLocations[index++].getLength());
        splitSize -= bytesInThisBlock;
    }

    long bytesInLastBlock = bytesInThisBlock;
    int endIndex = index - 1;

    Map<Node, NodeInfo> hostsMap = new IdentityHashMap<Node, NodeInfo>();
    Map<Node, NodeInfo> racksMap = new IdentityHashMap<Node, NodeInfo>();
    String[] allTopos = new String[0];

    // Build the hierarchy and aggregate the contribution of
    // bytes at each level. See TestGetSplitHosts.java

    for (index = startIndex; index <= endIndex; index++) {

        // Establish the bytes in this block
        if (index == startIndex) {
            bytesInThisBlock = bytesInFirstBlock;
        } else if (index == endIndex) {
            bytesInThisBlock = bytesInLastBlock;
        } else {
            bytesInThisBlock = blkLocations[index].getLength();
        }

        allTopos = blkLocations[index].getTopologyPaths();

        // If no topology information is available, just
        // prefix a fakeRack
        if (allTopos.length == 0) {
            allTopos = fakeRacks(blkLocations, index);
        }

        // NOTE: This code currently works only for one level of
        // hierarchy (rack/host). However, it is relatively easy
        // to extend this to support aggregation at different
        // levels

        for (String topo : allTopos) {

            Node node, parentNode;
            NodeInfo nodeInfo, parentNodeInfo;

            node = clusterMap.getNode(topo);

            if (node == null) {
                node = new NodeBase(topo);
                clusterMap.add(node);
            }

            nodeInfo = hostsMap.get(node);

            if (nodeInfo == null) {
                nodeInfo = new NodeInfo(node);
                hostsMap.put(node, nodeInfo);
                parentNode = node.getParent();
                parentNodeInfo = racksMap.get(parentNode);
                if (parentNodeInfo == null) {
                    parentNodeInfo = new NodeInfo(parentNode);
                    racksMap.put(parentNode, parentNodeInfo);
                }
                parentNodeInfo.addLeaf(nodeInfo);
            } else {
                nodeInfo = hostsMap.get(node);
                parentNode = node.getParent();
                parentNodeInfo = racksMap.get(parentNode);
            }

            nodeInfo.addValue(index, bytesInThisBlock);
            parentNodeInfo.addValue(index, bytesInThisBlock);

        } // for all topos

    } // for all indices

    return identifyHosts(allTopos.length, racksMap);
}

---

From source file:org.apache.hama.bsp.FileInputFormat.java

License:Apache License

/**
 * This function identifies and returns the hosts that contribute most for a
 * given split. For calculating the contribution, rack locality is treated on
 * par with host locality, so hosts from racks that contribute the most are
 * preferred over hosts on racks that contribute less
 * //from w  ww  .  j  a  v a 2s .c o m
 * @param blkLocations The list of block locations
 * @param offset
 * @param pSplitSize
 * @return array of hosts that contribute most to this split
 * @throws IOException
 */
protected String[] getSplitHosts(BlockLocation[] blkLocations, long offset, long pSplitSize,
        NetworkTopology clusterMap) throws IOException {
    long splitSize = pSplitSize;
    int startIndex = getBlockIndex(blkLocations, offset);

    long bytesInThisBlock = blkLocations[startIndex].getOffset() + blkLocations[startIndex].getLength()
            - offset;

    // If this is the only block, just return
    if (bytesInThisBlock >= splitSize) {
        return blkLocations[startIndex].getHosts();
    }

    long bytesInFirstBlock = bytesInThisBlock;
    int index = startIndex + 1;
    splitSize -= bytesInThisBlock;

    while (splitSize > 0) {
        bytesInThisBlock = Math.min(splitSize, blkLocations[index++].getLength());
        splitSize -= bytesInThisBlock;
    }

    long bytesInLastBlock = bytesInThisBlock;
    int endIndex = index - 1;

    Map<Node, NodeInfo> hostsMap = new IdentityHashMap<Node, NodeInfo>();
    Map<Node, NodeInfo> racksMap = new IdentityHashMap<Node, NodeInfo>();
    String[] allTopos = new String[0];

    // Build the hierarchy and aggregate the contribution of
    // bytes at each level. See TestGetSplitHosts.java

    for (index = startIndex; index <= endIndex; index++) {

        // Establish the bytes in this block
        if (index == startIndex) {
            bytesInThisBlock = bytesInFirstBlock;
        } else if (index == endIndex) {
            bytesInThisBlock = bytesInLastBlock;
        } else {
            bytesInThisBlock = blkLocations[index].getLength();
        }

        allTopos = blkLocations[index].getTopologyPaths();

        // If no topology information is available, just
        // prefix a fakeRack
        if (allTopos.length == 0) {
            allTopos = fakeRacks(blkLocations, index);
        }

        // NOTE: This code currently works only for one level of
        // hierarchy (rack/host). However, it is relatively easy
        // to extend this to support aggregation at different
        // levels

        for (String topo : allTopos) {

            Node node, parentNode;
            NodeInfo nodeInfo, parentNodeInfo;

            node = clusterMap.getNode(topo);

            if (node == null) {
                node = new NodeBase(topo);
                clusterMap.add(node);
            }

            nodeInfo = hostsMap.get(node);

            if (nodeInfo == null) {
                nodeInfo = new NodeInfo(node);
                hostsMap.put(node, nodeInfo);
                parentNode = node.getParent();
                parentNodeInfo = racksMap.get(parentNode);
                if (parentNodeInfo == null) {
                    parentNodeInfo = new NodeInfo(parentNode);
                    racksMap.put(parentNode, parentNodeInfo);
                }
                parentNodeInfo.addLeaf(nodeInfo);
            } else {
                nodeInfo = hostsMap.get(node);
                parentNode = node.getParent();
                parentNodeInfo = racksMap.get(parentNode);
            }

            nodeInfo.addValue(index, bytesInThisBlock);
            parentNodeInfo.addValue(index, bytesInThisBlock);

        } // for all topos

    } // for all indices

    return identifyHosts(allTopos.length, racksMap);
}

---