Example usage for org.apache.hadoop.hdfs.protocol LocatedBlock LocatedBlock

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs.protocol LocatedBlock LocatedBlock.

Prototype

public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs) 

Source Link

Usage

From source file:com.mellanox.r4h.DFSClient.java

License:Apache License

void reportChecksumFailure(String file, ExtendedBlock blk, DatanodeInfo dn) {
    DatanodeInfo[] dnArr = { dn };//w w  w. j av a  2  s.c  om
    LocatedBlock[] lblocks = { new LocatedBlock(blk, dnArr) };
    reportChecksumFailure(file, lblocks);
}

---

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

/**
 * DFSInputStream reports checksum failure.
 * Case I : client has tried multiple data nodes and at least one of the
 * attempts has succeeded. We report the other failures as corrupted block to
 * namenode./*from  w  w w. j  a  v  a  2  s  . co m*/
 * Case II: client has tried out all data nodes, but all failed. We
 * only report if the total number of replica is 1. We do not
 * report otherwise since this maybe due to the client is a handicapped client
 * (who can not read).
 * 
 * @param corruptedBlockMap
 *            map of corrupted blocks
 * @param dataNodeCount
 *            number of data nodes who contains the block replicas
 */
private void reportCheckSumFailure(Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap, int dataNodeCount) {
    if (corruptedBlockMap.isEmpty()) {
        return;
    }
    Iterator<Entry<ExtendedBlock, Set<DatanodeInfo>>> it = corruptedBlockMap.entrySet().iterator();
    Entry<ExtendedBlock, Set<DatanodeInfo>> entry = it.next();
    ExtendedBlock blk = entry.getKey();
    Set<DatanodeInfo> dnSet = entry.getValue();
    if (((dnSet.size() < dataNodeCount) && (dnSet.size() > 0))
            || ((dataNodeCount == 1) && (dnSet.size() == dataNodeCount))) {
        DatanodeInfo[] locs = new DatanodeInfo[dnSet.size()];
        int i = 0;
        for (DatanodeInfo dn : dnSet) {
            locs[i++] = dn;
        }
        LocatedBlock[] lblocks = { new LocatedBlock(blk, locs) };
        dfsClient.reportChecksumFailure(src, lblocks);
    }
    corruptedBlockMap.clear();
}

---

From source file:com.sun.grid.herd.HerdJsvTest.java

License:Open Source License

/**
 * Test of buildBlockRequests() method, of class HerdJsv.
 *///w w w  . j  a v  a 2s .  c om
public void testBuildBlockRequests() throws Exception {
    System.out.println("buildBlockRequests()");
    HerdJsv instance = new HerdJsv();
    List<LocatedBlock> blocks = new ArrayList<LocatedBlock>(5);
    Map<String, String> expectedResult = new HashMap<String, String>();
    Map result = null;
    long id1 = 0x1234567890abcdefL;
    long id2 = 0x0123456789abcdefL;
    long id3 = 0xfedcba0987654321L;
    long id4 = 0xfedcba9876543210L;
    long id5 = 0x0L;

    blocks.add(new LocatedBlock(new Block(id1), null));
    blocks.add(new LocatedBlock(new Block(id2), null));
    blocks.add(new LocatedBlock(new Block(id3), null));
    blocks.add(new LocatedBlock(new Block(id4), null));
    blocks.add(new LocatedBlock(new Block(id5), null));
    expectedResult.put("hdfs_B12", "*34567890abcdef*");
    expectedResult.put("hdfs_B01", "*23456789abcdef*");
    expectedResult.put("hdfs_Bfe", "*dcba0987654321*dcba9876543210*");
    expectedResult.put("hdfs_B00", "*00000000000000*");

    result = (Map) callPrivateMethod(instance, "buildBlockRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildBlockRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks.clear();
    blocks.add(new LocatedBlock(new Block(id3), null));
    blocks.add(new LocatedBlock(new Block(id4), null));
    expectedResult.clear();
    expectedResult.put("hdfs_Bfe", "*dcba0987654321*dcba9876543210*");

    result = (Map) callPrivateMethod(instance, "buildBlockRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildBlockRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks.clear();
    blocks.add(new LocatedBlock(new Block(id3), null));
    expectedResult.clear();
    expectedResult.put("hdfs_Bfe", "*dcba0987654321*");

    result = (Map) callPrivateMethod(instance, "buildBlockRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildBlockRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, null));
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "buildBlockRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildBlockRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks.clear();
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "buildBlockRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildBlockRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks = null;
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "buildBlockRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildBlockRequests() method did not build the expected requests: ", expectedResult,
            result);
}

---

From source file:com.sun.grid.herd.HerdJsvTest.java

License:Open Source License

/**
 * Test of buildRackRequests() method, of class HerdJsv.
 *///from  w  w  w.  j a  v  a2  s  .c o m
public void testBuildRackRequests() throws Exception {
    System.out.println("buildRackRequests()");
    HerdJsv instance = new HerdJsv();
    List<LocatedBlock> blocks = new ArrayList<LocatedBlock>(5);
    Map<String, String> expectedResult = new HashMap<String, String>();
    Map result = null;
    DatanodeInfo node1 = new DatanodeInfo(new DatanodeID("node1"));
    DatanodeInfo node2 = new DatanodeInfo(new DatanodeID("node2a"));
    DatanodeInfo node3 = new DatanodeInfo(new DatanodeID("node2b"));
    DatanodeInfo node4 = new DatanodeInfo(new DatanodeID("node4"));
    DatanodeInfo node5 = new DatanodeInfo(new DatanodeID("node5"));

    node1.setNetworkLocation("/rack1");
    node2.setNetworkLocation("/rack2a");
    node3.setNetworkLocation("/rack2b");
    node4.setNetworkLocation("/rack4");
    node5.setNetworkLocation("/rack5");

    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1, node2, node5 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1, node2, node3 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node2, node3, node4 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1, node4, node5 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1, node3, node5 }));
    expectedResult.put("hdfs_R", "/rack1");
    expectedResult.put("hdfs_r", "/rack1|/rack2a|/rack2b|/rack5|/rack4");

    result = (Map) callPrivateMethod(instance, "buildRackRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildRackRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1, node2, node5 }));
    expectedResult.put("hdfs_R", "/rack1");
    expectedResult.put("hdfs_r", "/rack1|/rack2a|/rack5");

    result = (Map) callPrivateMethod(instance, "buildRackRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildRackRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node3 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node5 }));
    expectedResult.put("hdfs_R", "/rack1");
    expectedResult.put("hdfs_r", "/rack1|/rack2b|/rack5");

    result = (Map) callPrivateMethod(instance, "buildRackRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildRackRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1 }));
    expectedResult.put("hdfs_R", "/rack1");
    expectedResult.put("hdfs_r", "/rack1");

    result = (Map) callPrivateMethod(instance, "buildRackRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildRackRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] {}));
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "buildRackRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildRackRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, null));
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "buildRackRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildRackRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks.clear();
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "buildRackRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildRackRequests() method did not build the expected requests: ", expectedResult,
            result);

    blocks = null;
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "buildRackRequests", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The buildRackRequests() method did not build the expected requests: ", expectedResult,
            result);
}

---

From source file:com.sun.grid.herd.HerdJsvTest.java

License:Open Source License

/**
 * Test of collateRacks() method, of class HerdJsv.
 *//*  ww w .java2  s.c  o  m*/
public void testCollateRacks() throws Exception {
    System.out.println("collateRacks()");
    HerdJsv instance = new HerdJsv();
    List<LocatedBlock> blocks = new ArrayList<LocatedBlock>(5);
    Map<String, Integer> expectedResult = new HashMap<String, Integer>();
    Map result = null;
    DatanodeInfo node1 = new DatanodeInfo(new DatanodeID("node1"));
    DatanodeInfo node2 = new DatanodeInfo(new DatanodeID("node2"));
    DatanodeInfo node3 = new DatanodeInfo(new DatanodeID("node3"));
    DatanodeInfo node4 = new DatanodeInfo(new DatanodeID("node4"));
    DatanodeInfo node5 = new DatanodeInfo(new DatanodeID("node5"));

    node1.setNetworkLocation("/rack1");
    node2.setNetworkLocation("/rack2");
    node3.setNetworkLocation("/rack3");
    node4.setNetworkLocation("/rack4");
    node5.setNetworkLocation("/rack5");

    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1, node2, node5 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1, node2, node3 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node2, node3, node4 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1, node4, node5 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1, node3, node5 }));
    expectedResult.put("/rack1", 4);
    expectedResult.put("/rack2", 3);
    expectedResult.put("/rack3", 3);
    expectedResult.put("/rack4", 2);
    expectedResult.put("/rack5", 3);

    result = (Map) callPrivateMethod(instance, "collateRacks", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The collateRacks() method did not count the racks correctly", expectedResult, result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1, node3, node5 }));
    expectedResult.clear();
    expectedResult.put("/rack1", 1);
    expectedResult.put("/rack3", 1);
    expectedResult.put("/rack5", 1);

    result = (Map) callPrivateMethod(instance, "collateRacks", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The collateRacks() method did not count the racks correctly", expectedResult, result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node3 }));
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node5 }));
    expectedResult.clear();
    expectedResult.put("/rack1", 1);
    expectedResult.put("/rack3", 1);
    expectedResult.put("/rack5", 1);

    result = (Map) callPrivateMethod(instance, "collateRacks", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The collateRacks() method did not count the racks correctly", expectedResult, result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] { node1 }));
    expectedResult.clear();
    expectedResult.put("/rack1", 1);

    result = (Map) callPrivateMethod(instance, "collateRacks", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The collateRacks() method did not count the racks correctly", expectedResult, result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, new DatanodeInfo[] {}));
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "collateRacks", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The collateRacks() method did not count the racks correctly", expectedResult, result);

    blocks.clear();
    blocks.add(new LocatedBlock(null, null));
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "collateRacks", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The collateRacks() method did not count the racks correctly", expectedResult, result);

    blocks.clear();
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "collateRacks", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The collateRacks() method did not count the racks correctly", expectedResult, result);

    blocks = null;
    expectedResult.clear();

    result = (Map) callPrivateMethod(instance, "collateRacks", new Class[] { Collection.class },
            new Object[] { blocks });
    assertEquals("The collateRacks() method did not count the racks correctly", expectedResult, result);
}

---

From source file:common.DataNode.java

License:Apache License

private void transferBlock(Block block, DatanodeInfo xferTargets[]) throws IOException {
    if (!data.isValidBlock(block)) {
        // block does not exist or is under-construction
        String errStr = "Can't send invalid block " + block;
        LOG.info(errStr);//from  www  . j a v a  2  s .  c o m
        namenode.errorReport(dnRegistration, DatanodeProtocol.INVALID_BLOCK, errStr);
        return;
    }

    // Check if NN recorded length matches on-disk length 
    long onDiskLength = data.getLength(block);
    if (block.getNumBytes() > onDiskLength) {
        // Shorter on-disk len indicates corruption so report NN the corrupt block
        namenode.reportBadBlocks(new LocatedBlock[] {
                new LocatedBlock(block, new DatanodeInfo[] { new DatanodeInfo(dnRegistration) }) });
        LOG.info("Can't replicate block " + block + " because on-disk length " + onDiskLength
                + " is shorter than NameNode recorded length " + block.getNumBytes());
        return;
    }

    int numTargets = xferTargets.length;
    if (numTargets > 0) {
        if (LOG.isInfoEnabled()) {
            StringBuilder xfersBuilder = new StringBuilder();
            for (int i = 0; i < numTargets; i++) {
                xfersBuilder.append(xferTargets[i].getName());
                xfersBuilder.append(" ");
            }
            LOG.info(dnRegistration + " Starting thread to transfer block " + block + " to " + xfersBuilder);
        }

        new Daemon(new DataTransfer(xferTargets, block, this)).start();
    }
}

---