Example usage for org.apache.hadoop.hdfs.protocol LocatedBlock getBlock

List of usage examples for org.apache.hadoop.hdfs.protocol LocatedBlock getBlock

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs.protocol LocatedBlock getBlock.

Prototype

public ExtendedBlock getBlock() 

Source Link

Usage

From source file:backup.namenode.NameNodeBackupBlockCheckProcessor.java

License:Apache License

private void addExtendedBlocksFromNameNode(BackupReportWriter writer,
        ExternalExtendedBlockSort<Addresses> nameNodeBlocks, DFSClient client, FileStatus fs,
        Set<Path> pathSetToIgnore) throws IOException {
    Path qualifiedPath = fileSystem.makeQualified(fs.getPath());
    if (shouldIgnore(pathSetToIgnore, qualifiedPath)) {
        return;/*from   w w w  .jav a  2  s  .com*/
    }

    String src = qualifiedPath.toUri().getPath();
    long start = 0;
    long length = fs.getLen();

    LocatedBlocks locatedBlocks = client.getLocatedBlocks(src, start, length);
    for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
        DatanodeInfo[] locations = locatedBlock.getLocations();
        ExtendedBlock extendedBlock = BackupUtil.fromHadoop(locatedBlock.getBlock());
        Addresses addresses = new Addresses(locations);
        nameNodeBlocks.add(extendedBlock, addresses);
        writer.statusExtendedBlocksFromNameNode(src, extendedBlock, locations);
    }
}

From source file:com.bigstep.datalake.JsonUtil.java

License:Apache License

/** Convert a LocatedBlock to a Json map. */
private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock) throws IOException {
    if (locatedblock == null) {
        return null;
    }//from   w ww . j  a v  a  2s.  c  om

    final Map<String, Object> m = new TreeMap<String, Object>();
    m.put("blockToken", toJsonMap(locatedblock.getBlockToken()));
    m.put("isCorrupt", locatedblock.isCorrupt());
    m.put("startOffset", locatedblock.getStartOffset());
    m.put("block", toJsonMap(locatedblock.getBlock()));
    m.put("locations", toJsonArray(locatedblock.getLocations()));
    m.put("cachedLocations", toJsonArray(locatedblock.getCachedLocations()));
    return m;
}

From source file:com.mellanox.r4h.DFSClient.java

License:Apache License

/**
 * Get the checksum of the whole file of a range of the file. Note that the
 * range always starts from the beginning of the file.
 * // ww  w  .j a v  a  2 s.c  o m
 * @param src
 *            The file path
 * @param length
 *            the length of the range, i.e., the range is [0, length]
 * @return The checksum
 * @see DistributedFileSystem#getFileChecksum(Path)
 */
public MD5MD5CRC32FileChecksum getFileChecksum(String src, long length) throws IOException {
    checkOpen();
    Preconditions.checkArgument(length >= 0);
    // get block locations for the file range
    LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, length);
    if (null == blockLocations) {
        throw new FileNotFoundException("File does not exist: " + src);
    }
    List<LocatedBlock> locatedblocks = blockLocations.getLocatedBlocks();
    final DataOutputBuffer md5out = new DataOutputBuffer();
    int bytesPerCRC = -1;
    DataChecksum.Type crcType = DataChecksum.Type.DEFAULT;
    long crcPerBlock = 0;
    boolean refetchBlocks = false;
    int lastRetriedIndex = -1;

    // get block checksum for each block
    long remaining = length;
    if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
        remaining = Math.min(length, blockLocations.getFileLength());
    }
    for (int i = 0; i < locatedblocks.size() && remaining > 0; i++) {
        if (refetchBlocks) { // refetch to get fresh tokens
            blockLocations = callGetBlockLocations(namenode, src, 0, length);
            if (null == blockLocations) {
                throw new FileNotFoundException("File does not exist: " + src);
            }
            locatedblocks = blockLocations.getLocatedBlocks();
            refetchBlocks = false;
        }
        LocatedBlock lb = locatedblocks.get(i);
        final ExtendedBlock block = lb.getBlock();
        if (remaining < block.getNumBytes()) {
            block.setNumBytes(remaining);
        }
        remaining -= block.getNumBytes();
        final DatanodeInfo[] datanodes = lb.getLocations();

        // try each datanode location of the block
        final int timeout = 3000 * datanodes.length + dfsClientConf.socketTimeout();
        boolean done = false;
        for (int j = 0; !done && j < datanodes.length; j++) {
            DataOutputStream out = null;
            DataInputStream in = null;

            try {
                // connect to a datanode
                IOStreamPair pair = connectToDN(datanodes[j], timeout, lb);
                out = new DataOutputStream(new BufferedOutputStream(pair.out, HdfsConstants.SMALL_BUFFER_SIZE));
                in = new DataInputStream(pair.in);

                if (LOG.isDebugEnabled()) {
                    LOG.debug("write to " + datanodes[j] + ": " + Op.BLOCK_CHECKSUM + ", block=" + block);
                }
                // get block MD5
                new Sender(out).blockChecksum(block, lb.getBlockToken());

                final BlockOpResponseProto reply = BlockOpResponseProto.parseFrom(PBHelper.vintPrefixed(in));

                String logInfo = "for block " + block + " from datanode " + datanodes[j];
                DataTransferProtoUtil.checkBlockOpStatus(reply, logInfo);

                OpBlockChecksumResponseProto checksumData = reply.getChecksumResponse();

                // read byte-per-checksum
                final int bpc = checksumData.getBytesPerCrc();
                if (i == 0) { // first block
                    bytesPerCRC = bpc;
                } else if (bpc != bytesPerCRC) {
                    throw new IOException(
                            "Byte-per-checksum not matched: bpc=" + bpc + " but bytesPerCRC=" + bytesPerCRC);
                }

                // read crc-per-block
                final long cpb = checksumData.getCrcPerBlock();
                if (locatedblocks.size() > 1 && i == 0) {
                    crcPerBlock = cpb;
                }

                // read md5
                final MD5Hash md5 = new MD5Hash(checksumData.getMd5().toByteArray());
                md5.write(md5out);

                // read crc-type
                final DataChecksum.Type ct;
                if (checksumData.hasCrcType()) {
                    ct = PBHelper.convert(checksumData.getCrcType());
                } else {
                    LOG.debug("Retrieving checksum from an earlier-version DataNode: "
                            + "inferring checksum by reading first byte");
                    ct = inferChecksumTypeByReading(lb, datanodes[j]);
                }

                if (i == 0) { // first block
                    crcType = ct;
                } else if (crcType != DataChecksum.Type.MIXED && crcType != ct) {
                    // if crc types are mixed in a file
                    crcType = DataChecksum.Type.MIXED;
                }

                done = true;

                if (LOG.isDebugEnabled()) {
                    if (i == 0) {
                        LOG.debug("set bytesPerCRC=" + bytesPerCRC + ", crcPerBlock=" + crcPerBlock);
                    }
                    LOG.debug("got reply from " + datanodes[j] + ": md5=" + md5);
                }
            } catch (InvalidBlockTokenException ibte) {
                if (i > lastRetriedIndex) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Got access token error in response to OP_BLOCK_CHECKSUM " + "for file " + src
                                + " for block " + block + " from datanode " + datanodes[j]
                                + ". Will retry the block once.");
                    }
                    lastRetriedIndex = i;
                    done = true; // actually it's not done; but we'll retry
                    i--; // repeat at i-th block
                    refetchBlocks = true;
                    break;
                }
            } catch (IOException ie) {
                LOG.warn("src=" + src + ", datanodes[" + j + "]=" + datanodes[j], ie);
            } finally {
                IOUtils.closeStream(in);
                IOUtils.closeStream(out);
            }
        }

        if (!done) {
            throw new IOException("Fail to get block MD5 for " + block);
        }
    }

    // compute file MD5
    final MD5Hash fileMD5 = MD5Hash.digest(md5out.getData());
    switch (crcType) {
    case CRC32:
        return new MD5MD5CRC32GzipFileChecksum(bytesPerCRC, crcPerBlock, fileMD5);
    case CRC32C:
        return new MD5MD5CRC32CastagnoliFileChecksum(bytesPerCRC, crcPerBlock, fileMD5);
    default:
        // If there is no block allocated for the file,
        // return one with the magic entry that matches what previous
        // hdfs versions return.
        if (locatedblocks.size() == 0) {
            return new MD5MD5CRC32GzipFileChecksum(0, 0, fileMD5);
        }

        // we should never get here since the validity was checked
        // when getCrcType() was called above.
        return null;
    }
}

From source file:com.mellanox.r4h.DFSClient.java

License:Apache License

/**
 * Infer the checksum type for a replica by sending an OP_READ_BLOCK
 * for the first byte of that replica. This is used for compatibility
 * with older HDFS versions which did not include the checksum type in
 * OpBlockChecksumResponseProto.//from   w  w w  .  j  a va 2  s. c  o  m
 * 
 * @param lb
 *            the located block
 * @param dn
 *            the connected datanode
 * @return the inferred checksum type
 * @throws IOException
 *             if an error occurs
 */
private Type inferChecksumTypeByReading(LocatedBlock lb, DatanodeInfo dn) throws IOException {
    IOStreamPair pair = connectToDN(dn, dfsClientConf.socketTimeout(), lb);

    try {
        DataOutputStream out = new DataOutputStream(
                new BufferedOutputStream(pair.out, HdfsConstants.SMALL_BUFFER_SIZE));
        DataInputStream in = new DataInputStream(pair.in);

        new Sender(out).readBlock(lb.getBlock(), lb.getBlockToken(), clientName, 0, 1, true,
                CachingStrategy.newDefaultStrategy());
        final BlockOpResponseProto reply = BlockOpResponseProto.parseFrom(PBHelper.vintPrefixed(in));
        String logInfo = "trying to read " + lb.getBlock() + " from datanode " + dn;
        DataTransferProtoUtil.checkBlockOpStatus(reply, logInfo);

        return PBHelper.convert(reply.getReadOpChecksumInfo().getChecksum().getType());
    } finally {
        IOUtils.cleanup(null, pair.in, pair.out);
    }
}

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

private long fetchLocatedBlocksAndGetLastBlockLength() throws IOException {
    final LocatedBlocks newInfo = dfsClient.getLocatedBlocks(src, 0);
    if (DFSClient.LOG.isDebugEnabled()) {
        DFSClient.LOG.debug("newInfo = " + newInfo);
    }//from  www  . j  av  a 2  s  .  c om
    if (newInfo == null) {
        throw new IOException("Cannot open filename " + src);
    }

    if (locatedBlocks != null) {
        Iterator<LocatedBlock> oldIter = locatedBlocks.getLocatedBlocks().iterator();
        Iterator<LocatedBlock> newIter = newInfo.getLocatedBlocks().iterator();
        while (oldIter.hasNext() && newIter.hasNext()) {
            if (!oldIter.next().getBlock().equals(newIter.next().getBlock())) {
                throw new IOException("Blocklist for " + src + " has changed!");
            }
        }
    }
    locatedBlocks = newInfo;
    long lastBlockBeingWrittenLength = 0;
    if (!locatedBlocks.isLastBlockComplete()) {
        final LocatedBlock last = locatedBlocks.getLastLocatedBlock();
        if (last != null) {
            if (last.getLocations().length == 0) {
                if (last.getBlockSize() == 0) {
                    // if the length is zero, then no data has been written to
                    // datanode. So no need to wait for the locations.
                    return 0;
                }
                return -1;
            }
            final long len = readBlockLength(last);
            last.getBlock().setNumBytes(len);
            lastBlockBeingWrittenLength = len;
        }
    }

    fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo();

    return lastBlockBeingWrittenLength;
}

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

/** Read the block length from one of the datanodes. */
private long readBlockLength(LocatedBlock locatedblock) throws IOException {
    assert locatedblock != null : "LocatedBlock cannot be null";
    int replicaNotFoundCount = locatedblock.getLocations().length;

    for (DatanodeInfo datanode : locatedblock.getLocations()) {
        ClientDatanodeProtocol cdp = null;

        try {/*ww w  .  ja v a2  s. c  o  m*/
            cdp = DFSUtil.createClientDatanodeProtocolProxy(datanode, dfsClient.getConfiguration(),
                    dfsClient.getConf().getSocketTimeout(), dfsClient.getConf().getConnectToDnViaHostname(),
                    locatedblock);

            final long n = cdp.getReplicaVisibleLength(locatedblock.getBlock());

            if (n >= 0) {
                return n;
            }
        } catch (IOException ioe) {
            if (ioe instanceof RemoteException
                    && (((RemoteException) ioe).unwrapRemoteException() instanceof ReplicaNotFoundException)) {
                // special case : replica might not be on the DN, treat as 0 length
                replicaNotFoundCount--;
            }

            if (DFSClient.LOG.isDebugEnabled()) {
                DFSClient.LOG.debug("Failed to getReplicaVisibleLength from datanode " + datanode
                        + " for block " + locatedblock.getBlock(), ioe);
            }
        } finally {
            if (cdp != null) {
                RPC.stopProxy(cdp);
            }
        }
    }

    // Namenode told us about these locations, but none know about the replica
    // means that we hit the race between pipeline creation start and end.
    // we require all 3 because some other exception could have happened
    // on a DN that has it. we want to report that error
    if (replicaNotFoundCount == 0) {
        return 0;
    }

    throw new IOException("Cannot obtain block length for " + locatedblock);
}

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

/**
 * Open a DataInputStream to a DataNode so that it can be read from.
 * We get block ID and the IDs of the destinations at startup, from the namenode.
 *//*from www .j  av  a  2  s.c om*/
private synchronized DatanodeInfo blockSeekTo(long target) throws IOException {
    if (target >= getFileLength()) {
        throw new IOException("Attempted to read past end of file");
    }

    // Will be getting a new BlockReader.
    closeCurrentBlockReader();

    //
    // Connect to best DataNode for desired Block, with potential offset
    //
    DatanodeInfo chosenNode = null;
    int refetchToken = 1; // only need to get a new access token once
    int refetchEncryptionKey = 1; // only need to get a new encryption key once

    boolean connectFailedOnce = false;

    while (true) {
        //
        // Compute desired block
        //
        LocatedBlock targetBlock = getBlockAt(target);

        // update current position
        this.pos = target;
        this.blockEnd = targetBlock.getStartOffset() + targetBlock.getBlockSize() - 1;
        this.currentLocatedBlock = targetBlock;

        assert (target == pos) : "Wrong postion " + pos + " expect " + target;
        long offsetIntoBlock = target - targetBlock.getStartOffset();

        DNAddrPair retval = chooseDataNode(targetBlock, null);
        chosenNode = retval.info;
        InetSocketAddress targetAddr = retval.addr;
        StorageType storageType = retval.storageType;

        try {
            ExtendedBlock blk = targetBlock.getBlock();
            Token<BlockTokenIdentifier> accessToken = targetBlock.getBlockToken();
            CachingStrategy curCachingStrategy;
            boolean shortCircuitForbidden;
            synchronized (infoLock) {
                curCachingStrategy = cachingStrategy;
                shortCircuitForbidden = shortCircuitForbidden();
            }
            blockReader = new BlockReaderFactory(dfsClient.getConf()).setInetSocketAddress(targetAddr)
                    .setRemotePeerFactory(dfsClient).setDatanodeInfo(chosenNode).setStorageType(storageType)
                    .setFileName(src).setBlock(blk).setBlockToken(accessToken).setStartOffset(offsetIntoBlock)
                    .setVerifyChecksum(verifyChecksum).setClientName(dfsClient.clientName)
                    .setLength(blk.getNumBytes() - offsetIntoBlock).setCachingStrategy(curCachingStrategy)
                    .setAllowShortCircuitLocalReads(!shortCircuitForbidden)
                    .setClientCacheContext(dfsClient.getClientContext()).setUserGroupInformation(dfsClient.ugi)
                    .setConfiguration(dfsClient.getConfiguration()).build();
            if (connectFailedOnce) {
                DFSClient.LOG.info("Successfully connected to " + targetAddr + " for " + blk);
            }
            return chosenNode;
        } catch (IOException ex) {
            if (ex instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
                DFSClient.LOG.info("Will fetch a new encryption key and retry, "
                        + "encryption key was invalid when connecting to " + targetAddr + " : " + ex);
                // The encryption key used is invalid.
                refetchEncryptionKey--;
                dfsClient.clearDataEncryptionKey();
            } else if (refetchToken > 0 && tokenRefetchNeeded(ex, targetAddr)) {
                refetchToken--;
                fetchBlockAt(target);
            } else {
                connectFailedOnce = true;
                DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block"
                        + ", add to deadNodes and continue. " + ex, ex);
                // Put chosen node into dead list, continue
                addToDeadNodes(chosenNode);
            }
        }
    }
}

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

private DNAddrPair chooseDataNode(LocatedBlock block, Collection<DatanodeInfo> ignoredNodes)
        throws IOException {
    while (true) {
        try {/*from  w  w  w.  ja va2 s .c  o  m*/
            return getBestNodeDNAddrPair(block, ignoredNodes);
        } catch (IOException ie) {
            String errMsg = getBestNodeDNAddrPairErrorString(block.getLocations(), deadNodes, ignoredNodes);
            String blockInfo = block.getBlock() + " file=" + src;
            if (failures >= dfsClient.getMaxBlockAcquireFailures()) {
                String description = "Could not obtain block: " + blockInfo;
                DFSClient.LOG.warn(description + errMsg + ". Throwing a BlockMissingException");
                throw new BlockMissingException(src, description, block.getStartOffset());
            }

            DatanodeInfo[] nodes = block.getLocations();
            if (nodes == null || nodes.length == 0) {
                DFSClient.LOG.info("No node available for " + blockInfo);
            }
            DFSClient.LOG.info("Could not obtain " + block.getBlock() + " from any node: " + ie + errMsg
                    + ". Will get new block locations from namenode and retry...");
            try {
                // Introducing a random factor to the wait time before another retry.
                // The wait time is dependent on # of failures and a random factor.
                // At the first time of getting a BlockMissingException, the wait time
                // is a random number between 0..3000 ms. If the first retry
                // still fails, we will wait 3000 ms grace period before the 2nd retry.
                // Also at the second retry, the waiting window is expanded to 6000 ms
                // alleviating the request rate from the server. Similarly the 3rd retry
                // will wait 6000ms grace period before retry and the waiting window is
                // expanded to 9000ms.
                final int timeWindow = dfsClient.getConf().getTimeWindow();
                double waitTime = timeWindow * failures + // grace period for the last round of attempt
                        timeWindow * (failures + 1) * DFSUtil.getRandom().nextDouble(); // expanding time window for each failure
                DFSClient.LOG.warn("DFS chooseDataNode: got # " + (failures + 1)
                        + " IOException, will wait for " + waitTime + " msec.");
                Thread.sleep((long) waitTime);
            } catch (InterruptedException iex) {
            }
            deadNodes.clear(); // 2nd option is to remove only nodes[blockId]
            openInfo();
            block = getBlockAt(block.getStartOffset());
            failures++;
            continue;
        }
    }
}

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

/**
 * Get the best node from which to stream the data.
 * /*w  w w.  ja v a  2s .  co  m*/
 * @param block
 *            LocatedBlock, containing nodes in priority order.
 * @param ignoredNodes
 *            Do not choose nodes in this array (may be null)
 * @return The DNAddrPair of the best node.
 * @throws IOException
 */
private DNAddrPair getBestNodeDNAddrPair(LocatedBlock block, Collection<DatanodeInfo> ignoredNodes)
        throws IOException {
    DatanodeInfo[] nodes = block.getLocations();
    StorageType[] storageTypes = block.getStorageTypes();
    DatanodeInfo chosenNode = null;
    StorageType storageType = null;
    if (nodes != null) {
        for (int i = 0; i < nodes.length; i++) {
            if (!deadNodes.containsKey(nodes[i])
                    && (ignoredNodes == null || !ignoredNodes.contains(nodes[i]))) {
                chosenNode = nodes[i];
                // Storage types are ordered to correspond with nodes, so use the same
                // index to get storage type.
                if (storageTypes != null && i < storageTypes.length) {
                    storageType = storageTypes[i];
                }
                break;
            }
        }
    }
    if (chosenNode == null) {
        throw new IOException("No live nodes contain block " + block.getBlock() + " after checking nodes = "
                + Arrays.toString(nodes) + ", ignoredNodes = " + ignoredNodes);
    }
    final String dnAddr = chosenNode.getXferAddr(dfsClient.getConf().getConnectToDnViaHostname());
    if (DFSClient.LOG.isDebugEnabled()) {
        DFSClient.LOG.debug("Connecting to datanode " + dnAddr);
    }
    InetSocketAddress targetAddr = NetUtils.createSocketAddr(dnAddr);
    return new DNAddrPair(chosenNode, targetAddr, storageType);
}

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

private void actualGetFromOneDataNode(final DNAddrPair datanode, LocatedBlock block, final long start,
        final long end, byte[] buf, int offset, Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
        throws IOException {
    DFSClientFaultInjector.get().startFetchFromDatanode();
    int refetchToken = 1; // only need to get a new access token once
    int refetchEncryptionKey = 1; // only need to get a new encryption key once

    while (true) {
        // cached block locations may have been updated by chooseDataNode()
        // or fetchBlockAt(). Always get the latest list of locations at the
        // start of the loop.
        CachingStrategy curCachingStrategy;
        boolean allowShortCircuitLocalReads;
        block = getBlockAt(block.getStartOffset());
        synchronized (infoLock) {
            curCachingStrategy = cachingStrategy;
            allowShortCircuitLocalReads = !shortCircuitForbidden();
        }//from www .  ja  v a  2  s .c  om
        DatanodeInfo chosenNode = datanode.info;
        InetSocketAddress targetAddr = datanode.addr;
        StorageType storageType = datanode.storageType;
        BlockReader reader = null;

        try {
            DFSClientFaultInjector.get().fetchFromDatanodeException();
            Token<BlockTokenIdentifier> blockToken = block.getBlockToken();
            int len = (int) (end - start + 1);
            reader = new BlockReaderFactory(dfsClient.getConf()).setInetSocketAddress(targetAddr)
                    .setRemotePeerFactory(dfsClient).setDatanodeInfo(chosenNode).setStorageType(storageType)
                    .setFileName(src).setBlock(block.getBlock()).setBlockToken(blockToken).setStartOffset(start)
                    .setVerifyChecksum(verifyChecksum).setClientName(dfsClient.clientName).setLength(len)
                    .setCachingStrategy(curCachingStrategy)
                    .setAllowShortCircuitLocalReads(allowShortCircuitLocalReads)
                    .setClientCacheContext(dfsClient.getClientContext()).setUserGroupInformation(dfsClient.ugi)
                    .setConfiguration(dfsClient.getConfiguration()).build();
            int nread = reader.readAll(buf, offset, len);
            updateReadStatistics(readStatistics, nread, reader);

            if (nread != len) {
                throw new IOException(
                        "truncated return from reader.read(): " + "excpected " + len + ", got " + nread);
            }
            DFSClientFaultInjector.get().readFromDatanodeDelay();
            return;
        } catch (ChecksumException e) {
            String msg = "fetchBlockByteRange(). Got a checksum exception for " + src + " at "
                    + block.getBlock() + ":" + e.getPos() + " from " + chosenNode;
            DFSClient.LOG.warn(msg);
            // we want to remember what we have tried
            addIntoCorruptedBlockMap(block.getBlock(), chosenNode, corruptedBlockMap);
            addToDeadNodes(chosenNode);
            throw new IOException(msg);
        } catch (IOException e) {
            if (e instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
                DFSClient.LOG.info("Will fetch a new encryption key and retry, "
                        + "encryption key was invalid when connecting to " + targetAddr + " : " + e);
                // The encryption key used is invalid.
                refetchEncryptionKey--;
                dfsClient.clearDataEncryptionKey();
                continue;
            } else if (refetchToken > 0 && tokenRefetchNeeded(e, targetAddr)) {
                refetchToken--;
                try {
                    fetchBlockAt(block.getStartOffset());
                } catch (IOException fbae) {
                    // ignore IOE, since we can retry it later in a loop
                }
                continue;
            } else {
                String msg = "Failed to connect to " + targetAddr + " for file " + src + " for block "
                        + block.getBlock() + ":" + e;
                DFSClient.LOG.warn("Connection failure: " + msg, e);
                addToDeadNodes(chosenNode);
                throw new IOException(msg);
            }
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
    }
}