Example usage for org.apache.hadoop.fs ChecksumException getPos

Introduction

In this page you can find the example usage for org.apache.hadoop.fs ChecksumException getPos.

Prototype

public long getPos() 

Source Link

Usage

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

private synchronized int readBuffer(ReaderStrategy reader, int off, int len,
        Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap) throws IOException {
    IOException ioe;//from w ww .ja  v  a 2  s .c  om

    /*
     * we retry current node only once. So this is set to true only here.
     * Intention is to handle one common case of an error that is not a
     * failure on datanode or client : when DataNode closes the connection
     * since client is idle. If there are other cases of "non-errors" then
     * then a datanode might be retried by setting this to true again.
     */
    boolean retryCurrentNode = true;

    while (true) {
        // retry as many times as seekToNewSource allows.
        try {
            return reader.doRead(blockReader, off, len);
        } catch (ChecksumException ce) {
            DFSClient.LOG.warn("Found Checksum error for " + getCurrentBlock() + " from " + currentNode + " at "
                    + ce.getPos());
            ioe = ce;
            retryCurrentNode = false;
            // we want to remember which block replicas we have tried
            addIntoCorruptedBlockMap(getCurrentBlock(), currentNode, corruptedBlockMap);
        } catch (IOException e) {
            if (!retryCurrentNode) {
                DFSClient.LOG.warn("Exception while reading from " + getCurrentBlock() + " of " + src + " from "
                        + currentNode, e);
            }
            ioe = e;
        }
        boolean sourceFound = false;
        if (retryCurrentNode) {
            /*
             * possibly retry the same node so that transient errors don't
             * result in application level failures (e.g. Datanode could have
             * closed the connection because the client is idle for too long).
             */
            sourceFound = seekToBlockSource(pos);
        } else {
            addToDeadNodes(currentNode);
            sourceFound = seekToNewSource(pos);
        }
        if (!sourceFound) {
            throw ioe;
        }
        retryCurrentNode = false;
    }
}

---

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

private void actualGetFromOneDataNode(final DNAddrPair datanode, LocatedBlock block, final long start,
        final long end, byte[] buf, int offset, Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
        throws IOException {
    DFSClientFaultInjector.get().startFetchFromDatanode();
    int refetchToken = 1; // only need to get a new access token once
    int refetchEncryptionKey = 1; // only need to get a new encryption key once

    while (true) {
        // cached block locations may have been updated by chooseDataNode()
        // or fetchBlockAt(). Always get the latest list of locations at the
        // start of the loop.
        CachingStrategy curCachingStrategy;
        boolean allowShortCircuitLocalReads;
        block = getBlockAt(block.getStartOffset());
        synchronized (infoLock) {
            curCachingStrategy = cachingStrategy;
            allowShortCircuitLocalReads = !shortCircuitForbidden();
        }/*  ww w .  j a va2 s .  c  om*/
        DatanodeInfo chosenNode = datanode.info;
        InetSocketAddress targetAddr = datanode.addr;
        StorageType storageType = datanode.storageType;
        BlockReader reader = null;

        try {
            DFSClientFaultInjector.get().fetchFromDatanodeException();
            Token<BlockTokenIdentifier> blockToken = block.getBlockToken();
            int len = (int) (end - start + 1);
            reader = new BlockReaderFactory(dfsClient.getConf()).setInetSocketAddress(targetAddr)
                    .setRemotePeerFactory(dfsClient).setDatanodeInfo(chosenNode).setStorageType(storageType)
                    .setFileName(src).setBlock(block.getBlock()).setBlockToken(blockToken).setStartOffset(start)
                    .setVerifyChecksum(verifyChecksum).setClientName(dfsClient.clientName).setLength(len)
                    .setCachingStrategy(curCachingStrategy)
                    .setAllowShortCircuitLocalReads(allowShortCircuitLocalReads)
                    .setClientCacheContext(dfsClient.getClientContext()).setUserGroupInformation(dfsClient.ugi)
                    .setConfiguration(dfsClient.getConfiguration()).build();
            int nread = reader.readAll(buf, offset, len);
            updateReadStatistics(readStatistics, nread, reader);

            if (nread != len) {
                throw new IOException(
                        "truncated return from reader.read(): " + "excpected " + len + ", got " + nread);
            }
            DFSClientFaultInjector.get().readFromDatanodeDelay();
            return;
        } catch (ChecksumException e) {
            String msg = "fetchBlockByteRange(). Got a checksum exception for " + src + " at "
                    + block.getBlock() + ":" + e.getPos() + " from " + chosenNode;
            DFSClient.LOG.warn(msg);
            // we want to remember what we have tried
            addIntoCorruptedBlockMap(block.getBlock(), chosenNode, corruptedBlockMap);
            addToDeadNodes(chosenNode);
            throw new IOException(msg);
        } catch (IOException e) {
            if (e instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
                DFSClient.LOG.info("Will fetch a new encryption key and retry, "
                        + "encryption key was invalid when connecting to " + targetAddr + " : " + e);
                // The encryption key used is invalid.
                refetchEncryptionKey--;
                dfsClient.clearDataEncryptionKey();
                continue;
            } else if (refetchToken > 0 && tokenRefetchNeeded(e, targetAddr)) {
                refetchToken--;
                try {
                    fetchBlockAt(block.getStartOffset());
                } catch (IOException fbae) {
                    // ignore IOE, since we can retry it later in a loop
                }
                continue;
            } else {
                String msg = "Failed to connect to " + targetAddr + " for file " + src + " for block "
                        + block.getBlock() + ":" + e;
                DFSClient.LOG.warn("Connection failure: " + msg, e);
                addToDeadNodes(chosenNode);
                throw new IOException(msg);
            }
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
    }
}

---