Example usage for org.apache.hadoop.hdfs.protocol LocatedBlock getBlockSize

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs.protocol LocatedBlock getBlockSize.

Prototype

public long getBlockSize() 

Source Link

Usage

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

private long fetchLocatedBlocksAndGetLastBlockLength() throws IOException {
    final LocatedBlocks newInfo = dfsClient.getLocatedBlocks(src, 0);
    if (DFSClient.LOG.isDebugEnabled()) {
        DFSClient.LOG.debug("newInfo = " + newInfo);
    }/*from  w  ww. j a  va 2s .c  o  m*/
    if (newInfo == null) {
        throw new IOException("Cannot open filename " + src);
    }

    if (locatedBlocks != null) {
        Iterator<LocatedBlock> oldIter = locatedBlocks.getLocatedBlocks().iterator();
        Iterator<LocatedBlock> newIter = newInfo.getLocatedBlocks().iterator();
        while (oldIter.hasNext() && newIter.hasNext()) {
            if (!oldIter.next().getBlock().equals(newIter.next().getBlock())) {
                throw new IOException("Blocklist for " + src + " has changed!");
            }
        }
    }
    locatedBlocks = newInfo;
    long lastBlockBeingWrittenLength = 0;
    if (!locatedBlocks.isLastBlockComplete()) {
        final LocatedBlock last = locatedBlocks.getLastLocatedBlock();
        if (last != null) {
            if (last.getLocations().length == 0) {
                if (last.getBlockSize() == 0) {
                    // if the length is zero, then no data has been written to
                    // datanode. So no need to wait for the locations.
                    return 0;
                }
                return -1;
            }
            final long len = readBlockLength(last);
            last.getBlock().setNumBytes(len);
            lastBlockBeingWrittenLength = len;
        }
    }

    fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo();

    return lastBlockBeingWrittenLength;
}

---

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

/**
 * Get blocks in the specified range./*from  ww  w  .  j  a va2 s .c  om*/
 * Includes only the complete blocks.
 * Fetch them from the namenode if not cached.
 */
private List<LocatedBlock> getFinalizedBlockRange(long offset, long length) throws IOException {
    synchronized (infoLock) {
        assert (locatedBlocks != null) : "locatedBlocks is null";
        List<LocatedBlock> blockRange = new ArrayList<LocatedBlock>();
        // search cached blocks first
        int blockIdx = locatedBlocks.findBlock(offset);
        if (blockIdx < 0) { // block is not cached
            blockIdx = LocatedBlocks.getInsertIndex(blockIdx);
        }
        long remaining = length;
        long curOff = offset;
        while (remaining > 0) {
            LocatedBlock blk = null;
            if (blockIdx < locatedBlocks.locatedBlockCount())
                blk = locatedBlocks.get(blockIdx);
            if (blk == null || curOff < blk.getStartOffset()) {
                LocatedBlocks newBlocks;
                newBlocks = dfsClient.getLocatedBlocks(src, curOff, remaining);
                locatedBlocks.insertRange(blockIdx, newBlocks.getLocatedBlocks());
                continue;
            }
            assert curOff >= blk.getStartOffset() : "Block not found";
            blockRange.add(blk);
            long bytesRead = blk.getStartOffset() + blk.getBlockSize() - curOff;
            remaining -= bytesRead;
            curOff += bytesRead;
            blockIdx++;
        }
        return blockRange;
    }
}

---

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

/**
 * Open a DataInputStream to a DataNode so that it can be read from.
 * We get block ID and the IDs of the destinations at startup, from the namenode.
 *//* ww w  .  ja v a 2s .com*/
private synchronized DatanodeInfo blockSeekTo(long target) throws IOException {
    if (target >= getFileLength()) {
        throw new IOException("Attempted to read past end of file");
    }

    // Will be getting a new BlockReader.
    closeCurrentBlockReader();

    //
    // Connect to best DataNode for desired Block, with potential offset
    //
    DatanodeInfo chosenNode = null;
    int refetchToken = 1; // only need to get a new access token once
    int refetchEncryptionKey = 1; // only need to get a new encryption key once

    boolean connectFailedOnce = false;

    while (true) {
        //
        // Compute desired block
        //
        LocatedBlock targetBlock = getBlockAt(target);

        // update current position
        this.pos = target;
        this.blockEnd = targetBlock.getStartOffset() + targetBlock.getBlockSize() - 1;
        this.currentLocatedBlock = targetBlock;

        assert (target == pos) : "Wrong postion " + pos + " expect " + target;
        long offsetIntoBlock = target - targetBlock.getStartOffset();

        DNAddrPair retval = chooseDataNode(targetBlock, null);
        chosenNode = retval.info;
        InetSocketAddress targetAddr = retval.addr;
        StorageType storageType = retval.storageType;

        try {
            ExtendedBlock blk = targetBlock.getBlock();
            Token<BlockTokenIdentifier> accessToken = targetBlock.getBlockToken();
            CachingStrategy curCachingStrategy;
            boolean shortCircuitForbidden;
            synchronized (infoLock) {
                curCachingStrategy = cachingStrategy;
                shortCircuitForbidden = shortCircuitForbidden();
            }
            blockReader = new BlockReaderFactory(dfsClient.getConf()).setInetSocketAddress(targetAddr)
                    .setRemotePeerFactory(dfsClient).setDatanodeInfo(chosenNode).setStorageType(storageType)
                    .setFileName(src).setBlock(blk).setBlockToken(accessToken).setStartOffset(offsetIntoBlock)
                    .setVerifyChecksum(verifyChecksum).setClientName(dfsClient.clientName)
                    .setLength(blk.getNumBytes() - offsetIntoBlock).setCachingStrategy(curCachingStrategy)
                    .setAllowShortCircuitLocalReads(!shortCircuitForbidden)
                    .setClientCacheContext(dfsClient.getClientContext()).setUserGroupInformation(dfsClient.ugi)
                    .setConfiguration(dfsClient.getConfiguration()).build();
            if (connectFailedOnce) {
                DFSClient.LOG.info("Successfully connected to " + targetAddr + " for " + blk);
            }
            return chosenNode;
        } catch (IOException ex) {
            if (ex instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
                DFSClient.LOG.info("Will fetch a new encryption key and retry, "
                        + "encryption key was invalid when connecting to " + targetAddr + " : " + ex);
                // The encryption key used is invalid.
                refetchEncryptionKey--;
                dfsClient.clearDataEncryptionKey();
            } else if (refetchToken > 0 && tokenRefetchNeeded(ex, targetAddr)) {
                refetchToken--;
                fetchBlockAt(target);
            } else {
                connectFailedOnce = true;
                DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block"
                        + ", add to deadNodes and continue. " + ex, ex);
                // Put chosen node into dead list, continue
                addToDeadNodes(chosenNode);
            }
        }
    }
}

---

From source file:com.mellanox.r4h.DFSInputStream.java

License:Apache License

private int pread(long position, byte[] buffer, int offset, int length) throws IOException {
    // sanity checks
    dfsClient.checkOpen();//from   w w w  .  j  a  v  a  2 s .  co m
    if (closed.get()) {
        throw new IOException("Stream closed");
    }
    failures = 0;
    long filelen = getFileLength();
    if ((position < 0) || (position >= filelen)) {
        return -1;
    }
    int realLen = length;
    if ((position + length) > filelen) {
        realLen = (int) (filelen - position);
    }

    // determine the block and byte range within the block
    // corresponding to position and realLen
    List<LocatedBlock> blockRange = getBlockRange(position, realLen);
    int remaining = realLen;
    Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap = new HashMap<ExtendedBlock, Set<DatanodeInfo>>();
    for (LocatedBlock blk : blockRange) {
        long targetStart = position - blk.getStartOffset();
        long bytesToRead = Math.min(remaining, blk.getBlockSize() - targetStart);
        try {
            if (dfsClient.isHedgedReadsEnabled()) {
                hedgedFetchBlockByteRange(blk, targetStart, targetStart + bytesToRead - 1, buffer, offset,
                        corruptedBlockMap);
            } else {
                fetchBlockByteRange(blk, targetStart, targetStart + bytesToRead - 1, buffer, offset,
                        corruptedBlockMap);
            }
        } finally {
            // Check and report if any block replicas are corrupted.
            // BlockMissingException may be caught if all block replicas are
            // corrupted.
            reportCheckSumFailure(corruptedBlockMap, blk.getLocations().length);
        }

        remaining -= bytesToRead;
        position += bytesToRead;
        offset += bytesToRead;
    }
    assert remaining == 0 : "Wrong number of bytes read.";
    if (dfsClient.stats != null) {
        dfsClient.stats.incrementBytesRead(realLen);
    }
    return realLen;
}

---

From source file:com.mellanox.r4h.DFSOutputStream.java

License:Apache License

/** Construct a new output stream for append. */
private DFSOutputStream(DFSClient dfsClient, String src, EnumSet<CreateFlag> flags, Progressable progress,
        LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum) throws IOException {
    this(dfsClient, src, progress, stat, checksum);
    initialFileSize = stat.getLen(); // length of file when opened

    this.shouldSyncBlock = flags.contains(CreateFlag.SYNC_BLOCK);
    boolean toNewBlock = flags.contains(CreateFlag.NEW_BLOCK);

    // The last partial block of the file has to be filled.
    if (!toNewBlock && lastBlock != null) {
        // indicate that we are appending to an existing block
        bytesCurBlock = lastBlock.getBlockSize();
        streamer = new DataStreamer(lastBlock, stat, bytesPerChecksum);
    } else {/*from   w  w  w.ja  v a 2s  . co m*/
        computePacketChunkSize(dfsClient.getConf().getWritePacketSize(), bytesPerChecksum);
        streamer = new DataStreamer(stat, lastBlock != null ? lastBlock.getBlock() : null);
    }
    this.fileEncryptionInfo = stat.getFileEncryptionInfo();
}

---

From source file:io.hops.erasure_coding.TestMapReduceBlockRepairManager.java

License:Apache License

@Test
public void testCorruptedRepair() throws IOException, InterruptedException {
    DistributedFileSystem dfs = (DistributedFileSystem) getFileSystem();
    TestDfsClient testDfsClient = new TestDfsClient(getConfig());
    testDfsClient.injectIntoDfs(dfs);//  w w  w  . j av a  2  s.c o  m

    MapReduceEncodingManager encodingManager = new MapReduceEncodingManager(conf);

    Util.createRandomFile(dfs, testFile, seed, TEST_BLOCK_COUNT, DFS_TEST_BLOCK_SIZE);
    Codec.initializeCodecs(conf);
    FileStatus testFileStatus = dfs.getFileStatus(testFile);
    EncodingPolicy policy = new EncodingPolicy("src", (short) 1);
    encodingManager.encodeFile(policy, testFile, parityFile);

    // Busy waiting until the encoding is done
    while (encodingManager.computeReports().size() > 0) {
        ;
    }

    String path = testFileStatus.getPath().toUri().getPath();
    int blockToLoose = new Random(seed)
            .nextInt((int) (testFileStatus.getLen() / testFileStatus.getBlockSize()));
    LocatedBlock lb = dfs.getClient().getLocatedBlocks(path, 0, Long.MAX_VALUE).get(blockToLoose);
    DataNodeUtil.loseBlock(getCluster(), lb);
    List<LocatedBlock> lostBlocks = new ArrayList<LocatedBlock>();
    lostBlocks.add(lb);
    LocatedBlocks locatedBlocks = new LocatedBlocks(0, false, lostBlocks, null, true);
    testDfsClient.setMissingLocatedBlocks(locatedBlocks);
    LOG.info("Loosing block " + lb.toString());
    getCluster().triggerBlockReports();

    dfs.getClient().addBlockChecksum(testFile.toUri().getPath(),
            (int) (lb.getStartOffset() / lb.getBlockSize()), 0);

    MapReduceBlockRepairManager repairManager = new MapReduceBlockRepairManager(conf);
    repairManager.repairSourceBlocks("src", testFile, parityFile);

    while (true) {
        List<Report> reports = repairManager.computeReports();
        if (reports.size() == 0) {
            break;
        }
        LOG.info(reports.get(0).getStatus());
        System.out.println("WAIT");
        Thread.sleep(1000);
    }

    try {
        FSDataInputStream in = dfs.open(testFile);
        byte[] buff = new byte[TEST_BLOCK_COUNT * DFS_TEST_BLOCK_SIZE];
        in.readFully(0, buff);
        fail("Repair succeeded with bogus checksum.");
    } catch (BlockMissingException e) {
    }
}

---