Example usage for org.apache.hadoop.hdfs.server.datanode.fsdataset FsDatasetSpi isValidBlock

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs.server.datanode.fsdataset FsDatasetSpi isValidBlock.

Prototype

boolean isValidBlock(ExtendedBlock b);

Source Link

Document

Is the block valid?

Usage

From source file:backup.datanode.DataNodeBackupProcessor.java

License:Apache License

@Override
protected long doBackup(ExtendedBlock extendedBlock, boolean force) throws Exception {
    if (!force) {
        long blockId = extendedBlock.getBlockId();
        DatanodeUuids result;/*  ww  w.ja va2 s  . c  o m*/
        try {
            result = _nameNodeClient.getDatanodeUuids(blockId);
        } catch (Exception e) {
            if (LOG.isDebugEnabled()) {
                LOG.error("datanode UUID lookup failed " + e.getCause(), e);
            } else {
                LOG.error("datanode UUID lookup failed {}", e.getCause());
            }
            result = new DatanodeUuids();
        }
        List<String> datanodeUuids = result.getDatanodeUuids();
        if (!datanodeUuids.isEmpty()) {
            LOG.debug("datanode UUIDs for block id {} {}", blockId, datanodeUuids);
            List<String> orderedDataNodeUuids = getOrderDataNodeUuids(datanodeUuids, blockId);
            String datanodeUuid = _datanode.getDatanodeUuid();
            int binarySearch = Collections.binarySearch(orderedDataNodeUuids, datanodeUuid);
            if (binarySearch == 0) {
                LOG.debug("datanode UUID {} first in list {} for block id {}, performing backup", datanodeUuid,
                        datanodeUuids, blockId);
            } else if (binarySearch > 0) {
                return binarySearch * _retryDelay;
            } else {
                LOG.debug("datanode UUID {} not found in list {} for block id {}, forcing backup", datanodeUuid,
                        datanodeUuids, blockId);
            }
        } else {
            LOG.debug("datanode UUIDs for block id {} empty, forcing backup", blockId);
        }
    }

    if (_backupStore.hasBlock(extendedBlock)) {
        LOG.debug("block {} already backed up", extendedBlock);
        return 0;
    }

    FsDatasetSpi<?> fsDataset = _datanode.getFSDataset();

    org.apache.hadoop.hdfs.protocol.ExtendedBlock heb = BackupUtil.toHadoop(extendedBlock);
    if (!fsDataset.isValidBlock(heb)) {
        return 0;
    }
    BlockLocalPathInfo info = fsDataset.getBlockLocalPathInfo(heb);
    String blockPath = info.getBlockPath();
    if (Files.isSymbolicLink(new File(blockPath).toPath())) {
        LOG.debug("block {} is symbolic link, not backing up", extendedBlock);
        return 0;
    }

    LOG.info("performing block {}", extendedBlock);
    long numBytes = heb.getNumBytes();
    try (LengthInputStream data = new LengthInputStream(trackThroughPut(fsDataset.getBlockInputStream(heb, 0)),
            numBytes)) {
        org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream tmeta;
        tmeta = fsDataset.getMetaDataInputStream(heb);
        try (LengthInputStream meta = new LengthInputStream(trackThroughPut(tmeta), tmeta.getLength())) {
            _backupStore.backupBlock(extendedBlock, data, meta);
            return 0;
        }
    } catch (IOException e) {
        if (LOG.isDebugEnabled() || !e.getMessage().contains("is not valid")) {
            LOG.error(e.getMessage(), e);
        } else {
            LOG.debug("block {} has been removed {}", extendedBlock);
        }
        return 0;
    }
}

---

From source file:backup.datanode.DataNodeRestoreProcessor.java

License:Apache License

public void restoreBlock(ExtendedBlock extendedBlock) throws Exception {
    if (!_backupStore.hasBlock(extendedBlock)) {
        LOG.error("Can not restore block, not in block store {}", extendedBlock);
        return;//from  www  .j  a va 2 s  . c o m
    }
    FsDatasetSpi<?> fsDataset = _datanode.getFSDataset();
    org.apache.hadoop.hdfs.protocol.ExtendedBlock heb = BackupUtil.toHadoop(extendedBlock);
    if (fsDataset.isValidBlock(heb)) {
        LOG.info("Block already restored {}", extendedBlock);
        return;
    }
    try {
        _restoreInProgress.incrementAndGet();
        LOG.info("Restoring block {}", extendedBlock);
        boolean allowLazyPersist = true;
        // org.apache.hadoop.fs.StorageType storageType =
        // org.apache.hadoop.fs.StorageType.DEFAULT;
        org.apache.hadoop.hdfs.StorageType storageType = org.apache.hadoop.hdfs.StorageType.DEFAULT;
        ReplicaHandler replicaHandler = fsDataset.createRbw(storageType, heb, allowLazyPersist);
        ReplicaInPipelineInterface pipelineInterface = replicaHandler.getReplica();
        boolean isCreate = true;
        DataChecksum requestedChecksum = DataChecksum.newDataChecksum(_checksumType, _bytesPerChecksum);
        int bytesCopied = 0;
        try (ReplicaOutputStreams streams = pipelineInterface.createStreams(isCreate, requestedChecksum)) {
            try (OutputStream checksumOut = streams.getChecksumOut()) {
                try (InputStream metaData = _backupStore.getMetaDataInputStream(extendedBlock)) {
                    LOG.info("Restoring meta data for block {}", extendedBlock);
                    IOUtils.copy(trackThroughPut(metaData), checksumOut);
                }
            }
            try (OutputStream dataOut = streams.getDataOut()) {
                try (InputStream data = _backupStore.getDataInputStream(extendedBlock)) {
                    LOG.info("Restoring data for block {}", extendedBlock);
                    bytesCopied = IOUtils.copy(trackThroughPut(data), dataOut);
                }
            }
        }
        pipelineInterface.setNumBytes(bytesCopied);
        LOG.info("Finalizing restored block {}", extendedBlock);
        fsDataset.finalizeBlock(heb);

        // datanode.notifyNamenodeReceivedBlock(extendedBlock, "",
        // pipelineInterface.getStorageUuid();
        _datanode.notifyNamenodeReceivedBlock(heb, "", pipelineInterface.getStorageUuid(),
                pipelineInterface.isOnTransientStorage());
    } catch (ReplicaAlreadyExistsException e) {
        LOG.info("Restoring block already exists {}", extendedBlock);
    } finally {
        _restoreInProgress.decrementAndGet();
    }
}

---