Example usage for org.apache.hadoop.util DataChecksum getBytesPerChecksum

Introduction

In this page you can find the example usage for org.apache.hadoop.util DataChecksum getBytesPerChecksum.

Prototype

public int getBytesPerChecksum() 

Source Link

Usage

From source file:com.mellanox.r4h.DFSOutputStream.java

License:Apache License

/**
 * @return the object for computing checksum.
 *         The type is NULL if checksum is not computed.
 *///from  w ww.j a v  a2 s .c om
private static DataChecksum getChecksum4Compute(DataChecksum checksum, HdfsFileStatus stat) {
    if (isLazyPersist(stat) && stat.getReplication() == 1) {
        // do not compute checksum for writing to single replica to memory
        return DataChecksum.newDataChecksum(Type.NULL, checksum.getBytesPerChecksum());
    }
    return checksum;
}

---

From source file:com.mellanox.r4h.DFSOutputStream.java

License:Apache License

protected DFSOutputStream(DFSClient dfsClient, String src, Progressable progress, HdfsFileStatus stat,
        DataChecksum checksum) throws IOException {
    super(getChecksum4Compute(checksum, stat));
    this.dfsClient = dfsClient;
    this.src = src;
    this.fileId = stat.getFileId();
    this.blockSize = stat.getBlockSize();
    this.blockReplication = stat.getReplication();
    this.fileEncryptionInfo = stat.getFileEncryptionInfo();
    this.progress = progress;
    this.cachingStrategy = new AtomicReference<CachingStrategy>(dfsClient.getDefaultWriteCachingStrategy());
    this.jxioResource = dfsClient.getJXIOResource();
    this.usingJxioClientResource = true;
    this.eventQHandler = jxioResource.getEqh();
    this.msgPool = jxioResource.getMsgPool();
    this.headerAckTimeoutUsec = dfsClient.getConf().getHeaderAckTimeoutUsec();

    if ((progress != null) && DFSOutputStream.LOG.isDebugEnabled()) {
        DFSOutputStream.LOG.debug("Set non-null progress callback on DFSOutputStream " + src);
    }/*from   w w w .j  a  v  a 2 s. co  m*/

    this.bytesPerChecksum = checksum.getBytesPerChecksum();
    if (bytesPerChecksum < 1 || blockSize % bytesPerChecksum != 0) {
        throw new IOException("io.bytes.per.checksum(" + bytesPerChecksum + ") and blockSize(" + blockSize
                + ") do not match. " + "blockSize should be a " + "multiple of io.bytes.per.checksum");

    }
    this.name = String.format("[hash=%X] ", DFSOutputStream.this.hashCode());
    this.checksum = checksum;

    if (toPrintBreakdown) {
        long now = System.nanoTime();
        DFSOutputStream.LOG.info(String.format("%.3f", (float) now / 1000000000.) + ", "
                + (now - lastOperationTS) / 1000000000. + " : DFSOutputStream constructed successfully.");
        synchronized (lastOperationTSLock) {
            lastOperationTS = now;
        }
        // } else {
        // LOG.info("Mellanox RDMA-accelerated DFSOutputStream constructed successfully.");
    }
}

---