Example usage for org.apache.hadoop.hdfs DFSClient getConf

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DFSClient getConf.

Prototype

public DfsClientConf getConf() 

Source Link

Usage

From source file:com.mellanox.r4h.DFSOutputStream.java

License:Apache License

/**
 * Create a socket for a write pipeline/*from   w  w  w.j a  v a2 s .  co m*/
 * 
 * @param first
 *            the first datanode
 * @param length
 *            the pipeline length
 * @param client
 *            client
 * @return the socket connected to the first datanode
 */
static Socket createSocketForPipeline(final DatanodeInfo first, final int length, final DFSClient client)
        throws IOException {
    final String dnAddr = first.getXferAddr(client.getConf().getConnectToDnViaHostname());
    if (DFSClient.LOG.isDebugEnabled()) {
        DFSClient.LOG.debug("Connecting to datanode " + dnAddr);
    }
    final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr);
    final Socket sock = client.socketFactory.createSocket();
    final int timeout = client.getDatanodeReadTimeout(length);
    NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), client.getConf().getSocketTimeout());
    sock.setSoTimeout(timeout);
    sock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE);
    if (DFSClient.LOG.isDebugEnabled()) {
        DFSClient.LOG.debug("Send buf size " + sock.getSendBufferSize());
    }
    return sock;
}

---

From source file:com.mellanox.r4h.DFSOutputStream.java

License:Apache License

protected DFSOutputStream(DFSClient dfsClient, String src, Progressable progress, HdfsFileStatus stat,
        DataChecksum checksum) throws IOException {
    super(getChecksum4Compute(checksum, stat));
    this.dfsClient = dfsClient;
    this.src = src;
    this.fileId = stat.getFileId();
    this.blockSize = stat.getBlockSize();
    this.blockReplication = stat.getReplication();
    this.fileEncryptionInfo = stat.getFileEncryptionInfo();
    this.progress = progress;
    this.cachingStrategy = new AtomicReference<CachingStrategy>(dfsClient.getDefaultWriteCachingStrategy());
    this.jxioResource = dfsClient.getJXIOResource();
    this.usingJxioClientResource = true;
    this.eventQHandler = jxioResource.getEqh();
    this.msgPool = jxioResource.getMsgPool();
    this.headerAckTimeoutUsec = dfsClient.getConf().getHeaderAckTimeoutUsec();

    if ((progress != null) && DFSOutputStream.LOG.isDebugEnabled()) {
        DFSOutputStream.LOG.debug("Set non-null progress callback on DFSOutputStream " + src);
    }/*w  ww  .ja  va2s .co  m*/

    this.bytesPerChecksum = checksum.getBytesPerChecksum();
    if (bytesPerChecksum < 1 || blockSize % bytesPerChecksum != 0) {
        throw new IOException("io.bytes.per.checksum(" + bytesPerChecksum + ") and blockSize(" + blockSize
                + ") do not match. " + "blockSize should be a " + "multiple of io.bytes.per.checksum");

    }
    this.name = String.format("[hash=%X] ", DFSOutputStream.this.hashCode());
    this.checksum = checksum;

    if (toPrintBreakdown) {
        long now = System.nanoTime();
        DFSOutputStream.LOG.info(String.format("%.3f", (float) now / 1000000000.) + ", "
                + (now - lastOperationTS) / 1000000000. + " : DFSOutputStream constructed successfully.");
        synchronized (lastOperationTSLock) {
            lastOperationTS = now;
        }
        // } else {
        // LOG.info("Mellanox RDMA-accelerated DFSOutputStream constructed successfully.");
    }
}

---

From source file:com.mellanox.r4h.DFSOutputStream.java

License:Apache License

/** Construct a new output stream for creating a file. */
private DFSOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat, EnumSet<CreateFlag> flag,
        Progressable progress, DataChecksum checksum, String[] favoredNodes) throws IOException {
    this(dfsClient, src, progress, stat, checksum);
    this.shouldSyncBlock = flag.contains(CreateFlag.SYNC_BLOCK);

    computePacketChunkSize(dfsClient.getConf().getWritePacketSize(), bytesPerChecksum);

    streamer = new DataStreamer(stat, null);
    if (favoredNodes != null && favoredNodes.length != 0) {
        streamer.setFavoredNodes(favoredNodes);
    }// w w  w. j a v a2  s.  com
}

---

From source file:com.mellanox.r4h.DFSOutputStream.java

License:Apache License

/** Construct a new output stream for append. */
private DFSOutputStream(DFSClient dfsClient, String src, EnumSet<CreateFlag> flags, Progressable progress,
        LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum) throws IOException {
    this(dfsClient, src, progress, stat, checksum);
    initialFileSize = stat.getLen(); // length of file when opened

    this.shouldSyncBlock = flags.contains(CreateFlag.SYNC_BLOCK);
    boolean toNewBlock = flags.contains(CreateFlag.NEW_BLOCK);

    // The last partial block of the file has to be filled.
    if (!toNewBlock && lastBlock != null) {
        // indicate that we are appending to an existing block
        bytesCurBlock = lastBlock.getBlockSize();
        streamer = new DataStreamer(lastBlock, stat, bytesPerChecksum);
    } else {/*  w  w  w. j  a  v a2 s .  c o  m*/
        computePacketChunkSize(dfsClient.getConf().getWritePacketSize(), bytesPerChecksum);
        streamer = new DataStreamer(stat, lastBlock != null ? lastBlock.getBlock() : null);
    }
    this.fileEncryptionInfo = stat.getFileEncryptionInfo();
}

---