List of usage examples for java.net Socket setSendBufferSize
public synchronized void setSendBufferSize(int size) throws SocketException
From source file:com.mirth.connect.connectors.tcp.TcpDispatcher.java
@Override public Response send(ConnectorProperties connectorProperties, ConnectorMessage message) { TcpDispatcherProperties tcpDispatcherProperties = (TcpDispatcherProperties) connectorProperties; Status responseStatus = Status.QUEUED; String responseData = null;//from w ww . j ava 2 s.c o m String responseStatusMessage = null; String responseError = null; boolean validateResponse = false; long dispatcherId = getDispatcherId(); String socketKey = dispatcherId + tcpDispatcherProperties.getRemoteAddress() + tcpDispatcherProperties.getRemotePort(); if (tcpDispatcherProperties.isOverrideLocalBinding()) { socketKey += tcpDispatcherProperties.getLocalAddress() + tcpDispatcherProperties.getLocalPort(); } Socket socket = null; Thread timeoutThread = null; try { // Do some validation first to avoid unnecessarily creating sockets if (StringUtils.isBlank(tcpDispatcherProperties.getRemoteAddress())) { throw new Exception("Remote address is blank."); } else if (NumberUtils.toInt(tcpDispatcherProperties.getRemotePort()) <= 0) { throw new Exception("Remote port is invalid."); } socket = connectedSockets.get(socketKey); timeoutThread = timeoutThreads.get(socketKey); // If keep connection open is true, then interrupt the thread so it won't close the socket if (tcpDispatcherProperties.isKeepConnectionOpen() && timeoutThread != null) { disposeThreadQuietly(socketKey); } // Initialize a new socket if our current one is invalid, the remote side has closed, or keep connection open is false if (!tcpDispatcherProperties.isKeepConnectionOpen() || socket == null || socket.isClosed() || (tcpDispatcherProperties.isCheckRemoteHost() && socket instanceof StateAwareSocketInterface && ((StateAwareSocketInterface) socket).remoteSideHasClosed())) { closeSocketQuietly(socketKey); logger.debug("Creating new socket (" + connectorProperties.getName() + " \"" + getDestinationName() + "\" on channel " + getChannelId() + ")."); String info = "Trying to connect on " + tcpDispatcherProperties.getRemoteAddress() + ":" + tcpDispatcherProperties.getRemotePort() + "..."; eventController.dispatchEvent(new ConnectionStatusEvent(getChannelId(), getMetaDataId(), getDestinationName(), ConnectionStatusEventType.CONNECTING, info)); if (tcpDispatcherProperties.isOverrideLocalBinding()) { socket = SocketUtil.createSocket(configuration, tcpDispatcherProperties.getLocalAddress(), NumberUtils.toInt(tcpDispatcherProperties.getLocalPort())); } else { socket = SocketUtil.createSocket(configuration); } ThreadUtils.checkInterruptedStatus(); connectedSockets.put(socketKey, socket); SocketUtil.connectSocket(socket, tcpDispatcherProperties.getRemoteAddress(), NumberUtils.toInt(tcpDispatcherProperties.getRemotePort()), responseTimeout); socket.setReuseAddress(true); socket.setReceiveBufferSize(bufferSize); socket.setSendBufferSize(bufferSize); socket.setSoTimeout(responseTimeout); socket.setKeepAlive(tcpDispatcherProperties.isKeepConnectionOpen()); eventController.dispatchEvent(new ConnectorCountEvent(getChannelId(), getMetaDataId(), getDestinationName(), ConnectionStatusEventType.CONNECTED, SocketUtil.getLocalAddress(socket) + " -> " + SocketUtil.getInetAddress(socket), true)); } ThreadUtils.checkInterruptedStatus(); // Send the message eventController.dispatchEvent(new ConnectionStatusEvent(getChannelId(), getMetaDataId(), getDestinationName(), ConnectionStatusEventType.SENDING, SocketUtil.getLocalAddress(socket) + " -> " + SocketUtil.getInetAddress(socket))); BufferedOutputStream bos = new BufferedOutputStream(socket.getOutputStream(), bufferSize); BatchStreamReader batchStreamReader = new DefaultBatchStreamReader(socket.getInputStream()); StreamHandler streamHandler = transmissionModeProvider.getStreamHandler(socket.getInputStream(), bos, batchStreamReader, tcpDispatcherProperties.getTransmissionModeProperties()); streamHandler.write(getTemplateBytes(tcpDispatcherProperties, message)); bos.flush(); if (!tcpDispatcherProperties.isIgnoreResponse()) { ThreadUtils.checkInterruptedStatus(); // Attempt to get the response from the remote endpoint try { String info = "Waiting for response from " + SocketUtil.getInetAddress(socket) + " (Timeout: " + tcpDispatcherProperties.getResponseTimeout() + " ms)... "; eventController.dispatchEvent(new ConnectionStatusEvent(getChannelId(), getMetaDataId(), getDestinationName(), ConnectionStatusEventType.WAITING_FOR_RESPONSE, info)); byte[] responseBytes = streamHandler.read(); if (responseBytes != null) { responseData = new String(responseBytes, CharsetUtils.getEncoding(tcpDispatcherProperties.getCharsetEncoding())); responseStatusMessage = "Message successfully sent."; } else { responseStatusMessage = "Message successfully sent, but no response received."; } streamHandler.commit(true); responseStatus = Status.SENT; // We only want to validate the response if we were able to retrieve it successfully validateResponse = tcpDispatcherProperties.getDestinationConnectorProperties() .isValidateResponse(); } catch (IOException e) { // An exception occurred while retrieving the response if (e instanceof SocketTimeoutException || e.getCause() != null && e.getCause() instanceof SocketTimeoutException) { responseStatusMessage = "Timeout waiting for response"; if (!tcpDispatcherProperties.isQueueOnResponseTimeout()) { responseStatus = Status.ERROR; } } else { responseStatusMessage = "Error receiving response"; } responseError = ErrorMessageBuilder.buildErrorMessage(connectorProperties.getName(), responseStatusMessage + ": " + e.getMessage(), e); logger.warn(responseStatusMessage + " (" + connectorProperties.getName() + " \"" + getDestinationName() + "\" on channel " + getChannelId() + ").", e); eventController.dispatchEvent(new ErrorEvent(getChannelId(), getMetaDataId(), message.getMessageId(), ErrorEventType.DESTINATION_CONNECTOR, getDestinationName(), connectorProperties.getName(), responseStatusMessage + ".", e)); eventController.dispatchEvent(new ConnectionStatusEvent(getChannelId(), getMetaDataId(), getDestinationName(), ConnectionStatusEventType.FAILURE, responseStatusMessage + " from " + SocketUtil.getInetAddress(socket))); closeSocketQuietly(socketKey); } } else { try { // MIRTH-2980: Since we're ignoring responses, flush out the socket's input stream so it doesn't continually grow socket.getInputStream().skip(socket.getInputStream().available()); } catch (IOException e) { logger.warn("Error flushing socket input stream.", e); } // We're ignoring the response, so always return a successful response responseStatus = Status.SENT; responseStatusMessage = "Message successfully sent."; } if (tcpDispatcherProperties.isKeepConnectionOpen() && (getCurrentState() == DeployedState.STARTED || getCurrentState() == DeployedState.STARTING)) { if (sendTimeout > 0) { // Close the connection after the send timeout has been reached startThread(socketKey); } } else { // If keep connection open is false, then close the socket right now closeSocketQuietly(socketKey); } } catch (Throwable t) { disposeThreadQuietly(socketKey); closeSocketQuietly(socketKey); String monitorMessage = "Error sending message (" + SocketUtil.getLocalAddress(socket) + " -> " + SocketUtil.getInetAddress(socket) + "): " + t.getMessage(); eventController.dispatchEvent(new ConnectionStatusEvent(getChannelId(), getMetaDataId(), getDestinationName(), ConnectionStatusEventType.FAILURE, monitorMessage)); // If an exception occurred then close the socket, even if keep connection open is true responseStatusMessage = t.getClass().getSimpleName() + ": " + t.getMessage(); responseError = ErrorMessageBuilder.buildErrorMessage(connectorProperties.getName(), t.getMessage(), t); String logMessage = "Error sending message via TCP (" + connectorProperties.getName() + " \"" + getDestinationName() + "\" on channel " + getChannelId() + ")."; if (t instanceof InterruptedException) { Thread.currentThread().interrupt(); } else if (t instanceof ConnectException || t.getCause() != null && t.getCause() instanceof ConnectException) { if (isQueueEnabled()) { logger.warn(logMessage, t); } else { logger.error(logMessage, t); } } else { logger.debug(logMessage, t); } eventController.dispatchEvent(new ErrorEvent(getChannelId(), getMetaDataId(), message.getMessageId(), ErrorEventType.DESTINATION_CONNECTOR, getDestinationName(), connectorProperties.getName(), "Error sending message via TCP.", t)); } finally { eventController.dispatchEvent(new ConnectorCountEvent(getChannelId(), getMetaDataId(), getDestinationName(), ConnectionStatusEventType.IDLE, SocketUtil.getLocalAddress(socket) + " -> " + SocketUtil.getInetAddress(socket), (Boolean) null)); } return new Response(responseStatus, responseData, responseStatusMessage, responseError, validateResponse); }
From source file:com.atomicleopard.thundr.ftp.commons.FTPClient.java
/** * Establishes a data connection with the FTP server, returning * a Socket for the connection if successful. If a restart * offset has been set with {@link #setRestartOffset(long)}, * a REST command is issued to the server with the offset as * an argument before establishing the data connection. Active * mode connections also cause a local PORT command to be issued. * <p>//from w ww . j a v a 2 s. co m * @param command The text representation of the FTP command to send. * @param arg The arguments to the FTP command. If this parameter is * set to null, then the command is sent with no argument. * @return A Socket corresponding to the established data connection. * Null is returned if an FTP protocol error is reported at * any point during the establishment and initialization of * the connection. * @exception IOException If an I/O error occurs while either sending a * command to the server or receiving a reply from the server. * @since 3.1 */ protected Socket _openDataConnection_(String command, String arg) throws IOException { if (__dataConnectionMode != ACTIVE_LOCAL_DATA_CONNECTION_MODE && __dataConnectionMode != PASSIVE_LOCAL_DATA_CONNECTION_MODE) { return null; } final boolean isInet6Address = getRemoteAddress() instanceof Inet6Address; Socket socket; if (__dataConnectionMode == ACTIVE_LOCAL_DATA_CONNECTION_MODE) { // if no activePortRange was set (correctly) -> getActivePort() = 0 // -> new ServerSocket(0) -> bind to any free local port ServerSocket server = _serverSocketFactory_.createServerSocket(getActivePort(), 1, getHostAddress()); try { // Try EPRT only if remote server is over IPv6, if not use PORT, // because EPRT has no advantage over PORT on IPv4. // It could even have the disadvantage, // that EPRT will make the data connection fail, because // today's intelligent NAT Firewalls are able to // substitute IP addresses in the PORT command, // but might not be able to recognize the EPRT command. if (isInet6Address) { if (!FTPReply.isPositiveCompletion(eprt(getReportHostAddress(), server.getLocalPort()))) { return null; } } else { if (!FTPReply.isPositiveCompletion(port(getReportHostAddress(), server.getLocalPort()))) { return null; } } if ((__restartOffset > 0) && !restart(__restartOffset)) { return null; } if (!FTPReply.isPositivePreliminary(sendCommand(command, arg))) { return null; } // For now, let's just use the data timeout value for waiting for // the data connection. It may be desirable to let this be a // separately configurable value. In any case, we really want // to allow preventing the accept from blocking indefinitely. if (__dataTimeout >= 0) { server.setSoTimeout(__dataTimeout); } socket = server.accept(); // Ensure the timeout is set before any commands are issued on the new socket if (__dataTimeout >= 0) { socket.setSoTimeout(__dataTimeout); } if (__receiveDataSocketBufferSize > 0) { socket.setReceiveBufferSize(__receiveDataSocketBufferSize); } if (__sendDataSocketBufferSize > 0) { socket.setSendBufferSize(__sendDataSocketBufferSize); } } finally { server.close(); } } else { // We must be in PASSIVE_LOCAL_DATA_CONNECTION_MODE // Try EPSV command first on IPv6 - and IPv4 if enabled. // When using IPv4 with NAT it has the advantage // to work with more rare configurations. // E.g. if FTP server has a static PASV address (external network) // and the client is coming from another internal network. // In that case the data connection after PASV command would fail, // while EPSV would make the client succeed by taking just the port. boolean attemptEPSV = isUseEPSVwithIPv4() || isInet6Address; if (attemptEPSV && epsv() == FTPReply.ENTERING_EPSV_MODE) { _parseExtendedPassiveModeReply(_replyLines.get(0)); } else { if (isInet6Address) { return null; // Must use EPSV for IPV6 } // If EPSV failed on IPV4, revert to PASV if (pasv() != FTPReply.ENTERING_PASSIVE_MODE) { return null; } _parsePassiveModeReply(_replyLines.get(0)); } socket = _socketFactory_.createSocket(); if (__receiveDataSocketBufferSize > 0) { socket.setReceiveBufferSize(__receiveDataSocketBufferSize); } if (__sendDataSocketBufferSize > 0) { socket.setSendBufferSize(__sendDataSocketBufferSize); } if (__passiveLocalHost != null) { socket.bind(new InetSocketAddress(__passiveLocalHost, 0)); } // For now, let's just use the data timeout value for waiting for // the data connection. It may be desirable to let this be a // separately configurable value. In any case, we really want // to allow preventing the accept from blocking indefinitely. if (__dataTimeout >= 0) { socket.setSoTimeout(__dataTimeout); } socket.connect(new InetSocketAddress(__passiveHost, __passivePort), connectTimeout); if ((__restartOffset > 0) && !restart(__restartOffset)) { socket.close(); return null; } if (!FTPReply.isPositivePreliminary(sendCommand(command, arg))) { socket.close(); return null; } } if (__remoteVerificationEnabled && !verifyRemote(socket)) { socket.close(); throw new IOException("Host attempting data connection " + socket.getInetAddress().getHostAddress() + " is not same as server " + getRemoteAddress().getHostAddress()); } return socket; }
From source file:org.apache.hadoop.hdfs.DataStreamer.java
/** * Create a socket for a write pipeline/*from w w w .ja v a2 s .c om*/ * * @param first the first datanode * @param length the pipeline length * @param client client * @return the socket connected to the first datanode */ static Socket createSocketForPipeline(final DatanodeInfo first, final int length, final DFSClient client) throws IOException { final String dnAddr = first.getXferAddr(client.getConf().connectToDnViaHostname); if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("Connecting to datanode " + dnAddr); } final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr); SocketFactory socketFactory = new StandardSocketFactory(); final Socket sock = socketFactory.createSocket(); final int timeout = client.getDatanodeReadTimeout(length); NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), client.getConf().socketTimeout); sock.setSoTimeout(timeout); sock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE); if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("Send buf size " + sock.getSendBufferSize()); } return sock; }
From source file:org.apache.hadoop.hdfs.DataStreamer.java
/** * Create a socket for a write pipeline/*from www. j ava 2 s . c o m*/ * * @param first the first datanode * @param length the pipeline length * @param client client * @return the socket connected to the first datanode */ static Socket createSocketForPipeline(final DatanodeInfo first, final int length, final DFSClient client) throws IOException { final DfsClientConf conf = client.getConf(); final String dnAddr = first.getXferAddr(conf.isConnectToDnViaHostname()); if (LOG.isDebugEnabled()) { LOG.debug("Connecting to datanode " + dnAddr); } final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr); final Socket sock = client.socketFactory.createSocket(); final int timeout = client.getDatanodeReadTimeout(length); NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), conf.getSocketTimeout()); sock.setSoTimeout(timeout); sock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE); if (LOG.isDebugEnabled()) { LOG.debug("Send buf size " + sock.getSendBufferSize()); } return sock; }
From source file:org.apache.hadoop.hdfs.server.datanode.CachingDataXceiver.java
@Override public void writeBlock(final ExtendedBlock block, final Token<BlockTokenIdentifier> blockToken, final String clientname, final DatanodeInfo[] targets, final DatanodeInfo srcDataNode, final BlockConstructionStage stage, final int pipelineSize, final long minBytesRcvd, final long maxBytesRcvd, final long latestGenerationStamp, DataChecksum requestedChecksum) throws IOException { previousOpClientName = clientname;/*from www .j a v a 2 s.c om*/ updateCurrentThreadName("Receiving block " + block); final boolean isDatanode = clientname.length() == 0; final boolean isClient = !isDatanode; final boolean isTransfer = stage == BlockConstructionStage.TRANSFER_RBW || stage == BlockConstructionStage.TRANSFER_FINALIZED; // check single target for transfer-RBW/Finalized if (isTransfer && targets.length > 0) { throw new IOException(stage + " does not support multiple targets " + Arrays.asList(targets)); } if (LOG.isDebugEnabled()) { LOG.debug("opWriteBlock: stage=" + stage + ", clientname=" + clientname + "\n block =" + block + ", newGs=" + latestGenerationStamp + ", bytesRcvd=[" + minBytesRcvd + ", " + maxBytesRcvd + "]" + "\n targets=" + Arrays.asList(targets) + "; pipelineSize=" + pipelineSize + ", srcDataNode=" + srcDataNode); LOG.debug("isDatanode=" + isDatanode + ", isClient=" + isClient + ", isTransfer=" + isTransfer); LOG.debug("writeBlock receive buf size " + s.getReceiveBufferSize() + " tcp no delay " + s.getTcpNoDelay()); } // We later mutate block's generation stamp and length, but we need to // forward the original version of the block to downstream mirrors, so // make a copy here. final ExtendedBlock originalBlock = new ExtendedBlock(block); block.setNumBytes(dataXceiverServer.estimateBlockSize); LOG.info("Receiving block " + block + " src: " + remoteAddress + " dest: " + localAddress); // reply to upstream datanode or client final DataOutputStream replyOut = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(s, dnConf.socketWriteTimeout), HdfsConstants.SMALL_BUFFER_SIZE)); checkAccess(replyOut, isClient, block, blockToken, Op.WRITE_BLOCK, BlockTokenSecretManager.AccessMode.WRITE); DataOutputStream mirrorOut = null; // stream to next target DataInputStream mirrorIn = null; // reply from next target Socket mirrorSock = null; // socket to next target BlockReceiver blockReceiver = null; // responsible for data handling String mirrorNode = null; // the name:port of next target String firstBadLink = ""; // first datanode that failed in connection setup Status mirrorInStatus = SUCCESS; try { if (isDatanode || stage != BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { // open a block receiver blockReceiver = new BlockReceiver(block, in, s.getRemoteSocketAddress().toString(), s.getLocalSocketAddress().toString(), stage, latestGenerationStamp, minBytesRcvd, maxBytesRcvd, clientname, srcDataNode, datanode, requestedChecksum); } else { datanode.data.recoverClose(block, latestGenerationStamp, minBytesRcvd); } // // Connect to downstream machine, if appropriate // if (targets.length > 0) { InetSocketAddress mirrorTarget = null; // Connect to backup machine mirrorNode = targets[0].getXferAddr(); mirrorTarget = NetUtils.createSocketAddr(mirrorNode); mirrorSock = datanode.newSocket(); try { int timeoutValue = dnConf.socketTimeout + (HdfsServerConstants.READ_TIMEOUT_EXTENSION * targets.length); int writeTimeout = dnConf.socketWriteTimeout + (HdfsServerConstants.WRITE_TIMEOUT_EXTENSION * targets.length); NetUtils.connect(mirrorSock, mirrorTarget, timeoutValue); mirrorSock.setSoTimeout(timeoutValue); mirrorSock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE); mirrorOut = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(mirrorSock, writeTimeout), HdfsConstants.SMALL_BUFFER_SIZE)); mirrorIn = new DataInputStream(NetUtils.getInputStream(mirrorSock)); new Sender(mirrorOut).writeBlock(originalBlock, blockToken, clientname, targets, srcDataNode, stage, pipelineSize, minBytesRcvd, maxBytesRcvd, latestGenerationStamp, requestedChecksum); mirrorOut.flush(); // read connect ack (only for clients, not for replication req) if (isClient) { BlockOpResponseProto connectAck = BlockOpResponseProto .parseFrom(HdfsProtoUtil.vintPrefixed(mirrorIn)); mirrorInStatus = connectAck.getStatus(); firstBadLink = connectAck.getFirstBadLink(); if (LOG.isDebugEnabled() || mirrorInStatus != SUCCESS) { LOG.info("Datanode " + targets.length + " got response for connect ack " + " from downstream datanode with firstbadlink as " + firstBadLink); } } } catch (IOException e) { if (isClient) { BlockOpResponseProto.newBuilder().setStatus(ERROR).setFirstBadLink(mirrorNode).build() .writeDelimitedTo(replyOut); replyOut.flush(); } IOUtils.closeStream(mirrorOut); mirrorOut = null; IOUtils.closeStream(mirrorIn); mirrorIn = null; IOUtils.closeSocket(mirrorSock); mirrorSock = null; if (isClient) { LOG.error(datanode + ":Exception transfering block " + block + " to mirror " + mirrorNode + ": " + e); throw e; } else { LOG.info(datanode + ":Exception transfering block " + block + " to mirror " + mirrorNode + ". continuing without the mirror.", e); } } } // send connect-ack to source for clients and not transfer-RBW/Finalized if (isClient && !isTransfer) { if (LOG.isDebugEnabled() || mirrorInStatus != SUCCESS) { LOG.info("Datanode " + targets.length + " forwarding connect ack to upstream firstbadlink is " + firstBadLink); } BlockOpResponseProto.newBuilder().setStatus(mirrorInStatus).setFirstBadLink(firstBadLink).build() .writeDelimitedTo(replyOut); replyOut.flush(); } // receive the block and mirror to the next target if (blockReceiver != null) { String mirrorAddr = (mirrorSock == null) ? null : mirrorNode; blockReceiver.receiveBlock(mirrorOut, mirrorIn, replyOut, mirrorAddr, null, targets); // send close-ack for transfer-RBW/Finalized if (isTransfer) { if (LOG.isTraceEnabled()) { LOG.trace("TRANSFER: send close-ack"); } writeResponse(SUCCESS, null, replyOut); } } // update its generation stamp if (isClient && stage == BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { block.setGenerationStamp(latestGenerationStamp); block.setNumBytes(minBytesRcvd); } // if this write is for a replication request or recovering // a failed close for client, then confirm block. For other client-writes, // the block is finalized in the PacketResponder. if (isDatanode || stage == BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { datanode.closeBlock(block, DataNode.EMPTY_DEL_HINT); LOG.info("Received block " + block + " src: " + remoteAddress + " dest: " + localAddress + " of size " + block.getNumBytes()); } } catch (IOException ioe) { LOG.info("opWriteBlock " + block + " received exception " + ioe); throw ioe; } finally { // close all opened streams IOUtils.closeStream(mirrorOut); IOUtils.closeStream(mirrorIn); IOUtils.closeStream(replyOut); IOUtils.closeSocket(mirrorSock); IOUtils.closeStream(blockReceiver); } // update metrics datanode.metrics.addWriteBlockOp(elapsed()); datanode.metrics.incrWritesFromClient(isLocal); }
From source file:org.apache.hadoop.hdfs.server.datanode.DataWriter.java
/** * Write a block to disk./*from w w w . ja v a 2 s .co m*/ * * @param in The stream to read from * @throws IOException */ private void writeBlock() throws IOException { DatanodeInfo srcDataNode = null; LOG.debug("writeBlock receive buf size " + s.getReceiveBufferSize() + " tcp no delay " + s.getTcpNoDelay()); // // Read in the header // long startTime = System.currentTimeMillis(); int namespaceid = in.readInt(); Block block = new Block(in.readLong(), dataXceiverServer.estimateBlockSize, in.readLong()); LOG.info("Receiving block " + block + " src: " + remoteAddress + " dest: " + localAddress); int pipelineSize = in.readInt(); // num of datanodes in entire pipeline boolean isRecovery = in.readBoolean(); // is this part of recovery? String client = Text.readString(in); // working on behalf of this client boolean hasSrcDataNode = in.readBoolean(); // is src node info present if (hasSrcDataNode) { srcDataNode = new DatanodeInfo(); srcDataNode.readFields(in); } int numTargets = in.readInt(); if (numTargets < 0) { throw new IOException("Mislabelled incoming datastream."); } DatanodeInfo targets[] = new DatanodeInfo[numTargets]; for (int i = 0; i < targets.length; i++) { DatanodeInfo tmp = new DatanodeInfo(); tmp.readFields(in); targets[i] = tmp; } DataOutputStream mirrorOut = null; // stream to next target DataInputStream mirrorIn = null; // reply from next target DataOutputStream replyOut = null; // stream to prev target Socket mirrorSock = null; // socket to next target BlockReceiver blockReceiver = null; // responsible for data handling String mirrorNode = null; // the name:port of next target String firstBadLink = ""; // first datanode that failed in connection setup updateCurrentThreadName("receiving block " + block + " client=" + client); try { // open a block receiver and check if the block does not exist blockReceiver = new BlockReceiver(namespaceid, block, in, s.getRemoteSocketAddress().toString(), s.getLocalSocketAddress().toString(), isRecovery, client, srcDataNode, datanode); // get a connection back to the previous target replyOut = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(s, datanode.socketWriteTimeout), SMALL_BUFFER_SIZE)); // // Open network conn to backup machine, if // appropriate // if (targets.length > 0) { InetSocketAddress mirrorTarget = null; // Connect to backup machine mirrorNode = targets[0].getName(); mirrorTarget = NetUtils.createSocketAddr(mirrorNode); mirrorSock = datanode.newSocket(); try { int timeoutValue = datanode.socketTimeout + (datanode.socketReadExtentionTimeout * numTargets); int writeTimeout = datanode.socketWriteTimeout + (datanode.socketWriteExtentionTimeout * numTargets); NetUtils.connect(mirrorSock, mirrorTarget, timeoutValue); mirrorSock.setSoTimeout(timeoutValue); mirrorSock.setSendBufferSize(DEFAULT_DATA_SOCKET_SIZE); mirrorOut = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(mirrorSock, writeTimeout), SMALL_BUFFER_SIZE)); mirrorIn = new DataInputStream(NetUtils.getInputStream(mirrorSock)); // Write header: Copied from DFSClient.java! mirrorOut.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION); mirrorOut.write(DataTransferProtocol.OP_WRITE_BLOCK); mirrorOut.writeInt(namespaceid); mirrorOut.writeLong(block.getBlockId()); mirrorOut.writeLong(block.getGenerationStamp()); mirrorOut.writeInt(pipelineSize); mirrorOut.writeBoolean(isRecovery); Text.writeString(mirrorOut, client); mirrorOut.writeBoolean(hasSrcDataNode); if (hasSrcDataNode) { // pass src node information srcDataNode.write(mirrorOut); } mirrorOut.writeInt(targets.length - 1); for (int i = 1; i < targets.length; i++) { targets[i].write(mirrorOut); } blockReceiver.writeChecksumHeader(mirrorOut); mirrorOut.flush(); // read connect ack (only for clients, not for replication req) if (client.length() != 0) { firstBadLink = Text.readString(mirrorIn); if (LOG.isDebugEnabled() || firstBadLink.length() > 0) { LOG.info("Datanode " + targets.length + " got response for connect ack " + " from downstream datanode with firstbadlink as " + firstBadLink); } } } catch (IOException e) { if (client.length() != 0) { Text.writeString(replyOut, mirrorNode); replyOut.flush(); } IOUtils.closeStream(mirrorOut); mirrorOut = null; IOUtils.closeStream(mirrorIn); mirrorIn = null; IOUtils.closeSocket(mirrorSock); mirrorSock = null; if (client.length() > 0) { throw e; } else { LOG.info(datanode.getDatanodeInfo() + ":Exception transfering block " + block + " to mirror " + mirrorNode + ". continuing without the mirror.\n" + StringUtils.stringifyException(e)); } } } // send connect ack back to source (only for clients) if (client.length() != 0) { if (LOG.isDebugEnabled() || firstBadLink.length() > 0) { LOG.info("Datanode " + targets.length + " forwarding connect ack to upstream firstbadlink is " + firstBadLink); } Text.writeString(replyOut, firstBadLink); replyOut.flush(); } // receive the block and mirror to the next target String mirrorAddr = (mirrorSock == null) ? null : mirrorNode; long totalReceiveSize = blockReceiver.receiveBlock(mirrorOut, mirrorIn, replyOut, mirrorAddr, null, targets.length); // if this write is for a replication request (and not // from a client), then confirm block. For client-writes, // the block is finalized in the PacketResponder. if (client.length() == 0) { datanode.notifyNamenodeReceivedBlock(namespaceid, block, null); LOG.info("Received block " + block + " src: " + remoteAddress + " dest: " + localAddress + " of size " + block.getNumBytes()); } else { // Log the fact that the block has been received by this datanode and // has been written to the local disk on this datanode. LOG.info("Received Block " + block + " src: " + remoteAddress + " dest: " + localAddress + " of size " + block.getNumBytes() + " and written to local disk"); } if (datanode.blockScanner != null) { datanode.blockScanner.addBlock(namespaceid, block); } long writeDuration = System.currentTimeMillis() - startTime; datanode.myMetrics.bytesWrittenLatency.inc(writeDuration); if (totalReceiveSize > KB_RIGHT_SHIFT_MIN) { datanode.myMetrics.bytesWrittenRate.inc((int) (totalReceiveSize >> KB_RIGHT_SHIFT_BITS), writeDuration); } } catch (IOException ioe) { LOG.info("writeBlock " + block + " received exception " + ioe); throw ioe; } finally { // close all opened streams IOUtils.closeStream(mirrorOut); IOUtils.closeStream(mirrorIn); IOUtils.closeStream(replyOut); IOUtils.closeSocket(mirrorSock); IOUtils.closeStream(blockReceiver); } }
From source file:org.apache.hadoop.hdfs.server.datanode.DataXceiver.java
/** * Write a block to disk.// ww w . jav a2 s . co m * * @param in The stream to read from * @throws IOException */ private void writeBlock(DataInputStream in) throws IOException { DatanodeInfo srcDataNode = null; LOG.debug("writeBlock receive buf size " + s.getReceiveBufferSize() + " tcp no delay " + s.getTcpNoDelay()); // // Read in the header // Block block = new Block(in.readLong(), dataXceiverServer.estimateBlockSize, in.readLong()); LOG.info("Receiving block " + block + " src: " + remoteAddress + " dest: " + localAddress); int pipelineSize = in.readInt(); // num of datanodes in entire pipeline boolean isRecovery = in.readBoolean(); // is this part of recovery? String client = Text.readString(in); // working on behalf of this client boolean hasSrcDataNode = in.readBoolean(); // is src node info present if (hasSrcDataNode) { srcDataNode = new DatanodeInfo(); srcDataNode.readFields(in); } int numTargets = in.readInt(); if (numTargets < 0) { throw new IOException("Mislabelled incoming datastream."); } DatanodeInfo targets[] = new DatanodeInfo[numTargets]; for (int i = 0; i < targets.length; i++) { DatanodeInfo tmp = new DatanodeInfo(); tmp.readFields(in); targets[i] = tmp; } Token<BlockTokenIdentifier> accessToken = new Token<BlockTokenIdentifier>(); accessToken.readFields(in); DataOutputStream replyOut = null; // stream to prev target replyOut = new DataOutputStream(NetUtils.getOutputStream(s, datanode.socketWriteTimeout)); if (datanode.isBlockTokenEnabled) { try { datanode.blockTokenSecretManager.checkAccess(accessToken, null, block, BlockTokenSecretManager.AccessMode.WRITE); } catch (InvalidToken e) { try { if (client.length() != 0) { replyOut.writeShort((short) DataTransferProtocol.OP_STATUS_ERROR_ACCESS_TOKEN); Text.writeString(replyOut, datanode.dnRegistration.getName()); replyOut.flush(); } throw new IOException("Access token verification failed, for client " + remoteAddress + " for OP_WRITE_BLOCK for block " + block); } finally { IOUtils.closeStream(replyOut); } } } DataOutputStream mirrorOut = null; // stream to next target DataInputStream mirrorIn = null; // reply from next target Socket mirrorSock = null; // socket to next target BlockReceiver blockReceiver = null; // responsible for data handling String mirrorNode = null; // the name:port of next target String firstBadLink = ""; // first datanode that failed in connection setup short mirrorInStatus = (short) DataTransferProtocol.OP_STATUS_SUCCESS; try { // open a block receiver and check if the block does not exist blockReceiver = new BlockReceiver(block, in, s.getRemoteSocketAddress().toString(), s.getLocalSocketAddress().toString(), isRecovery, client, srcDataNode, datanode); // // Open network conn to backup machine, if // appropriate // if (targets.length > 0) { InetSocketAddress mirrorTarget = null; // Connect to backup machine mirrorNode = targets[0].getName(); mirrorTarget = NetUtils.createSocketAddr(mirrorNode); mirrorSock = datanode.newSocket(); try { int timeoutValue = datanode.socketTimeout + (HdfsConstants.READ_TIMEOUT_EXTENSION * numTargets); int writeTimeout = datanode.socketWriteTimeout + (HdfsConstants.WRITE_TIMEOUT_EXTENSION * numTargets); NetUtils.connect(mirrorSock, mirrorTarget, timeoutValue); mirrorSock.setSoTimeout(timeoutValue); mirrorSock.setSendBufferSize(DEFAULT_DATA_SOCKET_SIZE); mirrorOut = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(mirrorSock, writeTimeout), SMALL_BUFFER_SIZE)); mirrorIn = new DataInputStream(NetUtils.getInputStream(mirrorSock)); // Write header: Copied from DFSClient.java! mirrorOut.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION); mirrorOut.write(DataTransferProtocol.OP_WRITE_BLOCK); mirrorOut.writeLong(block.getBlockId()); mirrorOut.writeLong(block.getGenerationStamp()); mirrorOut.writeInt(pipelineSize); mirrorOut.writeBoolean(isRecovery); Text.writeString(mirrorOut, client); mirrorOut.writeBoolean(hasSrcDataNode); if (hasSrcDataNode) { // pass src node information srcDataNode.write(mirrorOut); } mirrorOut.writeInt(targets.length - 1); for (int i = 1; i < targets.length; i++) { targets[i].write(mirrorOut); } accessToken.write(mirrorOut); blockReceiver.writeChecksumHeader(mirrorOut); mirrorOut.flush(); // read connect ack (only for clients, not for replication req) if (client.length() != 0) { mirrorInStatus = mirrorIn.readShort(); firstBadLink = Text.readString(mirrorIn); if (LOG.isDebugEnabled() || mirrorInStatus != DataTransferProtocol.OP_STATUS_SUCCESS) { LOG.info("Datanode " + targets.length + " got response for connect ack " + " from downstream datanode with firstbadlink as " + firstBadLink); } } } catch (IOException e) { if (client.length() != 0) { replyOut.writeShort((short) DataTransferProtocol.OP_STATUS_ERROR); Text.writeString(replyOut, mirrorNode); replyOut.flush(); } IOUtils.closeStream(mirrorOut); mirrorOut = null; IOUtils.closeStream(mirrorIn); mirrorIn = null; IOUtils.closeSocket(mirrorSock); mirrorSock = null; if (client.length() > 0) { throw e; } else { LOG.info(datanode.dnRegistration + ":Exception transfering block " + block + " to mirror " + mirrorNode + ". continuing without the mirror.\n" + StringUtils.stringifyException(e)); } } } // send connect ack back to source (only for clients) if (client.length() != 0) { if (LOG.isDebugEnabled() || mirrorInStatus != DataTransferProtocol.OP_STATUS_SUCCESS) { LOG.info("Datanode " + targets.length + " forwarding connect ack to upstream firstbadlink is " + firstBadLink); } replyOut.writeShort(mirrorInStatus); Text.writeString(replyOut, firstBadLink); replyOut.flush(); } // receive the block and mirror to the next target String mirrorAddr = (mirrorSock == null) ? null : mirrorNode; blockReceiver.receiveBlock(mirrorOut, mirrorIn, replyOut, mirrorAddr, null, targets.length); // if this write is for a replication request (and not // from a client), then confirm block. For client-writes, // the block is finalized in the PacketResponder. if (client.length() == 0) { datanode.notifyNamenodeReceivedBlock(block, DataNode.EMPTY_DEL_HINT); LOG.info("Received block " + block + " src: " + remoteAddress + " dest: " + localAddress + " of size " + block.getNumBytes()); } if (datanode.blockScanner != null) { datanode.blockScanner.addBlock(block); } } catch (IOException ioe) { LOG.info("writeBlock " + block + " received exception " + ioe); throw ioe; } finally { // close all opened streams IOUtils.closeStream(mirrorOut); IOUtils.closeStream(mirrorIn); IOUtils.closeStream(replyOut); IOUtils.closeSocket(mirrorSock); IOUtils.closeStream(blockReceiver); } }
From source file:org.apache.hadoop.hdfs.server.datanode.DWRRDataXceiver.java
@Override public void writeBlock(final ExtendedBlock block, final Token<BlockTokenIdentifier> blockToken, final String clientname, final DatanodeInfo[] targets, final DatanodeInfo srcDataNode, final BlockConstructionStage stage, final int pipelineSize, final long minBytesRcvd, final long maxBytesRcvd, final long latestGenerationStamp, DataChecksum requestedChecksum, CachingStrategy cachingStrategy) throws IOException { previousOpClientName = clientname;/* w w w . j a v a2 s.c o m*/ updateCurrentThreadName("Receiving block " + block); isDatanode = clientname.length() == 0; isClient = !isDatanode; isTransfer = stage == BlockConstructionStage.TRANSFER_RBW || stage == BlockConstructionStage.TRANSFER_FINALIZED; this.stage = stage; this.block = block; this.latestGenerationStamp = latestGenerationStamp; this.minBytesRcvd = minBytesRcvd; // check single target for transfer-RBW/Finalized if (isTransfer && targets.length > 0) { throw new IOException(stage + " does not support multiple targets " + Arrays.asList(targets)); } if (LOG.isDebugEnabled()) { LOG.debug("opWriteBlock: stage=" + stage + ", clientname=" + clientname + "\n block =" + block + ", newGs=" + latestGenerationStamp + ", bytesRcvd=[" + minBytesRcvd + ", " + maxBytesRcvd + "]" + "\n targets=" + Arrays.asList(targets) + "; pipelineSize=" + pipelineSize + ", srcDataNode=" + srcDataNode); LOG.debug("isDatanode=" + isDatanode + ", isClient=" + isClient + ", isTransfer=" + isTransfer); LOG.debug("writeBlock receive buf size " + peer.getReceiveBufferSize() + " tcp no delay " + peer.getTcpNoDelay()); } // We later mutate block's generation stamp and length, but we need to // forward the original version of the block to downstream mirrors, so // make a copy here. final ExtendedBlock originalBlock = new ExtendedBlock(block); block.setNumBytes(DataXceiverServer.estimateBlockSize); LOG.info("Receiving " + block + " src: " + remoteAddress + " dest: " + localAddress); // reply to upstream datanode or client replyOut = new DataOutputStream( new BufferedOutputStream(getOutputStream(), HdfsConstants.SMALL_BUFFER_SIZE)); checkAccess(replyOut, isClient, block, blockToken, Op.WRITE_BLOCK, BlockTokenSecretManager.AccessMode.WRITE); mirrorOut = null; // stream to next target mirrorIn = null; // reply from next target Socket mirrorSock = null; // socket to next target blockReceiver = null; // responsible for data handling String mirrorNode = null; // the name:port of next target String firstBadLink = ""; // first datanode that failed in connection setup Status mirrorInStatus = SUCCESS; try { if (isDatanode || stage != BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { // open a block receiver blockReceiver = new DWRRBlockReceiver(block, in, peer.getRemoteAddressString(), peer.getLocalAddressString(), stage, latestGenerationStamp, minBytesRcvd, maxBytesRcvd, clientname, srcDataNode, datanode, requestedChecksum, cachingStrategy); storageUuid = blockReceiver.getStorageUuid(); } else { storageUuid = datanode.data.recoverClose(block, latestGenerationStamp, minBytesRcvd); } // // Connect to downstream machine, if appropriate // if (targets.length > 0) { InetSocketAddress mirrorTarget = null; // Connect to backup machine mirrorNode = targets[0].getXferAddr(connectToDnViaHostname); if (LOG.isDebugEnabled()) { LOG.debug("Connecting to datanode " + mirrorNode); } mirrorTarget = NetUtils.createSocketAddr(mirrorNode); mirrorSock = datanode.newSocket(); try { int timeoutValue = dnConf.socketTimeout + (HdfsServerConstants.READ_TIMEOUT_EXTENSION * targets.length); int writeTimeout = dnConf.socketWriteTimeout + (HdfsServerConstants.WRITE_TIMEOUT_EXTENSION * targets.length); NetUtils.connect(mirrorSock, mirrorTarget, timeoutValue); mirrorSock.setSoTimeout(timeoutValue); mirrorSock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE); OutputStream unbufMirrorOut = NetUtils.getOutputStream(mirrorSock, writeTimeout); InputStream unbufMirrorIn = NetUtils.getInputStream(mirrorSock); if (dnConf.encryptDataTransfer && !dnConf.trustedChannelResolver.isTrusted(mirrorSock.getInetAddress())) { IOStreamPair encryptedStreams = DataTransferEncryptor.getEncryptedStreams(unbufMirrorOut, unbufMirrorIn, datanode.blockPoolTokenSecretManager .generateDataEncryptionKey(block.getBlockPoolId())); unbufMirrorOut = encryptedStreams.out; unbufMirrorIn = encryptedStreams.in; } mirrorOut = new DataOutputStream( new BufferedOutputStream(unbufMirrorOut, HdfsConstants.SMALL_BUFFER_SIZE)); mirrorIn = new DataInputStream(unbufMirrorIn); new Sender(mirrorOut).writeBlock(originalBlock, blockToken, clientname, targets, srcDataNode, stage, pipelineSize, minBytesRcvd, maxBytesRcvd, latestGenerationStamp, requestedChecksum, cachingStrategy); mirrorOut.flush(); // read connect ack (only for clients, not for replication req) if (isClient) { BlockOpResponseProto connectAck = BlockOpResponseProto .parseFrom(PBHelper.vintPrefixed(mirrorIn)); mirrorInStatus = connectAck.getStatus(); firstBadLink = connectAck.getFirstBadLink(); if (LOG.isDebugEnabled() || mirrorInStatus != SUCCESS) { LOG.info("Datanode " + targets.length + " got response for connect ack " + " from downstream datanode with firstbadlink as " + firstBadLink); } } } catch (IOException e) { if (isClient) { BlockOpResponseProto.newBuilder().setStatus(ERROR) // NB: Unconditionally using the xfer addr w/o hostname .setFirstBadLink(targets[0].getXferAddr()).build().writeDelimitedTo(replyOut); replyOut.flush(); } IOUtils.closeStream(mirrorOut); mirrorOut = null; IOUtils.closeStream(mirrorIn); mirrorIn = null; IOUtils.closeSocket(mirrorSock); mirrorSock = null; if (isClient) { LOG.error(datanode + ":Exception transfering block " + block + " to mirror " + mirrorNode + ": " + e); throw e; } else { LOG.info(datanode + ":Exception transfering " + block + " to mirror " + mirrorNode + "- continuing without the mirror", e); } } } // send connect-ack to source for clients and not transfer-RBW/Finalized if (isClient && !isTransfer) { if (LOG.isDebugEnabled() || mirrorInStatus != SUCCESS) { LOG.info("Datanode " + targets.length + " forwarding connect ack to upstream firstbadlink is " + firstBadLink); } BlockOpResponseProto.newBuilder().setStatus(mirrorInStatus).setFirstBadLink(firstBadLink).build() .writeDelimitedTo(replyOut); replyOut.flush(); } // receive the block and mirror to the next target if (blockReceiver != null) { LOG.info("CAMAMILLA " + classId + " receive the block and mirror to the next target"); // TODO TODO log entra String mirrorAddr = (mirrorSock == null) ? null : mirrorNode; int len = blockReceiver.receiveBlock(mirrorOut, mirrorIn, replyOut, mirrorAddr, null, targets); DWRRRequestObject newReq = new DWRRRequestObject(this, classId, op, len); dwrrmanager.addOp(newReq, classId); LOG.info("CAMAMILLA " + classId + " encuada peticio WRITE "); // TODO TODO log } } catch (IOException ioe) { LOG.info("opWriteBlock " + block + " received exception " + ioe); throw ioe; } finally { // close all opened streams IOUtils.closeSocket(mirrorSock); } }
From source file:org.darkphoenixs.pool.socket.SocketConnectionFactory.java
@Override public Socket createConnection() throws Exception { Socket socket = new Socket(); try {/*w w w . j a va 2s .co m*/ if (sendBufferSize > 0) socket.setSendBufferSize(sendBufferSize); if (receiveBufferSize > 0) socket.setReceiveBufferSize(receiveBufferSize); if (soTimeout > 0) socket.setSoTimeout(soTimeout); if (linger > 0) socket.setSoLinger(true, linger); if (keepAlive) socket.setKeepAlive(keepAlive); if (tcpNoDelay) socket.setTcpNoDelay(tcpNoDelay); if (performance != null) socket.setPerformancePreferences(Integer.parseInt(performance[0]), Integer.parseInt(performance[1]), Integer.parseInt(performance[2])); socket.connect(socketAddress, connectionTimeout); } catch (Exception se) { socket.close(); throw se; } return socket; }
From source file:org.globus.ftp.dc.GridFTPActiveConnectTask.java
public void execute() { Socket mySocket = null; if (logger.isDebugEnabled()) { logger.debug("connecting new socket to: " + hostPort.getHost() + " " + hostPort.getPort()); }/*from ww w . j av a 2s . c om*/ SocketFactory factory = SocketFactory.getDefault(); try { mySocket = factory.createSocket(hostPort.getHost(), hostPort.getPort()); // set TCP buffer size if (gSession.TCPBufferSize != Session.SERVER_DEFAULT) { logger.debug("setting socket's TCP buffer size to " + gSession.TCPBufferSize); mySocket.setReceiveBufferSize(gSession.TCPBufferSize); mySocket.setSendBufferSize(gSession.TCPBufferSize); } if (!gSession.dataChannelAuthentication.equals(DataChannelAuthentication.NONE)) { logger.debug("authenticating"); mySocket = GridFTPServerFacade.authenticate(mySocket, true, // this IS client socket gSession.credential, gSession.dataChannelProtection, gSession.dataChannelAuthentication); } else { logger.debug("not authenticating"); } // setting the Facade's socket list // synchronize to prevent race condidion against // the section in GridFTPServerFacade.setTCPBufferSize synchronized (box) { box.setSocket(mySocket); } } catch (Exception e) { FTPServerFacade.exceptionToControlChannel(e, "active connection to server failed", control); try { if (mySocket != null) { mySocket.close(); } } catch (Exception second) { } } }