List of usage examples for java.net Socket setSoTimeout
public synchronized void setSoTimeout(int timeout) throws SocketException
From source file:org.apache.hadoop.hdfs.server.datanode.DataWriter.java
/** * Write a block to disk./*from w w w .j a va2s . c o m*/ * * @param in The stream to read from * @throws IOException */ private void writeBlock() throws IOException { DatanodeInfo srcDataNode = null; LOG.debug("writeBlock receive buf size " + s.getReceiveBufferSize() + " tcp no delay " + s.getTcpNoDelay()); // // Read in the header // long startTime = System.currentTimeMillis(); int namespaceid = in.readInt(); Block block = new Block(in.readLong(), dataXceiverServer.estimateBlockSize, in.readLong()); LOG.info("Receiving block " + block + " src: " + remoteAddress + " dest: " + localAddress); int pipelineSize = in.readInt(); // num of datanodes in entire pipeline boolean isRecovery = in.readBoolean(); // is this part of recovery? String client = Text.readString(in); // working on behalf of this client boolean hasSrcDataNode = in.readBoolean(); // is src node info present if (hasSrcDataNode) { srcDataNode = new DatanodeInfo(); srcDataNode.readFields(in); } int numTargets = in.readInt(); if (numTargets < 0) { throw new IOException("Mislabelled incoming datastream."); } DatanodeInfo targets[] = new DatanodeInfo[numTargets]; for (int i = 0; i < targets.length; i++) { DatanodeInfo tmp = new DatanodeInfo(); tmp.readFields(in); targets[i] = tmp; } DataOutputStream mirrorOut = null; // stream to next target DataInputStream mirrorIn = null; // reply from next target DataOutputStream replyOut = null; // stream to prev target Socket mirrorSock = null; // socket to next target BlockReceiver blockReceiver = null; // responsible for data handling String mirrorNode = null; // the name:port of next target String firstBadLink = ""; // first datanode that failed in connection setup updateCurrentThreadName("receiving block " + block + " client=" + client); try { // open a block receiver and check if the block does not exist blockReceiver = new BlockReceiver(namespaceid, block, in, s.getRemoteSocketAddress().toString(), s.getLocalSocketAddress().toString(), isRecovery, client, srcDataNode, datanode); // get a connection back to the previous target replyOut = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(s, datanode.socketWriteTimeout), SMALL_BUFFER_SIZE)); // // Open network conn to backup machine, if // appropriate // if (targets.length > 0) { InetSocketAddress mirrorTarget = null; // Connect to backup machine mirrorNode = targets[0].getName(); mirrorTarget = NetUtils.createSocketAddr(mirrorNode); mirrorSock = datanode.newSocket(); try { int timeoutValue = datanode.socketTimeout + (datanode.socketReadExtentionTimeout * numTargets); int writeTimeout = datanode.socketWriteTimeout + (datanode.socketWriteExtentionTimeout * numTargets); NetUtils.connect(mirrorSock, mirrorTarget, timeoutValue); mirrorSock.setSoTimeout(timeoutValue); mirrorSock.setSendBufferSize(DEFAULT_DATA_SOCKET_SIZE); mirrorOut = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(mirrorSock, writeTimeout), SMALL_BUFFER_SIZE)); mirrorIn = new DataInputStream(NetUtils.getInputStream(mirrorSock)); // Write header: Copied from DFSClient.java! mirrorOut.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION); mirrorOut.write(DataTransferProtocol.OP_WRITE_BLOCK); mirrorOut.writeInt(namespaceid); mirrorOut.writeLong(block.getBlockId()); mirrorOut.writeLong(block.getGenerationStamp()); mirrorOut.writeInt(pipelineSize); mirrorOut.writeBoolean(isRecovery); Text.writeString(mirrorOut, client); mirrorOut.writeBoolean(hasSrcDataNode); if (hasSrcDataNode) { // pass src node information srcDataNode.write(mirrorOut); } mirrorOut.writeInt(targets.length - 1); for (int i = 1; i < targets.length; i++) { targets[i].write(mirrorOut); } blockReceiver.writeChecksumHeader(mirrorOut); mirrorOut.flush(); // read connect ack (only for clients, not for replication req) if (client.length() != 0) { firstBadLink = Text.readString(mirrorIn); if (LOG.isDebugEnabled() || firstBadLink.length() > 0) { LOG.info("Datanode " + targets.length + " got response for connect ack " + " from downstream datanode with firstbadlink as " + firstBadLink); } } } catch (IOException e) { if (client.length() != 0) { Text.writeString(replyOut, mirrorNode); replyOut.flush(); } IOUtils.closeStream(mirrorOut); mirrorOut = null; IOUtils.closeStream(mirrorIn); mirrorIn = null; IOUtils.closeSocket(mirrorSock); mirrorSock = null; if (client.length() > 0) { throw e; } else { LOG.info(datanode.getDatanodeInfo() + ":Exception transfering block " + block + " to mirror " + mirrorNode + ". continuing without the mirror.\n" + StringUtils.stringifyException(e)); } } } // send connect ack back to source (only for clients) if (client.length() != 0) { if (LOG.isDebugEnabled() || firstBadLink.length() > 0) { LOG.info("Datanode " + targets.length + " forwarding connect ack to upstream firstbadlink is " + firstBadLink); } Text.writeString(replyOut, firstBadLink); replyOut.flush(); } // receive the block and mirror to the next target String mirrorAddr = (mirrorSock == null) ? null : mirrorNode; long totalReceiveSize = blockReceiver.receiveBlock(mirrorOut, mirrorIn, replyOut, mirrorAddr, null, targets.length); // if this write is for a replication request (and not // from a client), then confirm block. For client-writes, // the block is finalized in the PacketResponder. if (client.length() == 0) { datanode.notifyNamenodeReceivedBlock(namespaceid, block, null); LOG.info("Received block " + block + " src: " + remoteAddress + " dest: " + localAddress + " of size " + block.getNumBytes()); } else { // Log the fact that the block has been received by this datanode and // has been written to the local disk on this datanode. LOG.info("Received Block " + block + " src: " + remoteAddress + " dest: " + localAddress + " of size " + block.getNumBytes() + " and written to local disk"); } if (datanode.blockScanner != null) { datanode.blockScanner.addBlock(namespaceid, block); } long writeDuration = System.currentTimeMillis() - startTime; datanode.myMetrics.bytesWrittenLatency.inc(writeDuration); if (totalReceiveSize > KB_RIGHT_SHIFT_MIN) { datanode.myMetrics.bytesWrittenRate.inc((int) (totalReceiveSize >> KB_RIGHT_SHIFT_BITS), writeDuration); } } catch (IOException ioe) { LOG.info("writeBlock " + block + " received exception " + ioe); throw ioe; } finally { // close all opened streams IOUtils.closeStream(mirrorOut); IOUtils.closeStream(mirrorIn); IOUtils.closeStream(replyOut); IOUtils.closeSocket(mirrorSock); IOUtils.closeStream(blockReceiver); } }
From source file:org.apache.hadoop.hdfs.server.datanode.DataXceiver.java
/** * Write a block to disk./*from w w w . ja v a 2 s. co m*/ * * @param in The stream to read from * @throws IOException */ private void writeBlock(DataInputStream in) throws IOException { DatanodeInfo srcDataNode = null; LOG.debug("writeBlock receive buf size " + s.getReceiveBufferSize() + " tcp no delay " + s.getTcpNoDelay()); // // Read in the header // Block block = new Block(in.readLong(), dataXceiverServer.estimateBlockSize, in.readLong()); LOG.info("Receiving block " + block + " src: " + remoteAddress + " dest: " + localAddress); int pipelineSize = in.readInt(); // num of datanodes in entire pipeline boolean isRecovery = in.readBoolean(); // is this part of recovery? String client = Text.readString(in); // working on behalf of this client boolean hasSrcDataNode = in.readBoolean(); // is src node info present if (hasSrcDataNode) { srcDataNode = new DatanodeInfo(); srcDataNode.readFields(in); } int numTargets = in.readInt(); if (numTargets < 0) { throw new IOException("Mislabelled incoming datastream."); } DatanodeInfo targets[] = new DatanodeInfo[numTargets]; for (int i = 0; i < targets.length; i++) { DatanodeInfo tmp = new DatanodeInfo(); tmp.readFields(in); targets[i] = tmp; } Token<BlockTokenIdentifier> accessToken = new Token<BlockTokenIdentifier>(); accessToken.readFields(in); DataOutputStream replyOut = null; // stream to prev target replyOut = new DataOutputStream(NetUtils.getOutputStream(s, datanode.socketWriteTimeout)); if (datanode.isBlockTokenEnabled) { try { datanode.blockTokenSecretManager.checkAccess(accessToken, null, block, BlockTokenSecretManager.AccessMode.WRITE); } catch (InvalidToken e) { try { if (client.length() != 0) { replyOut.writeShort((short) DataTransferProtocol.OP_STATUS_ERROR_ACCESS_TOKEN); Text.writeString(replyOut, datanode.dnRegistration.getName()); replyOut.flush(); } throw new IOException("Access token verification failed, for client " + remoteAddress + " for OP_WRITE_BLOCK for block " + block); } finally { IOUtils.closeStream(replyOut); } } } DataOutputStream mirrorOut = null; // stream to next target DataInputStream mirrorIn = null; // reply from next target Socket mirrorSock = null; // socket to next target BlockReceiver blockReceiver = null; // responsible for data handling String mirrorNode = null; // the name:port of next target String firstBadLink = ""; // first datanode that failed in connection setup short mirrorInStatus = (short) DataTransferProtocol.OP_STATUS_SUCCESS; try { // open a block receiver and check if the block does not exist blockReceiver = new BlockReceiver(block, in, s.getRemoteSocketAddress().toString(), s.getLocalSocketAddress().toString(), isRecovery, client, srcDataNode, datanode); // // Open network conn to backup machine, if // appropriate // if (targets.length > 0) { InetSocketAddress mirrorTarget = null; // Connect to backup machine mirrorNode = targets[0].getName(); mirrorTarget = NetUtils.createSocketAddr(mirrorNode); mirrorSock = datanode.newSocket(); try { int timeoutValue = datanode.socketTimeout + (HdfsConstants.READ_TIMEOUT_EXTENSION * numTargets); int writeTimeout = datanode.socketWriteTimeout + (HdfsConstants.WRITE_TIMEOUT_EXTENSION * numTargets); NetUtils.connect(mirrorSock, mirrorTarget, timeoutValue); mirrorSock.setSoTimeout(timeoutValue); mirrorSock.setSendBufferSize(DEFAULT_DATA_SOCKET_SIZE); mirrorOut = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(mirrorSock, writeTimeout), SMALL_BUFFER_SIZE)); mirrorIn = new DataInputStream(NetUtils.getInputStream(mirrorSock)); // Write header: Copied from DFSClient.java! mirrorOut.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION); mirrorOut.write(DataTransferProtocol.OP_WRITE_BLOCK); mirrorOut.writeLong(block.getBlockId()); mirrorOut.writeLong(block.getGenerationStamp()); mirrorOut.writeInt(pipelineSize); mirrorOut.writeBoolean(isRecovery); Text.writeString(mirrorOut, client); mirrorOut.writeBoolean(hasSrcDataNode); if (hasSrcDataNode) { // pass src node information srcDataNode.write(mirrorOut); } mirrorOut.writeInt(targets.length - 1); for (int i = 1; i < targets.length; i++) { targets[i].write(mirrorOut); } accessToken.write(mirrorOut); blockReceiver.writeChecksumHeader(mirrorOut); mirrorOut.flush(); // read connect ack (only for clients, not for replication req) if (client.length() != 0) { mirrorInStatus = mirrorIn.readShort(); firstBadLink = Text.readString(mirrorIn); if (LOG.isDebugEnabled() || mirrorInStatus != DataTransferProtocol.OP_STATUS_SUCCESS) { LOG.info("Datanode " + targets.length + " got response for connect ack " + " from downstream datanode with firstbadlink as " + firstBadLink); } } } catch (IOException e) { if (client.length() != 0) { replyOut.writeShort((short) DataTransferProtocol.OP_STATUS_ERROR); Text.writeString(replyOut, mirrorNode); replyOut.flush(); } IOUtils.closeStream(mirrorOut); mirrorOut = null; IOUtils.closeStream(mirrorIn); mirrorIn = null; IOUtils.closeSocket(mirrorSock); mirrorSock = null; if (client.length() > 0) { throw e; } else { LOG.info(datanode.dnRegistration + ":Exception transfering block " + block + " to mirror " + mirrorNode + ". continuing without the mirror.\n" + StringUtils.stringifyException(e)); } } } // send connect ack back to source (only for clients) if (client.length() != 0) { if (LOG.isDebugEnabled() || mirrorInStatus != DataTransferProtocol.OP_STATUS_SUCCESS) { LOG.info("Datanode " + targets.length + " forwarding connect ack to upstream firstbadlink is " + firstBadLink); } replyOut.writeShort(mirrorInStatus); Text.writeString(replyOut, firstBadLink); replyOut.flush(); } // receive the block and mirror to the next target String mirrorAddr = (mirrorSock == null) ? null : mirrorNode; blockReceiver.receiveBlock(mirrorOut, mirrorIn, replyOut, mirrorAddr, null, targets.length); // if this write is for a replication request (and not // from a client), then confirm block. For client-writes, // the block is finalized in the PacketResponder. if (client.length() == 0) { datanode.notifyNamenodeReceivedBlock(block, DataNode.EMPTY_DEL_HINT); LOG.info("Received block " + block + " src: " + remoteAddress + " dest: " + localAddress + " of size " + block.getNumBytes()); } if (datanode.blockScanner != null) { datanode.blockScanner.addBlock(block); } } catch (IOException ioe) { LOG.info("writeBlock " + block + " received exception " + ioe); throw ioe; } finally { // close all opened streams IOUtils.closeStream(mirrorOut); IOUtils.closeStream(mirrorIn); IOUtils.closeStream(replyOut); IOUtils.closeSocket(mirrorSock); IOUtils.closeStream(blockReceiver); } }
From source file:catalina.core.StandardServer.java
/** * Wait until a proper shutdown command is received, then return. */// www . j a v a 2 s. co m public void await() { // Set up a server socket to wait on ServerSocket serverSocket = null; try { serverSocket = new ServerSocket(port, 1, InetAddress.getByName("127.0.0.1")); } catch (IOException e) { System.err.println("StandardServer.await: create[" + port + "]: " + e); e.printStackTrace(); System.exit(1); } // Loop waiting for a connection and a valid command while (true) { // Wait for the next connection Socket socket = null; InputStream stream = null; try { socket = serverSocket.accept(); socket.setSoTimeout(10 * 1000); // Ten seconds stream = socket.getInputStream(); } catch (AccessControlException ace) { System.err.println("StandardServer.accept security exception: " + ace.getMessage()); continue; } catch (IOException e) { System.err.println("StandardServer.await: accept: " + e); e.printStackTrace(); System.exit(1); } // Read a set of characters from the socket StringBuffer command = new StringBuffer(); int expected = 1024; // Cut off to avoid DoS attack while (expected < shutdown.length()) { if (random == null) random = new Random(System.currentTimeMillis()); expected += (random.nextInt() % 1024); } while (expected > 0) { int ch = -1; try { ch = stream.read(); } catch (IOException e) { System.err.println("StandardServer.await: read: " + e); e.printStackTrace(); ch = -1; } if (ch < 32) // Control character or EOF terminates loop break; command.append((char) ch); expected--; } // Close the socket now that we are done with it try { socket.close(); } catch (IOException e) { ; } // Match against our command string boolean match = command.toString().equals(shutdown); if (match) { break; } else System.err.println("StandardServer.await: Invalid command '" + command.toString() + "' received"); } // Close the server socket and return try { serverSocket.close(); } catch (IOException e) { ; } }
From source file:org.apache.geode.internal.cache.tier.sockets.HandShake.java
/** * HandShake Constructor used by server side connection *//*from w w w . j a v a 2s . co m*/ public HandShake(Socket sock, int timeout, DistributedSystem sys, Version clientVersion, CommunicationMode communicationMode, SecurityService securityService) throws IOException, AuthenticationRequiredException { this.clientVersion = clientVersion; this.system = sys; this.securityService = securityService; { int soTimeout = -1; try { soTimeout = sock.getSoTimeout(); sock.setSoTimeout(timeout); InputStream is = sock.getInputStream(); int valRead = is.read(); // this.code = (byte)is.read(); if (valRead == -1) { throw new EOFException( LocalizedStrings.HandShake_HANDSHAKE_EOF_REACHED_BEFORE_CLIENT_CODE_COULD_BE_READ .toLocalizedString()); } this.code = (byte) valRead; if (this.code != REPLY_OK) { throw new IOException( LocalizedStrings.HandShake_HANDSHAKE_REPLY_CODE_IS_NOT_OK.toLocalizedString()); } try { DataInputStream dis = new DataInputStream(is); DataOutputStream dos = new DataOutputStream(sock.getOutputStream()); this.clientReadTimeout = dis.readInt(); if (clientVersion.compareTo(Version.CURRENT) < 0) { // versioned streams allow object serialization code to deal with older clients dis = new VersionedDataInputStream(dis, clientVersion); dos = new VersionedDataOutputStream(dos, clientVersion); } this.id = ClientProxyMembershipID.readCanonicalized(dis); // Note: credentials should always be the last piece in handshake for // Diffie-Hellman key exchange to work if (clientVersion.compareTo(Version.GFE_603) >= 0) { setOverrides(new byte[] { dis.readByte() }); } else { setClientConflation(dis.readByte()); } // Hitesh if (this.clientVersion.compareTo(Version.GFE_65) < 0 || communicationMode.isWAN()) { this.credentials = readCredentials(dis, dos, sys, this.securityService); } else { this.credentials = this.readCredential(dis, dos, sys); } } catch (IOException ioe) { this.code = -2; throw ioe; } catch (ClassNotFoundException cnfe) { this.code = -3; throw new IOException( LocalizedStrings.HandShake_CLIENTPROXYMEMBERSHIPID_CLASS_COULD_NOT_BE_FOUND_WHILE_DESERIALIZING_THE_OBJECT .toLocalizedString()); } } finally { if (soTimeout != -1) { try { sock.setSoTimeout(soTimeout); } catch (IOException ignore) { } } } } }
From source file:com.mellanox.r4h.DFSOutputStream.java
/** * Create a socket for a write pipeline/*from w ww . j av a 2 s. com*/ * * @param first * the first datanode * @param length * the pipeline length * @param client * client * @return the socket connected to the first datanode */ static Socket createSocketForPipeline(final DatanodeInfo first, final int length, final DFSClient client) throws IOException { final String dnAddr = first.getXferAddr(client.getConf().getConnectToDnViaHostname()); if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("Connecting to datanode " + dnAddr); } final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr); final Socket sock = client.socketFactory.createSocket(); final int timeout = client.getDatanodeReadTimeout(length); NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), client.getConf().getSocketTimeout()); sock.setSoTimeout(timeout); sock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE); if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("Send buf size " + sock.getSendBufferSize()); } return sock; }
From source file:org.xwoot.jxta.JxtaPeer.java
/** {@inheritDoc} **/ public Object sendObject(Object object, PipeAdvertisement pipeAdv) throws JxtaException { if (!this.isConnectedToGroup()) { throw new PeerGroupException("The peer has not yet joined a group and contacted a RDV peer."); }/* ww w . jav a2s. c o m*/ if (!(object instanceof Serializable)) { throw new IllegalArgumentException( "The object does not implement the interface java.io.Serializable and can not be sent."); } boolean failed = false; Socket socket = null; InputStream is = null; OutputStream os = null; ObjectInputStream ois = null; ObjectOutputStream oos = null; try { socket = new JxtaSocket(this.currentJoinedGroup, pipeAdv); socket.setSoTimeout(WAIT_INTERVAL_FOR_DIRECT_COMMUNICATION_CONNECTIONS); } catch (Exception e) { throw new JxtaException("Failed to create a direct connection using the provided pipe advertisement.", e); } try { os = socket.getOutputStream(); oos = new ObjectOutputStream(os); oos.writeObject(object); oos.flush(); os.flush(); } catch (Exception e) { failed = true; throw new JxtaException( "Failed to send an object through a direct connection using the provided pipe advertisement.", e); } finally { try { if (oos != null) { oos.close(); } if (os != null) { os.close(); } // Close the socket only if this step has failed. if (failed && socket != null) { socket.close(); } } catch (Exception e) { // Just log it. this.logger.warn("Failed to close streams for this conenction."); } } Object replyMessage = null; try { is = socket.getInputStream(); ois = new ObjectInputStream(is); replyMessage = ois.readObject(); } catch (EOFException eof) { this.logger.debug("There is no reply to this message. Returning null."); replyMessage = null; } catch (Exception e) { throw new JxtaException("Failed to receive reply message.", e); } finally { try { if (ois != null) { ois.close(); } } catch (Exception e) { // Just log it. this.logger.warn("Failed to close object input stream for this conenction.", e); } try { if (is != null) { is.close(); } } catch (Exception e) { // Just log it. this.logger.warn("Failed to close input stream for this conenction.", e); } try { // Close the socket, we are done. if (socket != null) { socket.close(); } } catch (Exception e) { // Just log it. this.logger.warn("Failed to close socket for this conenction."); } } return replyMessage; }
From source file:org.lockss.protocol.BlockingStreamComm.java
/** Setup all socket options. Should be called before any read/write * calls *//* ww w . j a v a2s .c o m*/ void setupOpenSocket(Socket sock) throws SocketException { if (log.isDebug3()) { log.debug3(sock + "SO_TIMEOUT: " + getSoTimeout() + ", TcpNoDelay: " + isTcpNoDelay() + ", KeepAlive: " + paramSoKeepAlive); } sock.setSoTimeout((int) getSoTimeout()); sock.setTcpNoDelay(isTcpNoDelay()); sock.setKeepAlive(paramSoKeepAlive); }
From source file:org.apache.hadoop.hdfs.AvatarClient.java
/** * Get the checksum of a file./*from ww w.j a v a2 s. c o m*/ * @param src The file path * @return The checksum */ public static MD5MD5CRC32FileChecksum getFileChecksum(String src, AvatarProtocol namenode, SocketFactory socketFactory, int socketTimeout) throws IOException { //get all block locations final List<LocatedBlock> locatedblocks = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE) .getLocatedBlocks(); final DataOutputBuffer md5out = new DataOutputBuffer(); int bytesPerCRC = 0; long crcPerBlock = 0; //get block checksum for each block for (int i = 0; i < locatedblocks.size(); i++) { LocatedBlock lb = locatedblocks.get(i); final Block block = lb.getBlock(); final DatanodeInfo[] datanodes = lb.getLocations(); //try each datanode location of the block final int timeout = 3000 * datanodes.length + socketTimeout; boolean done = false; for (int j = 0; !done && j < datanodes.length; j++) { //connect to a datanode final Socket sock = socketFactory.createSocket(); NetUtils.connect(sock, NetUtils.createSocketAddr(datanodes[j].getName()), timeout); sock.setSoTimeout(timeout); DataOutputStream out = new DataOutputStream( new BufferedOutputStream(NetUtils.getOutputStream(sock), DataNode.SMALL_BUFFER_SIZE)); DataInputStream in = new DataInputStream(NetUtils.getInputStream(sock)); // get block MD5 try { if (LOG.isDebugEnabled()) { LOG.debug("write to " + datanodes[j].getName() + ": " + DataTransferProtocol.OP_BLOCK_CHECKSUM + ", block=" + block); } out.writeShort(DataTransferProtocol.DATA_TRANSFER_VERSION); out.write(DataTransferProtocol.OP_BLOCK_CHECKSUM); out.writeLong(block.getBlockId()); out.writeLong(block.getGenerationStamp()); out.flush(); final short reply = in.readShort(); if (reply != DataTransferProtocol.OP_STATUS_SUCCESS) { throw new IOException("Bad response " + reply + " for block " + block + " from datanode " + datanodes[j].getName()); } //read byte-per-checksum final int bpc = in.readInt(); if (i == 0) { //first block bytesPerCRC = bpc; } else if (bpc != bytesPerCRC) { throw new IOException( "Byte-per-checksum not matched: bpc=" + bpc + " but bytesPerCRC=" + bytesPerCRC); } //read crc-per-block final long cpb = in.readLong(); if (locatedblocks.size() > 1 && i == 0) { crcPerBlock = cpb; } //read md5 final MD5Hash md5 = MD5Hash.read(in); md5.write(md5out); done = true; if (LOG.isDebugEnabled()) { if (i == 0) { LOG.debug("set bytesPerCRC=" + bytesPerCRC + ", crcPerBlock=" + crcPerBlock); } LOG.debug("got reply from " + datanodes[j].getName() + ": md5=" + md5); } } catch (IOException ie) { LOG.warn("src=" + src + ", datanodes[" + j + "].getName()=" + datanodes[j].getName(), ie); } finally { IOUtils.closeStream(in); IOUtils.closeStream(out); IOUtils.closeSocket(sock); } } if (!done) { throw new IOException("Fail to get block MD5 for " + block); } } //compute file MD5 final MD5Hash fileMD5 = MD5Hash.digest(md5out.getData()); return new MD5MD5CRC32FileChecksum(bytesPerCRC, crcPerBlock, fileMD5); }
From source file:de.prozesskraft.pradar.parts.PradarPartUi3.java
/** * asks for entities from the first pradar-server that responds * @return void//from ww w.ja v a2 s .c o m */ void loadData() { for (String portAtMachineAsString : this.pradar_server_port_at_hostname) { String[] port_and_machine = portAtMachineAsString.split("@"); int portNumber = Integer.parseInt(port_and_machine[0]); String machineName = port_and_machine[1]; log("info", "want to load data from pradar-server"); log("info", "trying pradar-server " + portNumber + "@" + machineName); try { // socket einrichten und Out/Input-Streams setzen // log("debug", "machineName="+machineName+" | portNumber="+portNumber); // log("debug", "server objekt erstellen"); Socket connectToServerSocket = new Socket(machineName, portNumber); connectToServerSocket.setSoTimeout(20000); // log("debug", "outputStream erstellen"); OutputStream streamToServer = connectToServerSocket.getOutputStream(); // log("debug", "outputStream flushen"); streamToServer.flush(); // log("debug", "objectOutputStream erstellen"); ObjectOutputStream objectToServer = new ObjectOutputStream(streamToServer); // log("debug", "objectOutputStream flushen"); objectToServer.flush(); // Objekte zum server uebertragen // log("debug", "write: getallfromuser"); objectToServer.writeObject("getallfromuser"); objectToServer.writeObject(System.getProperty("user.name")); // log("debug", "outputStream flushen"); streamToServer.flush(); // log("debug", "objectOutputStream flushen"); objectToServer.flush(); // sende-object zerstoeren - wird nicht mehr gebraucht // log("debug", "objectOutputStream schliessen"); // objectToServer.close(); // log("debug", "inputStream erstellen"); InputStream streamFromServer = connectToServerSocket.getInputStream(); // log("debug", "objectInputStream erstellen"); ObjectInputStream objectFromServer = new ObjectInputStream(streamFromServer); // Antwort vom Server lesen - ein array aller Entities try { // log("debug", "reading"); Object serverAnswer = objectFromServer.readObject(); // log("debug", "reading done"); // lese-object zerstoeren - wird nicht mehr gebraucht objectFromServer.close(); // log("debug", "objectFromServer closed"); ArrayList<Object> serverAnswer2 = null; if (serverAnswer instanceof ArrayList) { // log("debug", "serverAnswer is an ArrayList"); serverAnswer2 = (ArrayList<Object>) serverAnswer; } // alle existierenden entities loeschen this.entities_all.clear(); // die neuen entities casten und in einem map unterbringen id->Entities erstellen um die children bei ihren parents einsortieren zu koennen Map<String, Entity> entities_all = new HashMap<String, Entity>(); for (Object actObject : serverAnswer2) { // if (actObject instanceof Entity) { // log("debug", "item of ArrayList<Object> is an Entity ---> adding to ArrayList<Entity>"); Entity newEntity = (Entity) actObject; this.entities_all.add(newEntity); } } // log("debug", "reading done! closing "); objectFromServer.close(); // log("debug", "read finished"); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } // daten holen aus db log("info", "refreshing data..."); connectToServerSocket.close(); } catch (UnknownHostException e) { log("warn", "unknown host " + machineName); this.pradar_server_port_at_hostname = null; } catch (IOException e) { log("warn", "input / output problems at " + portNumber + "@" + machineName); e.printStackTrace(); } this.refresh_last = Calendar.getInstance(); this.refresh_next = Calendar.getInstance(); this.refresh_next.add(13, this.refresh_interval); } }
From source file:org.apache.hadoop.hdfs.server.datanode.CachingDataXceiver.java
@Override public void writeBlock(final ExtendedBlock block, final Token<BlockTokenIdentifier> blockToken, final String clientname, final DatanodeInfo[] targets, final DatanodeInfo srcDataNode, final BlockConstructionStage stage, final int pipelineSize, final long minBytesRcvd, final long maxBytesRcvd, final long latestGenerationStamp, DataChecksum requestedChecksum) throws IOException { previousOpClientName = clientname;/* w ww . jav a 2 s . c om*/ updateCurrentThreadName("Receiving block " + block); final boolean isDatanode = clientname.length() == 0; final boolean isClient = !isDatanode; final boolean isTransfer = stage == BlockConstructionStage.TRANSFER_RBW || stage == BlockConstructionStage.TRANSFER_FINALIZED; // check single target for transfer-RBW/Finalized if (isTransfer && targets.length > 0) { throw new IOException(stage + " does not support multiple targets " + Arrays.asList(targets)); } if (LOG.isDebugEnabled()) { LOG.debug("opWriteBlock: stage=" + stage + ", clientname=" + clientname + "\n block =" + block + ", newGs=" + latestGenerationStamp + ", bytesRcvd=[" + minBytesRcvd + ", " + maxBytesRcvd + "]" + "\n targets=" + Arrays.asList(targets) + "; pipelineSize=" + pipelineSize + ", srcDataNode=" + srcDataNode); LOG.debug("isDatanode=" + isDatanode + ", isClient=" + isClient + ", isTransfer=" + isTransfer); LOG.debug("writeBlock receive buf size " + s.getReceiveBufferSize() + " tcp no delay " + s.getTcpNoDelay()); } // We later mutate block's generation stamp and length, but we need to // forward the original version of the block to downstream mirrors, so // make a copy here. final ExtendedBlock originalBlock = new ExtendedBlock(block); block.setNumBytes(dataXceiverServer.estimateBlockSize); LOG.info("Receiving block " + block + " src: " + remoteAddress + " dest: " + localAddress); // reply to upstream datanode or client final DataOutputStream replyOut = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(s, dnConf.socketWriteTimeout), HdfsConstants.SMALL_BUFFER_SIZE)); checkAccess(replyOut, isClient, block, blockToken, Op.WRITE_BLOCK, BlockTokenSecretManager.AccessMode.WRITE); DataOutputStream mirrorOut = null; // stream to next target DataInputStream mirrorIn = null; // reply from next target Socket mirrorSock = null; // socket to next target BlockReceiver blockReceiver = null; // responsible for data handling String mirrorNode = null; // the name:port of next target String firstBadLink = ""; // first datanode that failed in connection setup Status mirrorInStatus = SUCCESS; try { if (isDatanode || stage != BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { // open a block receiver blockReceiver = new BlockReceiver(block, in, s.getRemoteSocketAddress().toString(), s.getLocalSocketAddress().toString(), stage, latestGenerationStamp, minBytesRcvd, maxBytesRcvd, clientname, srcDataNode, datanode, requestedChecksum); } else { datanode.data.recoverClose(block, latestGenerationStamp, minBytesRcvd); } // // Connect to downstream machine, if appropriate // if (targets.length > 0) { InetSocketAddress mirrorTarget = null; // Connect to backup machine mirrorNode = targets[0].getXferAddr(); mirrorTarget = NetUtils.createSocketAddr(mirrorNode); mirrorSock = datanode.newSocket(); try { int timeoutValue = dnConf.socketTimeout + (HdfsServerConstants.READ_TIMEOUT_EXTENSION * targets.length); int writeTimeout = dnConf.socketWriteTimeout + (HdfsServerConstants.WRITE_TIMEOUT_EXTENSION * targets.length); NetUtils.connect(mirrorSock, mirrorTarget, timeoutValue); mirrorSock.setSoTimeout(timeoutValue); mirrorSock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE); mirrorOut = new DataOutputStream(new BufferedOutputStream( NetUtils.getOutputStream(mirrorSock, writeTimeout), HdfsConstants.SMALL_BUFFER_SIZE)); mirrorIn = new DataInputStream(NetUtils.getInputStream(mirrorSock)); new Sender(mirrorOut).writeBlock(originalBlock, blockToken, clientname, targets, srcDataNode, stage, pipelineSize, minBytesRcvd, maxBytesRcvd, latestGenerationStamp, requestedChecksum); mirrorOut.flush(); // read connect ack (only for clients, not for replication req) if (isClient) { BlockOpResponseProto connectAck = BlockOpResponseProto .parseFrom(HdfsProtoUtil.vintPrefixed(mirrorIn)); mirrorInStatus = connectAck.getStatus(); firstBadLink = connectAck.getFirstBadLink(); if (LOG.isDebugEnabled() || mirrorInStatus != SUCCESS) { LOG.info("Datanode " + targets.length + " got response for connect ack " + " from downstream datanode with firstbadlink as " + firstBadLink); } } } catch (IOException e) { if (isClient) { BlockOpResponseProto.newBuilder().setStatus(ERROR).setFirstBadLink(mirrorNode).build() .writeDelimitedTo(replyOut); replyOut.flush(); } IOUtils.closeStream(mirrorOut); mirrorOut = null; IOUtils.closeStream(mirrorIn); mirrorIn = null; IOUtils.closeSocket(mirrorSock); mirrorSock = null; if (isClient) { LOG.error(datanode + ":Exception transfering block " + block + " to mirror " + mirrorNode + ": " + e); throw e; } else { LOG.info(datanode + ":Exception transfering block " + block + " to mirror " + mirrorNode + ". continuing without the mirror.", e); } } } // send connect-ack to source for clients and not transfer-RBW/Finalized if (isClient && !isTransfer) { if (LOG.isDebugEnabled() || mirrorInStatus != SUCCESS) { LOG.info("Datanode " + targets.length + " forwarding connect ack to upstream firstbadlink is " + firstBadLink); } BlockOpResponseProto.newBuilder().setStatus(mirrorInStatus).setFirstBadLink(firstBadLink).build() .writeDelimitedTo(replyOut); replyOut.flush(); } // receive the block and mirror to the next target if (blockReceiver != null) { String mirrorAddr = (mirrorSock == null) ? null : mirrorNode; blockReceiver.receiveBlock(mirrorOut, mirrorIn, replyOut, mirrorAddr, null, targets); // send close-ack for transfer-RBW/Finalized if (isTransfer) { if (LOG.isTraceEnabled()) { LOG.trace("TRANSFER: send close-ack"); } writeResponse(SUCCESS, null, replyOut); } } // update its generation stamp if (isClient && stage == BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { block.setGenerationStamp(latestGenerationStamp); block.setNumBytes(minBytesRcvd); } // if this write is for a replication request or recovering // a failed close for client, then confirm block. For other client-writes, // the block is finalized in the PacketResponder. if (isDatanode || stage == BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { datanode.closeBlock(block, DataNode.EMPTY_DEL_HINT); LOG.info("Received block " + block + " src: " + remoteAddress + " dest: " + localAddress + " of size " + block.getNumBytes()); } } catch (IOException ioe) { LOG.info("opWriteBlock " + block + " received exception " + ioe); throw ioe; } finally { // close all opened streams IOUtils.closeStream(mirrorOut); IOUtils.closeStream(mirrorIn); IOUtils.closeStream(replyOut); IOUtils.closeSocket(mirrorSock); IOUtils.closeStream(blockReceiver); } // update metrics datanode.metrics.addWriteBlockOp(elapsed()); datanode.metrics.incrWritesFromClient(isLocal); }