List of usage examples for java.nio ByteBuffer hasRemaining
public final boolean hasRemaining()
From source file:co.elastic.tealess.SSLChecker.java
private void checkHandshake(SSLReport sslReport, SocketChannel socket) { final InetSocketAddress address = sslReport.getAddress(); final String name = sslReport.getHostname(); IOObserver ioObserver = new IOObserver(); ObservingSSLEngine sslEngine = new ObservingSSLEngine(ctx.createSSLEngine(name, address.getPort()), ioObserver);//from w w w .jav a 2 s. com sslReport.setIOObserver(ioObserver); sslEngine.setUseClientMode(true); try { sslEngine.beginHandshake(); } catch (SSLException e) { sslReport.setFailed(e); Throwable cause = Blame.get(e); logger.warn("beginHandshake failed: [{}] {}", cause.getClass(), cause.getMessage()); } // TODO: Is this enough bytes? int size = sslEngine.getSession().getApplicationBufferSize() * 2; ByteBuffer localText = ByteBuffer.allocate(size); ByteBuffer localWire = ByteBuffer.allocate(size); ByteBuffer peerText = ByteBuffer.allocate(size); ByteBuffer peerWire = ByteBuffer.allocate(size); // TODO: I wonder... do we need to send any data at all? localText.put("SSL TEST. HELLO.".getBytes()); localText.flip(); SSLEngineResult result; logger.info("Starting SSL handshake [{}] ", address); try { SSLEngineResult.HandshakeStatus state; state = sslEngine.getHandshakeStatus(); while (state != FINISHED) { // XXX: Use a Selector to wait for data. //logger.trace("State: {} [{}]", state, address); switch (state) { case NEED_TASK: sslEngine.getDelegatedTask().run(); state = sslEngine.getHandshakeStatus(); break; case NEED_WRAP: localWire.clear(); result = sslEngine.wrap(localText, localWire); state = result.getHandshakeStatus(); localWire.flip(); while (localWire.hasRemaining()) { socket.write(localWire); //logger.trace("Sent {} bytes [{}]", bytes, address); } localWire.compact(); break; case NEED_UNWRAP: // Try reading until we get data. Selector selector = Selector.open(); while (peerWire.position() == 0) { socket.read(peerWire); try { Thread.currentThread().sleep(5); } catch (InterruptedException e) { e.printStackTrace(); } } System.out.println("Read " + peerWire.position() + " bytes"); peerWire.flip(); result = sslEngine.unwrap(peerWire, peerText); state = result.getHandshakeStatus(); peerWire.compact(); break; } } } catch (IOException e) { sslReport.setFailed(e); sslReport.setSSLSession(sslEngine.getHandshakeSession()); sslReport.setPeerCertificateDetails(peerCertificateDetails); logger.warn("beginHandshake failed", e); return; } logger.info("handshake completed [{}]", address); // Handshake OK! sslReport.setSSLSession(sslEngine.getSession()); }
From source file:x10.x10rt.yarn.ApplicationMaster.java
protected void handleX10() { // handle X10 place requests Iterator<SelectionKey> events = null; while (running) { try {//from www.ja v a 2 s. c o m SelectionKey key; // check for previously unhandled events if (events != null && events.hasNext()) { key = events.next(); events.remove(); } else if (selector.select() == 0) // check for new events continue; // nothing to process, go back and block on select again else { // select returned some events events = selector.selectedKeys().iterator(); key = events.next(); events.remove(); } // process the selectionkey if (key.isAcceptable()) { LOG.info("New connection from X10 detected"); // accept any connections on the server socket, and look for things to read from it ServerSocketChannel ssc = (ServerSocketChannel) key.channel(); SocketChannel sc = ssc.accept(); sc.configureBlocking(false); sc.register(selector, SelectionKey.OP_READ); } if (key.isReadable()) { SocketChannel sc = (SocketChannel) key.channel(); ByteBuffer incomingMsg; if (pendingReads.containsKey(sc)) incomingMsg = pendingReads.remove(sc); else incomingMsg = ByteBuffer.allocateDirect(headerLength).order(ByteOrder.nativeOrder()); LOG.info("Reading message from X10"); try { if (sc.read(incomingMsg) == -1) { // socket closed sc.close(); key.cancel(); pendingReads.remove(sc); } else if (incomingMsg.hasRemaining()) { LOG.info("Message header partially read. " + incomingMsg.remaining() + " bytes remaining"); pendingReads.put(sc, incomingMsg); } else { // buffer is full if (incomingMsg.capacity() == headerLength) { // check to see if there is a body associated with this message header int datalen = incomingMsg.getInt(headerLength - 4); //System.err.println("Byte order is "+incomingMsg.order()+" datalen="+datalen); if (datalen == 0) processMessage(incomingMsg, sc); else { // create a larger array to hold the header+body ByteBuffer newBuffer = ByteBuffer.allocateDirect(headerLength + datalen) .order(ByteOrder.nativeOrder()); incomingMsg.rewind(); newBuffer.put(incomingMsg); incomingMsg = newBuffer; sc.read(incomingMsg); // read in the body, if available if (incomingMsg.hasRemaining()) { LOG.info("Message partially read. " + incomingMsg.remaining() + " bytes remaining"); pendingReads.put(sc, incomingMsg); } else processMessage(incomingMsg, sc); } } } } catch (IOException e) { LOG.warn("Error reading in message from socket channel", e); } } } catch (IOException e) { LOG.warn("Error handling X10 links", e); } } }
From source file:edu.hawaii.soest.kilonalu.flntu.FLNTUSource.java
/** * A method that executes the streaming of data from the source to the RBNB * server after all configuration of settings, connections to hosts, and * thread initiatizing occurs. This method contains the detailed code for * streaming the data and interpreting the stream. */// w w w .j a v a2 s. c om protected boolean execute() { logger.debug("FLNTUSource.execute() called."); // do not execute the stream if there is no connection if (!isConnected()) return false; boolean failed = false; SocketChannel socket = getSocketConnection(); // while data are being sent, read them into the buffer try { // create four byte placeholders used to evaluate up to a four-byte // window. The FIFO layout looks like: // ------------------------- // in ---> | One | Two |Three|Four | ---> out // ------------------------- byte byteOne = 0x00, // set initial placeholder values byteTwo = 0x00, byteThree = 0x00, byteFour = 0x00; // Create a buffer that will store the sample bytes as they are read ByteBuffer sampleBuffer = ByteBuffer.allocate(getBufferSize()); // create a byte buffer to store bytes from the TCP stream ByteBuffer buffer = ByteBuffer.allocateDirect(getBufferSize()); // add a channel of data that will be pushed to the server. // Each sample will be sent to the Data Turbine as an rbnb frame. ChannelMap rbnbChannelMap = new ChannelMap(); // while there are bytes to read from the socket ... while (socket.read(buffer) != -1 || buffer.position() > 0) { // prepare the buffer for reading buffer.flip(); // while there are unread bytes in the ByteBuffer while (buffer.hasRemaining()) { byteOne = buffer.get(); logger.debug("char: " + (char) byteOne + "\t" + "b1: " + new String(Hex.encodeHex((new byte[] { byteOne }))) + "\t" + "b2: " + new String(Hex.encodeHex((new byte[] { byteTwo }))) + "\t" + "b3: " + new String(Hex.encodeHex((new byte[] { byteThree }))) + "\t" + "b4: " + new String(Hex.encodeHex((new byte[] { byteFour }))) + "\t" + "sample pos: " + sampleBuffer.position() + "\t" + "sample rem: " + sampleBuffer.remaining() + "\t" + "sample cnt: " + sampleByteCount + "\t" + "buffer pos: " + buffer.position() + "\t" + "buffer rem: " + buffer.remaining() + "\t" + "state: " + state); // Use a State Machine to process the byte stream. switch (state) { case 0: // sample sets begin with 'mvs 1\r\n' and end with 'mvs 0\r\n'. Find the // beginning of the sample set using the 4-byte window (s 1\r\n) // note bytes are in reverse order in the FIFO window if (byteOne == 0x0A && byteTwo == 0x0D && byteThree == 0x31 && byteFour == 0x20) { // we've found the beginning of a sample set, move on state = 1; break; } else { break; } case 1: // read the rest of the bytes to the next EOL characters // sample line is terminated by record delimiter byte (\r\n) // note bytes are in reverse order in the FIFO window if (byteOne == 0x0A && byteTwo == 0x0D && byteThree == 0x30 && byteFour == 0x20) { // we've found the sample set ending, clear buffers and return // to state 0 to wait for the next set byteOne = 0x00; byteTwo = 0x00; byteThree = 0x00; byteFour = 0x00; sampleBuffer.clear(); sampleByteCount = 0; rbnbChannelMap.Clear(); logger.debug("Cleared b1,b2,b3,b4. Cleared sampleBuffer. Cleared rbnbChannelMap."); state = 0; // if we're not at the sample set end, look for individual samples } else if (byteOne == 0x0A && byteTwo == 0x0D) { // found the sample ending delimiter // add in the sample delimiter to the sample buffer if (sampleBuffer.remaining() > 0) { sampleBuffer.put(byteOne); sampleByteCount++; } else { sampleBuffer.compact(); logger.debug("Compacting sampleBuffer ..."); sampleBuffer.put(byteOne); sampleByteCount++; } // extract just the length of the sample bytes out of the // sample buffer, and place it in the channel map as a // byte array. Then, send it to the data turbine. byte[] sampleArray = new byte[sampleByteCount]; sampleBuffer.flip(); sampleBuffer.get(sampleArray); // send the sample to the data turbine rbnbChannelMap.PutTimeAuto("server"); String sampleString = new String(sampleArray, "US-ASCII"); int channelIndex = rbnbChannelMap.Add(getRBNBChannelName()); rbnbChannelMap.PutMime(channelIndex, "text/plain"); rbnbChannelMap.PutDataAsString(channelIndex, sampleString); getSource().Flush(rbnbChannelMap); logger.info("Sample: " + sampleString.substring(0, sampleString.length() - 2) + " sent data to the DataTurbine. "); byteOne = 0x00; byteTwo = 0x00; byteThree = 0x00; byteFour = 0x00; sampleBuffer.clear(); sampleByteCount = 0; rbnbChannelMap.Clear(); logger.debug("Cleared b1,b2,b3,b4. Cleared sampleBuffer. Cleared rbnbChannelMap."); break; } else { // not 0x0 // still in the middle of the sample, keep adding bytes sampleByteCount++; // add each byte found if (sampleBuffer.remaining() > 0) { sampleBuffer.put(byteOne); } else { sampleBuffer.compact(); logger.debug("Compacting sampleBuffer ..."); sampleBuffer.put(byteOne); } break; } // end if for 0x0D20 EOL } // end switch statement // shift the bytes in the FIFO window byteFour = byteThree; byteThree = byteTwo; byteTwo = byteOne; } //end while (more unread bytes) // prepare the buffer to read in more bytes from the stream buffer.compact(); } // end while (more socket bytes to read) socket.close(); } catch (IOException e) { // handle exceptions // In the event of an i/o exception, log the exception, and allow execute() // to return false, which will prompt a retry. failed = true; e.printStackTrace(); return !failed; } catch (SAPIException sapie) { // In the event of an RBNB communication exception, log the exception, // and allow execute() to return false, which will prompt a retry. failed = true; sapie.printStackTrace(); return !failed; } return !failed; }
From source file:org.carbondata.core.util.CarbonUtil.java
public static short[] getUnCompressColumnIndex(int totalLength, byte[] columnIndexData) { ByteBuffer buffer = ByteBuffer.wrap(columnIndexData); buffer.rewind();/*from w w w . j a va 2s. c om*/ int indexDataLength = buffer.getInt(); short[] indexData = new short[indexDataLength / 2]; short[] indexMap = new short[(totalLength - indexDataLength - CarbonCommonConstants.INT_SIZE_IN_BYTE) / 2]; int counter = 0; while (counter < indexData.length) { indexData[counter] = buffer.getShort(); counter++; } counter = 0; while (buffer.hasRemaining()) { indexMap[counter++] = buffer.getShort(); } return UnBlockIndexer.uncompressIndex(indexData, indexMap); }
From source file:org.apache.carbondata.core.indexstore.blockletindex.BlockDataMap.java
/** * Method to get the relative blocklet ID. Absolute blocklet ID is the blocklet Id as per * task level but relative blocklet ID is id as per carbondata file/block level * * @param absoluteBlockletId//from w w w. j av a 2 s . c o m * @return */ private ExtendedBlocklet createBlockletFromRelativeBlockletId(int absoluteBlockletId) { short relativeBlockletId = -1; int rowIndex = 0; // return 0 if absoluteBlockletId is 0 if (absoluteBlockletId == 0) { relativeBlockletId = (short) absoluteBlockletId; } else { int diff = absoluteBlockletId; ByteBuffer byteBuffer = ByteBuffer.wrap(getBlockletRowCountForEachBlock()); // Example: absoluteBlockletID = 17, blockletRowCountForEachBlock = {4,3,2,5,7} // step1: diff = 17-4, diff = 13 // step2: diff = 13-3, diff = 10 // step3: diff = 10-2, diff = 8 // step4: diff = 8-5, diff = 3 // step5: diff = 3-7, diff = -4 (satisfies <= 0) // step6: relativeBlockletId = -4+7, relativeBlockletId = 3 (4th index starting from 0) while (byteBuffer.hasRemaining()) { short blockletCount = byteBuffer.getShort(); diff = diff - blockletCount; if (diff < 0) { relativeBlockletId = (short) (diff + blockletCount); break; } rowIndex++; } } DataMapRow row = memoryDMStore.getDataMapRow(getFileFooterEntrySchema(), rowIndex); String filePath = getFilePath(); return createBlocklet(row, getFileNameWithFilePath(row, filePath), relativeBlockletId, false); }
From source file:edu.hawaii.soest.kilonalu.tchain.TChainSource.java
/** * A method that executes the streaming of data from the source to the RBNB * server after all configuration of settings, connections to hosts, and * thread initiatizing occurs. This method contains the detailed code for * streaming the data and interpreting the stream. *///from w ww . jav a 2 s. c om protected boolean execute() { logger.debug("TChainSource.execute() called."); // do not execute the stream if there is no connection if (!isConnected()) return false; boolean failed = false; SocketChannel socket = getSocketConnection(); // while data are being sent, read them into the buffer try { // create four byte placeholders used to evaluate up to a four-byte // window. The FIFO layout looks like: // ------------------------- // in ---> | One | Two |Three|Four | ---> out // ------------------------- byte byteOne = 0x00, // set initial placeholder values byteTwo = 0x00, byteThree = 0x00, byteFour = 0x00; // Create a buffer that will store the sample bytes as they are read ByteBuffer sampleBuffer = ByteBuffer.allocate(getBufferSize()); // create a byte buffer to store bytes from the TCP stream ByteBuffer buffer = ByteBuffer.allocateDirect(getBufferSize()); // add a channel of data that will be pushed to the server. // Each sample will be sent to the Data Turbine as an rbnb frame. ChannelMap rbnbChannelMap = new ChannelMap(); // while there are bytes to read from the socket ... while (socket.read(buffer) != -1 || buffer.position() > 0) { // prepare the buffer for reading buffer.flip(); // while there are unread bytes in the ByteBuffer while (buffer.hasRemaining()) { byteOne = buffer.get(); logger.debug("char: " + (char) byteOne + "\t" + "b1: " + new String(Hex.encodeHex((new byte[] { byteOne }))) + "\t" + "b2: " + new String(Hex.encodeHex((new byte[] { byteTwo }))) + "\t" + "b3: " + new String(Hex.encodeHex((new byte[] { byteThree }))) + "\t" + "b4: " + new String(Hex.encodeHex((new byte[] { byteFour }))) + "\t" + "sample pos: " + sampleBuffer.position() + "\t" + "sample rem: " + sampleBuffer.remaining() + "\t" + "sample cnt: " + sampleByteCount + "\t" + "buffer pos: " + buffer.position() + "\t" + "buffer rem: " + buffer.remaining() + "\t" + "state: " + state); // Use a State Machine to process the byte stream. // Start building an rbnb frame for the entire sample, first by // inserting a timestamp into the channelMap. This time is merely // the time of insert into the data turbine, not the time of // observations of the measurements. That time should be parsed out // of the sample in the Sink client code switch (state) { case 0: // sample line ending is '\r\n' (carraige return, newline) // note bytes are in reverse order in the FIFO window if (byteOne == this.firstDelimiterByte && byteTwo == this.secondDelimiterByte) { // we've found the end of a sample, move on state = 1; break; } else { break; } case 1: // read the rest of the bytes to the next EOL characters // sample line is terminated by record delimiter bytes (usually \r\n or \n) // note bytes are in reverse order in the FIFO window if (byteOne == this.firstDelimiterByte && byteTwo == this.secondDelimiterByte) { // rewind the sample to overwrite the line ending so we can add // in the timestamp (then add the line ending) sampleBuffer.position(sampleBuffer.position() - 1); --sampleByteCount; // add the delimiter to the end of the sample. byte[] delimiterAsBytes = getFieldDelimiter().getBytes("US-ASCII"); for (byte delim : delimiterAsBytes) { sampleBuffer.put(delim); sampleByteCount++; } // then add a timestamp to the end of the sample DATE_FORMAT.setTimeZone(TZ); byte[] sampleDateAsBytes = DATE_FORMAT.format(new Date()).getBytes("US-ASCII"); for (byte b : sampleDateAsBytes) { sampleBuffer.put(b); sampleByteCount++; } // add the last two bytes found (usually \r\n) to the sample buffer if (sampleBuffer.remaining() > 0) { sampleBuffer.put(byteOne); sampleByteCount++; sampleBuffer.put(byteTwo); sampleByteCount++; } else { sampleBuffer.compact(); sampleBuffer.put(byteOne); sampleByteCount++; sampleBuffer.put(byteTwo); sampleByteCount++; } // extract just the length of the sample bytes out of the // sample buffer, and place it in the channel map as a // byte array. Then, send it to the data turbine. byte[] sampleArray = new byte[sampleByteCount]; sampleBuffer.flip(); sampleBuffer.get(sampleArray); // send the sample to the data turbine rbnbChannelMap.PutTimeAuto("server"); String sampleString = new String(sampleArray, "US-ASCII"); int channelIndex = rbnbChannelMap.Add(getRBNBChannelName()); rbnbChannelMap.PutMime(channelIndex, "text/plain"); rbnbChannelMap.PutDataAsString(channelIndex, sampleString); getSource().Flush(rbnbChannelMap); logger.info("Sample: " + sampleString.substring(0, sampleString.length() - 2) + " sent data to the DataTurbine. "); byteOne = 0x00; byteTwo = 0x00; byteThree = 0x00; byteFour = 0x00; sampleBuffer.clear(); sampleByteCount = 0; rbnbChannelMap.Clear(); logger.debug("Cleared b1,b2,b3,b4. Cleared sampleBuffer. Cleared rbnbChannelMap."); //state = 0; } else { // not 0x0D20 // still in the middle of the sample, keep adding bytes sampleByteCount++; // add each byte found if (sampleBuffer.remaining() > 0) { sampleBuffer.put(byteOne); } else { sampleBuffer.compact(); logger.debug("Compacting sampleBuffer ..."); sampleBuffer.put(byteOne); } break; } // end if for 0x0D20 EOL } // end switch statement // shift the bytes in the FIFO window byteFour = byteThree; byteThree = byteTwo; byteTwo = byteOne; } //end while (more unread bytes) // prepare the buffer to read in more bytes from the stream buffer.compact(); } // end while (more socket bytes to read) socket.close(); } catch (IOException e) { // handle exceptions // In the event of an i/o exception, log the exception, and allow execute() // to return false, which will prompt a retry. failed = true; e.printStackTrace(); return !failed; } catch (SAPIException sapie) { // In the event of an RBNB communication exception, log the exception, // and allow execute() to return false, which will prompt a retry. failed = true; sapie.printStackTrace(); return !failed; } return !failed; }
From source file:com.koda.integ.hbase.storage.FileExtStorage.java
@Override public void flush() throws IOException { //TODO this method flashes only internal buffer //and does not touch internal flusher queue LOG.info("Flushing internal buffer to the storage"); long start = System.currentTimeMillis(); writeLock.writeLock().lock();// w w w .j av a 2s . c o m try { ByteBuffer buf = activeBuffer.get(); if (bufferOffset.get() == 0) { // skip flush LOG.info("Skipping flush"); return; } if (buf != null) { if (buf.position() != 0) buf.flip(); while (buf.hasRemaining()) { currentForWrite.getChannel().write(buf); } buf.clear(); bufferOffset.set(0); // we advance to next file; } else { LOG.warn("Active buffer is NULL"); } } catch (Exception e) { LOG.error(e); } finally { writeLock.writeLock().unlock(); // Close file currentForWrite.close(); } LOG.info("Flushing completed in " + (System.currentTimeMillis() - start) + "ms"); }
From source file:x10.x10rt.yarn.ApplicationMaster.java
private void processMessage(ByteBuffer msg, SocketChannel sc) { assert (msg.capacity() >= headerLength); msg.rewind(); // reset the buffer for reading from the beginning CTRL_MSG_TYPE type = CTRL_MSG_TYPE.values()[msg.getInt()]; int destination = msg.getInt(); final int source = msg.getInt(); int datalen = msg.getInt(); assert (datalen == msg.remaining()); LOG.info("Processing message of type " + type + ", size " + (headerLength + datalen) + " from place " + source);//w w w . j av a2 s . c om /* System.err.print("Message contents:"); for (int i=0; i<msg.capacity(); i++) System.err.print(" "+Integer.toHexString(msg.get(i)).toUpperCase()); System.err.println(); */ switch (type) { case HELLO: { // read the port information, and update the record for this place assert (datalen == 4 && source < numRequestedContainers.get() && source >= 0); final CommunicationLink linkInfo; LOG.info("Getting link for place " + source); synchronized (links) { linkInfo = links.get(source); } linkInfo.port = ((msg.get() & 0xFF) << 8) | (msg.get() & 0xFF); // unsigned short in network order linkInfo.sc = sc; // check if there are pending port requests for this place if (linkInfo.pendingPortRequests != null) { // prepare response message String linkString = linkInfo.node.getHost() + ":" + linkInfo.port; byte[] linkBytes = linkString.getBytes(); // matches existing code. TODO: switch to UTF8 ByteBuffer response = ByteBuffer.allocateDirect(headerLength + linkBytes.length) .order(ByteOrder.nativeOrder()); response.putInt(CTRL_MSG_TYPE.PORT_RESPONSE.ordinal()); response.putInt(-1); response.putInt(source); response.putInt(linkBytes.length); response.put(linkBytes); // send response to each waiting place for (int place : linkInfo.pendingPortRequests) { response.putInt(4, place); // set the destination to the requesting place response.rewind(); // TODO: this code may get stuck here if the reciever isn't reading properly try { while (response.hasRemaining()) links.get(place).sc.write(response); } catch (IOException e) { LOG.warn("Unable to send out port response for place " + place + " to place " + source, e); } } linkInfo.pendingPortRequests = null; } LOG.info("HELLO from place " + source + " at port " + linkInfo.port); if (pendingKills != null && pendingKills.containsKey(source)) { int delay = pendingKills.remove(source); LOG.info("Scheduling a takedown of place " + source + " in " + delay + " seconds"); placeKiller.schedule(new Runnable() { @Override public void run() { LOG.info("KILLING CONTAINER FOR PLACE " + source); nodeManager.stopContainerAsync(linkInfo.container, linkInfo.node); } }, delay, TimeUnit.SECONDS); } } break; case GOODBYE: { try { CommunicationLink link = links.get(source); assert (link.pendingPortRequests == null); sc.close(); link.port = PORT_DEAD; } catch (IOException e) { LOG.warn("Error closing socket channel", e); } LOG.info("GOODBYE to place " + source); } break; case PORT_REQUEST: { LOG.info("Got PORT_REQUEST from place " + source + " for place " + destination); // check to see if we know the requested information CommunicationLink linkInfo = links.get(destination); if (linkInfo.port != PORT_UNKNOWN) { String linkString; if (linkInfo.port == PORT_DEAD) linkString = DEAD; else linkString = linkInfo.node.getHost() + ":" + linkInfo.port; LOG.info("Telling place " + source + " that place " + destination + " is at " + linkString); byte[] linkBytes = linkString.getBytes(); // matches existing code. TODO: switch to UTF8 ByteBuffer response = ByteBuffer.allocateDirect(headerLength + linkBytes.length) .order(ByteOrder.nativeOrder()); response.putInt(CTRL_MSG_TYPE.PORT_RESPONSE.ordinal()); response.putInt(source); response.putInt(destination); response.putInt(linkBytes.length); response.put(linkBytes); response.rewind(); // TODO: this code may get stuck here if the reciever isn't reading properly try { while (response.hasRemaining()) sc.write(response); } catch (IOException e) { LOG.warn("Unable to send out port response for place " + destination + " to place " + source, e); } } else { // port is not known. remember we have a place asking for it when it becomes available if (linkInfo.pendingPortRequests == null) linkInfo.pendingPortRequests = new ArrayList<Integer>(2); linkInfo.pendingPortRequests.add(source); LOG.info("Stashing PORT_REQUEST from place " + source + " for place " + destination + " until the answer is known"); } } break; case LAUNCH_REQUEST: { assert (datalen == 8); int numPlacesRequested = (int) msg.getLong(); int oldvalue = numRequestedContainers.getAndAdd((int) numPlacesRequested); // Send request for containers to RM for (int i = 0; i < numPlacesRequested; i++) { Resource capability = Resource.newInstance(memoryPerPlaceInMb, coresPerPlace); ContainerRequest request = new ContainerRequest(capability, null, null, Priority.newInstance(0)); LOG.info("Adding a new container request " + request.toString()); resourceManager.addContainerRequest(request); pendingRequests.add(request); } LOG.info("Requested an increase of " + numPlacesRequested + " places on top of the previous " + oldvalue + " places"); msg.rewind(); msg.putInt(CTRL_MSG_TYPE.LAUNCH_RESPONSE.ordinal()); msg.rewind(); try { while (msg.hasRemaining()) sc.write(msg); } catch (IOException e) { LOG.warn("Unable to send out launch response to place " + source, e); } } break; default: LOG.warn("unknown message type " + type); } LOG.info("Finished processing message of size " + (headerLength + datalen) + " from place " + source); }
From source file:org.alfresco.contentstore.ChecksumTest.java
@Test public void test11() throws IOException { try (InputStream in = getClass().getClassLoader().getResourceAsStream("marbles-uncompressed.tif"); InputStream in1 = getClass().getClassLoader().getResourceAsStream("marbles-uncompressed1.tif")) { ByteBuffer buf1 = ByteBuffer.allocate(8192); ByteBuffer buf2 = ByteBuffer.allocate(8192); ReadableByteChannel channel1 = Channels.newChannel(in); ReadableByteChannel channel2 = Channels.newChannel(in1); int numRead1 = -1; int numRead2 = -1; int total = 0; int same = 0; int i = 0; do {//from w w w. j a v a 2 s. c om total = 0; same = 0; numRead1 = channel1.read(buf1); numRead2 = channel2.read(buf2); i += 8192; // buf1.clear(); // buf2.clear(); // // numRead1 = channel1.read(buf1); // numRead2 = channel2.read(buf2); buf1.flip(); buf2.flip(); if (numRead1 > 0 && numRead2 > 0) { if (numRead1 <= numRead2) { while (buf1.hasRemaining()) { total++; byte b1 = buf1.get(); byte b2 = buf2.get(); if (b1 == b2) { same++; } } } else { while (buf2.hasRemaining()) { total++; byte b1 = buf1.get(); byte b2 = buf2.get(); if (b1 == b2) { same++; } } } } buf1.clear(); buf2.clear(); } while (numRead1 > 0 && numRead2 > 0 && same < total); // while(numRead1 > 0 && numRead1 == numRead2); System.out.println(i + ", " + numRead1 + ", " + numRead2 + ", " + total + ", " + same + ", " + (double) same / total); } }
From source file:org.apache.hadoop.hive.ql.exec.tez.DynamicPartitionPruner.java
@SuppressWarnings("deprecation") @VisibleForTesting//from www . j a v a 2 s . c o m protected String processPayload(ByteBuffer payload, String sourceName) throws SerDeException, IOException { DataInputStream in = new DataInputStream(new ByteBufferBackedInputStream(payload)); try { String columnName = in.readUTF(); LOG.info("Source of event: " + sourceName); List<SourceInfo> infos = this.sourceInfoMap.get(sourceName); if (infos == null) { throw new IllegalStateException("no source info for event source: " + sourceName); } SourceInfo info = null; for (SourceInfo si : infos) { if (columnName.equals(si.columnName)) { info = si; break; } } if (info == null) { throw new IllegalStateException("no source info for column: " + columnName); } if (info.skipPruning.get()) { // Marked as skipped previously. Don't bother processing the rest of the payload. } else { boolean skip = in.readBoolean(); if (skip) { info.skipPruning.set(true); } else { while (payload.hasRemaining()) { writable.readFields(in); Object row = info.deserializer.deserialize(writable); Object value = info.soi.getStructFieldData(row, info.field); value = ObjectInspectorUtils.copyToStandardObject(value, info.fieldInspector); if (LOG.isDebugEnabled()) { LOG.debug("Adding: " + value + " to list of required partitions"); } info.values.add(value); } } } } finally { if (in != null) { in.close(); } } return sourceName; }