List of usage examples for io.netty.buffer PooledByteBufAllocator PooledByteBufAllocator
@Deprecated public PooledByteBufAllocator(boolean preferDirect, int nHeapArena, int nDirectArena, int pageSize, int maxOrder)
From source file:io.urmia.st.Main.java
License:Open Source License
public static void main(String[] args) throws Exception { final int port; final String base; boolean autoRegister = ArgumentParseUtil.isAutoRegister(args); String zkURL = ArgumentParseUtil.getZooKeeperURL(args); log.info("starting with zk at: {}, auto register: {}", zkURL, autoRegister); ns = new ZkNamingServiceImpl(zkURL, AZ); Optional<ServiceInstance<NodeType>> meOpt = ns.whoAmI(NodeType.ODS, autoRegister); if (!meOpt.isPresent()) { System.err.println("unable to find my instance. use auto register or cli-admin to add my node"); System.exit(1);// w w w . j av a 2s . co m return; } Runtime.getRuntime().addShutdownHook(new ShutdownHook()); EventLoopGroup bossGroup = new NioEventLoopGroup(/*1*/); //EventLoopGroup bossGroup = new EpollEventLoopGroup(1); //EventLoopGroup workerGroup = new NioEventLoopGroup(); try { me = meOpt.get(); log.info("my service instance: {}", me); ns.register(me); base = me.getUriSpec().getParts().get(0).getValue(); port = me.getPort(); if (!(new File(base).isDirectory())) { System.err.println("base in not directory: " + base); return; } int nHeapArena = 1; int nDirectArena = 1; int pageSize = /*8192*/4096; int maxOrder = 1; // http://normanmaurer.me/presentations/2014-facebook-eng-netty/slides.html#14.0 ServerBootstrap b = new ServerBootstrap(); b.group(bossGroup).channel(NioServerSocketChannel.class).childOption(ChannelOption.AUTO_READ, false) .childOption(ChannelOption.ALLOCATOR, new PooledByteBufAllocator(true, nHeapArena, nDirectArena, pageSize, maxOrder)) .childHandler(new HttpUploadServerInitializer(base)); Channel ch = b.bind(port).sync().channel(); log.info("object storage Server (ODS) at port: {}", port); System.err.println("starting ODS " + me.getId() + " on port: " + port + ", base: " + base); ch.closeFuture().sync(); } finally { bossGroup.shutdownGracefully(); //workerGroup.shutdownGracefully(); } }
From source file:org.apache.flink.runtime.io.network.netty.NettyBufferPool.java
License:Apache License
/** * Creates Netty's buffer pool with the specified number of direct arenas. * * @param numberOfArenas Number of arenas (recommended: 2 * number of task * slots)//from www.j av a2 s . c o m */ NettyBufferPool(int numberOfArenas) { checkArgument(numberOfArenas >= 1, "Number of arenas"); this.numberOfArenas = numberOfArenas; if (!PlatformDependent.hasUnsafe()) { LOG.warn("Using direct buffers, but sun.misc.Unsafe not available."); } // We strictly prefer direct buffers and disallow heap allocations. boolean preferDirect = true; // Arenas allocate chunks of pageSize << maxOrder bytes. With these // defaults, this results in chunks of 16 MB. int pageSize = 8192; int maxOrder = 11; this.chunkSize = pageSize << maxOrder; // Number of direct arenas. Each arena allocates a chunk of 16 MB, i.e. // we allocate numDirectArenas * 16 MB of direct memory. This can grow // to multiple chunks per arena during runtime, but this should only // happen with a large amount of connections per task manager. We // control the memory allocations with low/high watermarks when writing // to the TCP channels. Chunks are allocated lazily. int numDirectArenas = numberOfArenas; // No heap arenas, please. int numHeapArenas = 0; this.alloc = new PooledByteBufAllocator(preferDirect, numHeapArenas, numDirectArenas, pageSize, maxOrder); Object[] allocDirectArenas = null; try { Field directArenasField = alloc.getClass().getDeclaredField("directArenas"); directArenasField.setAccessible(true); allocDirectArenas = (Object[]) directArenasField.get(alloc); } catch (Exception ignored) { LOG.warn("Memory statistics not available"); } finally { this.directArenas = allocDirectArenas; } }
From source file:org.apache.flink.runtime.io.network.netty.NettyConnectionManager.java
License:Apache License
@Override public void start(ChannelManager channelManager) throws IOException { LOG.info(String.format("Starting with %d incoming and %d outgoing connection threads.", numInThreads, numOutThreads));//w w w. ja v a 2s . c o m LOG.info(String.format("Setting low water mark to %d and high water mark to %d bytes.", lowWaterMark, highWaterMark)); LOG.info(String.format("Close channels after idle for %d ms.", closeAfterIdleForMs)); final BufferProviderBroker bufferProviderBroker = channelManager; final EnvelopeDispatcher envelopeDispatcher = channelManager; int numHeapArenas = 0; int numDirectArenas = numInThreads + numOutThreads; int pageSize = bufferSize << 1; int chunkSize = 16 << 20; // 16 MB // shift pageSize maxOrder times to get to chunkSize int maxOrder = (int) (Math.log(chunkSize / pageSize) / Math.log(2)); PooledByteBufAllocator pooledByteBufAllocator = new PooledByteBufAllocator(true, numHeapArenas, numDirectArenas, pageSize, maxOrder); String msg = String.format( "Instantiated PooledByteBufAllocator with direct arenas: %d, heap arenas: %d, " + "page size (bytes): %d, chunk size (bytes): %d.", numDirectArenas, numHeapArenas, pageSize, (pageSize << maxOrder)); LOG.info(msg); // -------------------------------------------------------------------- // server bootstrap (incoming connections) // -------------------------------------------------------------------- in = new ServerBootstrap(); in.group(new NioEventLoopGroup(numInThreads)).channel(NioServerSocketChannel.class) .localAddress(bindAddress, bindPort).childHandler(new ChannelInitializer<SocketChannel>() { @Override public void initChannel(SocketChannel channel) throws Exception { channel.pipeline().addLast(new InboundEnvelopeDecoder(bufferProviderBroker)) .addLast(new InboundEnvelopeDispatcher(envelopeDispatcher)); } }).option(ChannelOption.RCVBUF_ALLOCATOR, new FixedRecvByteBufAllocator(pageSize)) .option(ChannelOption.ALLOCATOR, pooledByteBufAllocator); // -------------------------------------------------------------------- // client bootstrap (outgoing connections) // -------------------------------------------------------------------- out = new Bootstrap(); out.group(new NioEventLoopGroup(numOutThreads)).channel(NioSocketChannel.class) .handler(new ChannelInitializer<SocketChannel>() { @Override public void initChannel(SocketChannel channel) throws Exception { channel.pipeline().addLast(new OutboundEnvelopeEncoder()); } }).option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, lowWaterMark) .option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, highWaterMark) .option(ChannelOption.ALLOCATOR, pooledByteBufAllocator).option(ChannelOption.TCP_NODELAY, false) .option(ChannelOption.SO_KEEPALIVE, true); try { in.bind().sync(); } catch (InterruptedException e) { throw new IOException(e); } if (LOG.isDebugEnabled()) { new Thread(new Runnable() { @Override public void run() { Date date = new Date(); while (true) { try { Thread.sleep(DEBUG_PRINT_QUEUED_ENVELOPES_EVERY_MS); date.setTime(System.currentTimeMillis()); System.out.println(date); System.out.println(getNonZeroNumQueuedEnvelopes()); } catch (InterruptedException e) { e.printStackTrace(); } } } }).start(); } }