Example usage for org.apache.hadoop.yarn.client.api.impl ContainerManagementProtocolProxy ContainerManagementProtocolProxy

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api.impl ContainerManagementProtocolProxy ContainerManagementProtocolProxy.

Prototype

public ContainerManagementProtocolProxy(Configuration conf) 

Source Link

Usage

From source file:org.apache.tez.dag.app.launcher.ContainerLauncherImpl.java

License:Apache License

@Override
public void serviceStart() {
    cmProxy = new ContainerManagementProtocolProxy(getConfig());

    ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat("ContainerLauncher #%d").setDaemon(true)
            .build();//  www.ja  va  2 s  .  c om

    // Start with a default core-pool size of 10 and change it dynamically.
    launcherPool = new ThreadPoolExecutor(INITIAL_POOL_SIZE, Integer.MAX_VALUE, 1, TimeUnit.HOURS,
            new LinkedBlockingQueue<Runnable>(), tf, new CustomizedRejectedExecutionHandler());
    eventHandlingThread = new Thread() {
        @Override
        public void run() {
            NMCommunicatorEvent event = null;
            while (!Thread.currentThread().isInterrupted()) {
                try {
                    event = eventQueue.take();
                } catch (InterruptedException e) {
                    if (!serviceStopped.get()) {
                        LOG.error("Returning, interrupted : " + e);
                    }
                    return;
                }
                int poolSize = launcherPool.getCorePoolSize();

                // See if we need up the pool size only if haven't reached the
                // maximum limit yet.
                if (poolSize != limitOnPoolSize) {

                    // nodes where containers will run at *this* point of time. This is
                    // *not* the cluster size and doesn't need to be.
                    int numNodes = context.getNodeTracker().getNumNodes();
                    int idealPoolSize = Math.min(limitOnPoolSize, numNodes);

                    if (poolSize < idealPoolSize) {
                        // Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the
                        // later is just a buffer so we are not always increasing the
                        // pool-size
                        int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize + INITIAL_POOL_SIZE);
                        LOG.info("Setting ContainerLauncher pool size to " + newPoolSize
                                + " as number-of-nodes to talk to is " + numNodes);
                        launcherPool.setCorePoolSize(newPoolSize);
                    }
                }

                // the events from the queue are handled in parallel
                // using a thread pool
                launcherPool.execute(createEventProcessor(event));

                // TODO: Group launching of multiple containers to a single
                // NodeManager into a single connection
            }
        }
    };
    eventHandlingThread.setName("ContainerLauncher Event Handler");
    eventHandlingThread.start();
}

---

From source file:org.apache.tez.dag.app.launcher.TezContainerLauncherImpl.java

License:Apache License

@Override
public void start() {
    // pass a copy of config to ContainerManagementProtocolProxy until YARN-3497 is fixed
    cmProxy = new ContainerManagementProtocolProxy(conf);

    ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat("ContainerLauncher #%d").setDaemon(true)
            .build();/*from   ww  w  .j av a2s. c  om*/

    // Start with a default core-pool size of 10 and change it dynamically.
    launcherPool = new ThreadPoolExecutor(INITIAL_POOL_SIZE, Integer.MAX_VALUE, 1, TimeUnit.HOURS,
            new LinkedBlockingQueue<Runnable>(), tf, new CustomizedRejectedExecutionHandler());
    eventHandlingThread = new Thread() {
        @Override
        public void run() {
            ContainerOp event = null;
            while (!Thread.currentThread().isInterrupted()) {
                try {
                    event = eventQueue.take();
                } catch (InterruptedException e) {
                    if (!serviceStopped.get()) {
                        LOG.error("Returning, interrupted : " + e);
                    }
                    return;
                }
                int poolSize = launcherPool.getCorePoolSize();

                // See if we need up the pool size only if haven't reached the
                // maximum limit yet.
                if (poolSize != limitOnPoolSize) {

                    // nodes where containers will run at *this* point of time. This is
                    // *not* the cluster size and doesn't need to be.
                    int numNodes = getContext().getNumNodes(TezConstants.getTezYarnServicePluginName());
                    int idealPoolSize = Math.min(limitOnPoolSize, numNodes);

                    if (poolSize < idealPoolSize) {
                        // Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the
                        // later is just a buffer so we are not always increasing the
                        // pool-size
                        int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize + INITIAL_POOL_SIZE);
                        LOG.info("Setting ContainerLauncher pool size to " + newPoolSize
                                + " as number-of-nodes to talk to is " + numNodes);
                        launcherPool.setCorePoolSize(newPoolSize);
                    }
                }

                // the events from the queue are handled in parallel
                // using a thread pool
                launcherPool.execute(createEventProcessor(event));

                // TODO: Group launching of multiple containers to a single
                // NodeManager into a single connection
            }
        }
    };
    eventHandlingThread.setName("ContainerLauncher Event Handler");
    eventHandlingThread.start();
}

---