Example usage for org.apache.hadoop.yarn.api.records Resource newInstance

List of usage examples for org.apache.hadoop.yarn.api.records Resource newInstance

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. Resource
  6. newInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Resource newInstance.

Prototype

@Public
    @Stable
    public static Resource newInstance(long memory, int vCores)

Source Link

Usage

From source file:ApplicationMaster.java

License:Apache License

/**
 * Setup the request that will be sent to the RM for the container ask.
 *
 * @return the setup ResourceRequest to be sent to RM
 *///from   w ww  .  j a va 2s.  c o m
private ContainerRequest setupContainerAskForRM() {
    // setup requirements for hosts
    // using * as any host will do for the distributed shell app
    // set the priority for the request
    // TODO - what is the range for priority? how to decide?
    Priority pri = Priority.newInstance(requestPriority);

    // Set up resource type requirements
    // For now, memory and CPU are supported so we set memory and cpu requirements
    Resource capability = Resource.newInstance(containerMemory, containerVirtualCores);

    ContainerRequest request = new ContainerRequest(capability, null, null, pri);
    LOG.info("Requested container ask: " + request.toString());
    return request;
}

From source file:alluxio.yarn.ApplicationMasterTest.java

License:Apache License

/**
 * Returns an argument matcher which matches the expected worker container request for the
 * specified hosts.//  ww  w .j av  a 2s. c om
 *
 * @param hosts the hosts in the container request
 * @return the argument matcher
 */
private ArgumentMatcher<ContainerRequest> getWorkerContainerMatcher(final List<String> hosts) {
    return new ArgumentMatcher<ContainerRequest>() {
        public boolean matches(Object arg) {
            Assert.assertTrue(arg instanceof ContainerRequest);
            ContainerRequest argContainer = (ContainerRequest) arg;
            // Wrap hosts with Sets to ignore ordering
            return argContainer.getCapability()
                    .equals(Resource.newInstance(WORKER_MEM_MB + RAMDISK_MEM_MB, WORKER_CPU))
                    && Sets.newHashSet(argContainer.getNodes()).equals(Sets.newHashSet(hosts))
                    && argContainer.getRacks() == null
                    && argContainer.getPriority().equals(Priority.newInstance(1))
                    && !argContainer.getRelaxLocality();
        }
    };
}

From source file:alluxio.yarn.ApplicationMasterTest.java

License:Apache License

/**
 * Returns an argument matcher which matches the expected master container request.
 *
 * @return the argument matcher//from  ww  w  . j  a v a  2  s .  c o m
 */
private ArgumentMatcher<ContainerRequest> getMasterContainerMatcher() {
    return new ArgumentMatcher<ContainerRequest>() {
        public boolean matches(Object arg) {
            boolean requireLocality = MASTER_ADDRESS.equals("localhost");
            ContainerRequest expectedWorkerContainerRequest = new ContainerRequest(
                    Resource.newInstance(MASTER_MEM_MB, MASTER_CPU), new String[] { MASTER_ADDRESS }, null,
                    Priority.newInstance(0), requireLocality);
            return EqualsBuilder.reflectionEquals(arg, expectedWorkerContainerRequest);
        }
    };
}

From source file:alluxio.yarn.Client.java

License:Apache License

/**
 * Sets up the application submission context.
 *//*from  www . ja  v  a  2s .c  o m*/
private void setupApplicationSubmissionContext() {
    // set the application name
    mAppContext.setApplicationName(mAppName);

    // Set up resource type requirements
    // For now, both memory and vcores are supported, so we set memory and vcores requirements
    Resource capability = Resource.newInstance(mAmMemoryInMB, mAmVCores);
    mAppContext.setResource(capability);

    // Set the queue to which this application is to be submitted in the RM
    mAppContext.setQueue(mAmQueue);

    // Set the AM container spec
    mAppContext.setAMContainerSpec(mAmContainer);

    // Set the priority for the application master
    mAppContext.setPriority(Priority.newInstance(mAmPriority));
}

From source file:alluxio.yarn.ClientTest.java

License:Apache License

@Test
public void notEnoughMemoryForApplicationMaster() throws Exception {
    int appMasterMem = 1024;
    Resource resource = Resource.newInstance(appMasterMem / 2, 4);
    generateMaxAllocation(resource);/*  ww  w .j  av a 2  s.c o  m*/
    mThrown.expect(RuntimeException.class);
    mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("ApplicationMaster", "memory",
            appMasterMem, resource.getMemory()));
    String[] args = new String[] { "-resource_path", "test", "-am_memory", Integer.toString(appMasterMem),
            "-am_vcores", "2" };
    Client client = new Client(args);
    client.run();
}

From source file:alluxio.yarn.ClientTest.java

License:Apache License

@Test
public void notEnoughVCoreForApplicationMaster() throws Exception {
    int appMasterMem = 1024;
    int appMasterCore = 2;
    Resource resource = Resource.newInstance(appMasterMem, appMasterCore / 2);
    generateMaxAllocation(resource);/*from   w  w  w .j  a v a  2s . c om*/
    mThrown.expect(RuntimeException.class);
    mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("ApplicationMaster",
            "virtual cores", appMasterCore, resource.getVirtualCores()));
    String[] args = new String[] { "-resource_path", "test", "-am_memory", Integer.toString(appMasterMem),
            "-am_vcores", Integer.toString(appMasterCore) };
    Client client = new Client(args);
    client.run();
}

From source file:alluxio.yarn.ClientTest.java

License:Apache License

@Test
public void notEnoughMemoryForAlluxioMaster() throws Exception {
    Configuration.set(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM, "2048.00MB");
    Configuration.set(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU, "4");
    int masterMemInMB = (int) (Configuration.getBytes(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM)
            / Constants.MB);/* ww  w  .  ja v  a  2 s.c  o  m*/
    Resource resource = Resource.newInstance(masterMemInMB / 2, 4);
    generateMaxAllocation(resource);
    mThrown.expect(RuntimeException.class);
    mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("Alluxio Master", "memory",
            masterMemInMB, resource.getMemory()));
    Client client = new Client();
    client.run();
}

From source file:alluxio.yarn.ClientTest.java

License:Apache License

@Test
public void notEnoughVCoreForAlluxioMaster() throws Exception {
    Configuration.set(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM, "2048.00MB");
    Configuration.set(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU, "4");
    int masterMemInMB = (int) (Configuration.getBytes(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM)
            / Constants.MB);//from  w  ww . ja v a  2  s  .com
    int masterVCores = Configuration.getInt(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU);
    Resource resource = Resource.newInstance(masterMemInMB, 3);
    generateMaxAllocation(resource);
    mThrown.expect(RuntimeException.class);
    mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("Alluxio Master",
            "virtual cores", masterVCores, resource.getVirtualCores()));
    Client client = new Client();
    client.run();
}

From source file:alluxio.yarn.ClientTest.java

License:Apache License

@Test
public void notEnoughMemoryForAlluxioWorker() throws Exception {
    Configuration.set(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM, "2048.00MB");
    Configuration.set(PropertyKey.WORKER_MEMORY_SIZE, "4096.00MB");
    Configuration.set(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU, "8");
    int workerMemInMB = (int) (Configuration.getBytes(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM)
            / Constants.MB);//from   www .  j a va  2  s .  c  o  m
    int ramdiskMemInMB = (int) (Configuration.getBytes(PropertyKey.WORKER_MEMORY_SIZE) / Constants.MB);
    Resource resource = Resource.newInstance((workerMemInMB + ramdiskMemInMB) / 2, 4);
    generateMaxAllocation(resource);
    mThrown.expect(RuntimeException.class);
    mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("Alluxio Worker", "memory",
            (workerMemInMB + ramdiskMemInMB), resource.getMemory()));
    Client client = new Client();
    client.run();
}

From source file:alluxio.yarn.ClientTest.java

License:Apache License

@Test
public void notEnoughVCoreForAlluxioWorker() throws Exception {
    Configuration.set(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM, "2048.00MB");
    Configuration.set(PropertyKey.WORKER_MEMORY_SIZE, "4096.00MB");
    Configuration.set(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU, "8");
    int workerMemInMB = (int) (Configuration.getBytes(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM)
            / Constants.MB);/*from  w ww.  ja  v  a 2  s  .co  m*/
    int ramdiskMemInMB = (int) (Configuration.getBytes(PropertyKey.WORKER_MEMORY_SIZE) / Constants.MB);
    int workerVCore = Configuration.getInt(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU);
    Resource resource = Resource.newInstance((workerMemInMB + ramdiskMemInMB), 4);
    generateMaxAllocation(resource);
    mThrown.expect(RuntimeException.class);
    mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("Alluxio Worker",
            "virtual cores", workerVCore, resource.getVirtualCores()));
    Client client = new Client();
    client.run();
}