List of usage examples for org.apache.hadoop.yarn.server.utils BuilderUtils newNodeReport
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, String httpAddress, String rackName, Resource used, Resource capability, int numContainers, String healthReport, long lastHealthReportTime)
From source file:com.datatorrent.stram.AffinityRulesTest.java
License:Apache License
@Test public void testOperatorPartitionsAntiAffinity() { LogicalPlan dag = new LogicalPlan(); TestGeneratorInputOperator o1 = dag.addOperator("O1", new TestGeneratorInputOperator()); GenericTestOperator o2 = dag.addOperator("O2", new GenericTestOperator()); GenericTestOperator o3 = dag.addOperator("O3", new GenericTestOperator()); dag.addStream("stream1", o1.outport, o2.inport1); dag.addStream("stream2", o2.outport1, o3.inport1); dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(5)); AffinityRulesSet ruleSet = new AffinityRulesSet(); // Valid case: List<AffinityRule> rules = new ArrayList<>(); ruleSet.setAffinityRules(rules);// w ww . j av a2 s. co m AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O2"); rules.add(rule1); dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet); dag.validate(); dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath()); dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); StreamingContainerManager scm = new StreamingContainerManager(dag); // Check Physical Plan assigns anti-affinity preferences correctly for (ContainerStartRequest csr : scm.containerStartRequests) { PTContainer container = csr.container; // Check that for containers for O2 partitions, anti-affinity Preference // are set properly if (container.getOperators().get(0).getName().equals("O2")) { Assert.assertEquals("Anti-affinity containers set should have 4 containers for other partitions ", 4, container.getStrictAntiPrefs().size()); for (PTContainer c : container.getStrictAntiPrefs()) { for (PTOperator operator : c.getOperators()) { Assert.assertEquals("Partion for O2 should be Anti Prefs", "O2", operator.getName()); } } } } // Check resource handler assigns different hosts for each partition ResourceRequestHandler rr = new ResourceRequestHandler(); int containerMem = 1000; Map<String, NodeReport> nodeReports = Maps.newHashMap(); for (int i = 0; i < 10; i++) { String hostName = "host" + i; NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId(hostName, 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0); nodeReports.put(nr.getNodeId().getHost(), nr); } // set resources rr.updateNodeReports(Lists.newArrayList(nodeReports.values())); Set<String> partitionHostNames = new HashSet<>(); for (ContainerStartRequest csr : scm.containerStartRequests) { String host = rr.getHost(csr, true); csr.container.host = host; if (csr.container.getOperators().get(0).getName().equals("O2")) { Assert.assertNotNull("Host name should not be null", host); LOG.info("Partition {} for operator O2 has host = {} ", csr.container.getId(), host); Assert.assertTrue("Each Partition should have a different host", !partitionHostNames.contains(host)); partitionHostNames.add(host); } } }
From source file:com.datatorrent.stram.AffinityRulesTest.java
License:Apache License
@Test public void testAntiAffinityInOperators() { LogicalPlan dag = new LogicalPlan(); dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath()); dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); GenericTestOperator o1 = dag.addOperator("O1", GenericTestOperator.class); dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256); GenericTestOperator o2 = dag.addOperator("O2", GenericTestOperator.class); dag.setOperatorAttribute(o2, OperatorContext.MEMORY_MB, 256); dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1"); AffinityRulesSet ruleSet = new AffinityRulesSet(); List<AffinityRule> rules = new ArrayList<>(); ruleSet.setAffinityRules(rules);/*from www .j a v a 2 s. c o m*/ AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O2"); rules.add(rule1); dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet); dag.addStream("o1_outport1", o1.outport1, o2.inport1);// .setLocality(Locality.NODE_LOCAL); StreamingContainerManager scm = new StreamingContainerManager(dag); ResourceRequestHandler rr = new ResourceRequestHandler(); int containerMem = 1000; Map<String, NodeReport> nodeReports = Maps.newHashMap(); NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0); nodeReports.put(nr.getNodeId().getHost(), nr); nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0); nodeReports.put(nr.getNodeId().getHost(), nr); // set resources rr.updateNodeReports(Lists.newArrayList(nodeReports.values())); for (ContainerStartRequest csr : scm.containerStartRequests) { String host = rr.getHost(csr, true); csr.container.host = host; if (csr.container.getOperators().get(0).getName().equals("O1")) { Assert.assertEquals("Hosts set to host1 for Operator O1", "host1", host); } if (csr.container.getOperators().get(0).getName().equals("O2")) { Assert.assertEquals("Hosts set to host2 for Operator O2", "host2", host); } } }
From source file:com.datatorrent.stram.HostLocalTest.java
License:Apache License
@Test public void testPartitionLocality() { int partitionCount = 3; LogicalPlan dag = new LogicalPlan(); dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath()); dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class); GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class); LocalityPartitioner partitioner = new LocalityPartitioner(); partitioner.setPartitionCount(partitionCount); dag.getMeta(partitioned).getAttributes().put(OperatorContext.PARTITIONER, partitioner); dag.addStream("o1_outport1", o1.outport1, partitioned.inport1); StreamingContainerManager scm = new StreamingContainerManager(dag); ResourceRequestHandler rr = new ResourceRequestHandler(); int containerMem = 1000; Map<String, NodeReport> nodeReports = Maps.newHashMap(); for (int i = 0; i < partitionCount; i++) { NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host" + (i + 1), 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0); nodeReports.put(nr.getNodeId().getHost(), nr); }/*from w ww . j a v a 2 s . co m*/ // set resources rr.updateNodeReports(Lists.newArrayList(nodeReports.values())); Set<String> expectedHosts = Sets.newHashSet(); for (int i = 0; i < partitionCount; i++) { expectedHosts.add("host" + (i + 1)); } for (ContainerStartRequest csr : scm.containerStartRequests) { String host = rr.getHost(csr, true); if (host != null) { expectedHosts.remove(host); } } Assert.assertTrue("All the allocated hosts removed", expectedHosts.isEmpty()); }
From source file:com.datatorrent.stram.HostLocalTest.java
License:Apache License
@Test public void testNodeLocal() { LogicalPlan dag = new LogicalPlan(); dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath()); dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class); dag.setAttribute(o1, OperatorContext.MEMORY_MB, 256); GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class); dag.setAttribute(partitioned, OperatorContext.MEMORY_MB, 256); dag.getMeta(partitioned).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1"); dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.NODE_LOCAL); StreamingContainerManager scm = new StreamingContainerManager(dag); ResourceRequestHandler rr = new ResourceRequestHandler(); int containerMem = 1000; Map<String, NodeReport> nodeReports = Maps.newHashMap(); NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0); nodeReports.put(nr.getNodeId().getHost(), nr); nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);/*from w ww .j ava2 s .com*/ nodeReports.put(nr.getNodeId().getHost(), nr); // set resources rr.updateNodeReports(Lists.newArrayList(nodeReports.values())); for (ContainerStartRequest csr : scm.containerStartRequests) { String host = rr.getHost(csr, true); csr.container.host = host; Assert.assertEquals("Hosts set to host1", "host1", host); } }
From source file:com.datatorrent.stram.HostLocalTest.java
License:Apache License
@Test public void testThreadLocal() { LogicalPlan dag = new LogicalPlan(); dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath()); dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class); dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2"); GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class); dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.THREAD_LOCAL); dag.setAttribute(o1, OperatorContext.MEMORY_MB, 256); dag.setAttribute(partitioned, OperatorContext.MEMORY_MB, 256); StreamingContainerManager scm = new StreamingContainerManager(dag); ResourceRequestHandler rr = new ResourceRequestHandler(); int containerMem = 1000; Map<String, NodeReport> nodeReports = Maps.newHashMap(); NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0); nodeReports.put(nr.getNodeId().getHost(), nr); nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);// w w w.jav a 2s .c o m nodeReports.put(nr.getNodeId().getHost(), nr); // set resources rr.updateNodeReports(Lists.newArrayList(nodeReports.values())); Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size()); for (ContainerStartRequest csr : scm.containerStartRequests) { String host = rr.getHost(csr, true); csr.container.host = host; Assert.assertEquals("Hosts set to host2", "host2", host); } }
From source file:com.datatorrent.stram.HostLocalTest.java
License:Apache License
@Test public void testContainerLocal() { LogicalPlan dag = new LogicalPlan(); dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath()); dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class); dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2"); GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class); dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL); dag.setAttribute(o1, OperatorContext.MEMORY_MB, 256); dag.setAttribute(partitioned, OperatorContext.MEMORY_MB, 256); StreamingContainerManager scm = new StreamingContainerManager(dag); ResourceRequestHandler rr = new ResourceRequestHandler(); int containerMem = 1000; Map<String, NodeReport> nodeReports = Maps.newHashMap(); NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0); nodeReports.put(nr.getNodeId().getHost(), nr); nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);//from w w w . j a va 2s. com nodeReports.put(nr.getNodeId().getHost(), nr); // set resources rr.updateNodeReports(Lists.newArrayList(nodeReports.values())); Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size()); for (ContainerStartRequest csr : scm.containerStartRequests) { String host = rr.getHost(csr, true); csr.container.host = host; Assert.assertEquals("Hosts set to host2", "host2", host); } }
From source file:com.datatorrent.stram.HostLocalTest.java
License:Apache License
@Test public void testContainerLocalWithVCores() { LogicalPlan dag = new LogicalPlan(); dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath()); dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class); dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2"); GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class); dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL); dag.setAttribute(o1, OperatorContext.MEMORY_MB, 256); dag.setAttribute(o1, OperatorContext.VCORES, 1); dag.setAttribute(partitioned, OperatorContext.VCORES, 1); StreamingContainerManager scm = new StreamingContainerManager(dag); ResourceRequestHandler rr = new ResourceRequestHandler(); int containerMem = 1000; Map<String, NodeReport> nodeReports = Maps.newHashMap(); NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0); nodeReports.put(nr.getNodeId().getHost(), nr); nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);//w ww . ja va 2 s .c om nodeReports.put(nr.getNodeId().getHost(), nr); // set resources rr.updateNodeReports(Lists.newArrayList(nodeReports.values())); Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size()); for (ContainerStartRequest csr : scm.containerStartRequests) { String host = rr.getHost(csr, true); csr.container.host = host; Assert.assertEquals("number of vcores", 2, csr.container.getRequiredVCores()); Assert.assertEquals("Hosts set to host2", "host2", host); } }
From source file:com.datatorrent.stram.HostLocalTest.java
License:Apache License
@Test public void testUnavailableResources() { LogicalPlan dag = new LogicalPlan(); dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath()); dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class); dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2"); GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class); dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL); dag.setAttribute(o1, OperatorContext.MEMORY_MB, 256); dag.setAttribute(o1, OperatorContext.VCORES, 2); dag.setAttribute(partitioned, OperatorContext.VCORES, 1); StreamingContainerManager scm = new StreamingContainerManager(dag); ResourceRequestHandler rr = new ResourceRequestHandler(); int containerMem = 1000; Map<String, NodeReport> nodeReports = Maps.newHashMap(); NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0); nodeReports.put(nr.getNodeId().getHost(), nr); nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);/*from w w w .ja v a 2 s . c o m*/ nodeReports.put(nr.getNodeId().getHost(), nr); // set resources rr.updateNodeReports(Lists.newArrayList(nodeReports.values())); Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size()); for (ContainerStartRequest csr : scm.containerStartRequests) { String host = rr.getHost(csr, true); Assert.assertEquals("number of vcores", 3, csr.container.getRequiredVCores()); Assert.assertNull("Host is null", host); } }
From source file:com.datatorrent.stram.LocalityTest.java
License:Apache License
@Test public void testNodeLocal() { LogicalPlan dag = new LogicalPlan(); dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", LocalityTest.class.getName()).getAbsolutePath()); dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class); GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class); dag.getMeta(partitioned).getAttributes().put(OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2)); GenericTestOperator partitionedParallel = dag.addOperator("partitionedParallel", GenericTestOperator.class); dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(null); dag.addStream("partitioned_outport1", partitioned.outport1, partitionedParallel.inport2) .setLocality(Locality.NODE_LOCAL); dag.setInputPortAttribute(partitionedParallel.inport2, PortContext.PARTITION_PARALLEL, true); GenericTestOperator single = dag.addOperator("single", GenericTestOperator.class); dag.addStream("partitionedParallel_outport1", partitionedParallel.outport1, single.inport1); int maxContainers = 7; dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers); StreamingContainerManager scm = new StreamingContainerManager(dag); Assert.assertEquals("number required containers", 7, scm.containerStartRequests.size()); ResourceRequestHandler rr = new ResourceRequestHandler(); int containerMem = 2000; Map<String, NodeReport> nodeReports = Maps.newHashMap(); NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0); nodeReports.put(nr.getNodeId().getHost(), nr); nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);/*from w w w .jav a 2 s .c om*/ nodeReports.put(nr.getNodeId().getHost(), nr); // set resources rr.updateNodeReports(Lists.newArrayList(nodeReports.values())); Map<PTContainer, String> requestedHosts = Maps.newHashMap(); for (ContainerStartRequest csr : scm.containerStartRequests) { String host = rr.getHost(csr, true); csr.container.host = host; // update the node report if (host != null) { requestedHosts.put(csr.container, host); nr = nodeReports.get(host); nr.getUsed().setMemory(nr.getUsed().getMemory() + containerMem); } } Assert.assertEquals("" + requestedHosts, nodeReports.keySet(), Sets.newHashSet(requestedHosts.values())); for (Map.Entry<PTContainer, String> e : requestedHosts.entrySet()) { for (PTOperator oper : e.getKey().getOperators()) { if (oper.getNodeLocalOperators().getOperatorSet().size() > 1) { String expHost = null; for (PTOperator nodeLocalOper : oper.getNodeLocalOperators().getOperatorSet()) { Assert.assertNotNull("host null " + nodeLocalOper.getContainer(), nodeLocalOper.getContainer().host); if (expHost == null) { expHost = nodeLocalOper.getContainer().host; } else { Assert.assertEquals("expected same host " + nodeLocalOper, expHost, nodeLocalOper.getContainer().host); } } } } } }