List of usage examples for org.apache.hadoop.yarn.api.records Container newInstance
@Private
@Unstable
public static Container newInstance(ContainerId containerId, NodeId nodeId, String nodeHttpAddress,
Resource resource, Priority priority, Token containerToken)
From source file:com.datatorrent.stram.StreamingAppMasterService.java
License:Apache License
/** * Check for containers that were allocated in a previous attempt. * If the containers are still alive, wait for them to check in via heartbeat. *///from ww w . j a v a2 s .c om private void checkContainerStatus() { Collection<StreamingContainerAgent> containers = this.dnmgr.getContainerAgents(); for (StreamingContainerAgent ca : containers) { ContainerId containerId = ConverterUtils.toContainerId(ca.container.getExternalId()); NodeId nodeId = ConverterUtils.toNodeId(ca.container.host); // put container back into the allocated list org.apache.hadoop.yarn.api.records.Token containerToken = null; Resource resource = Resource.newInstance(ca.container.getAllocatedMemoryMB(), ca.container.getAllocatedVCores()); Priority priority = Priority.newInstance(ca.container.getResourceRequestPriority()); Container yarnContainer = Container.newInstance(containerId, nodeId, ca.container.nodeHttpAddress, resource, priority, containerToken); this.allocatedContainers.put(containerId.toString(), new AllocatedContainer(yarnContainer)); // check the status nmClient.getContainerStatusAsync(containerId, nodeId); } }
From source file:de.huberlin.wbi.hiway.scheduler.TestScheduler.java
License:Apache License
private static void run(Scheduler scheduler, List<String> nodeNames, List<String> taskNames, int[][] runtimes) { Queue<NodeId> availableNodes = new LinkedList<>(); for (String nodeName : nodeNames) { NodeId node = NodeId.newInstance(nodeName, 0); availableNodes.add(node);/*from w w w . j a va2 s .co m*/ } Map<Container, TaskInstance> runningTasks = new HashMap<>(); Map<Container, Long> finishTimes = new HashMap<>(); int runningId = 0; long clock = 0; while (!scheduler.nothingToSchedule() || !runningTasks.isEmpty()) { if (!scheduler.nothingToSchedule() && !availableNodes.isEmpty()) { NodeId nodeId = availableNodes.remove(); ContainerId containerId = ContainerId.newContainerId(null, runningId++); Container container = Container.newInstance(containerId, nodeId, "", null, null, null); TaskInstance task = scheduler.getNextTask(container); runningTasks.put(container, task); long runtime = (runtimes == null) ? 1 : runtimes[nodeNames.indexOf(nodeId.getHost())][taskNames.indexOf(task.getTaskName())]; finishTimes.put(container, clock + runtime); } for (Container container : finishTimes.keySet()) { if (clock == finishTimes.get(container)) { NodeId nodeId = container.getNodeId(); ContainerStatus containerStatus = ContainerStatus.newInstance(container.getId(), null, "", 0); TaskInstance task = runningTasks.get(container); task.setCompleted(); long runtime = (runtimes == null) ? 1 : runtimes[nodeNames.indexOf(nodeId.getHost())][taskNames.indexOf(task.getTaskName())]; scheduler.taskCompleted(task, containerStatus, runtime); runningTasks.remove(container); availableNodes.add(nodeId); } } clock++; } }
From source file:org.apache.gobblin.yarn.YarnServiceTest.java
License:Apache License
@Test(groups = { "gobblin.yarn", "disabledOnTravis" }, dependsOnMethods = "testReleasedContainerCache")
public void testBuildContainerCommand() throws Exception {
Config modifiedConfig = this.config
.withValue(GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY,
ConfigValueFactory.fromAnyRef("10"))
.withValue(GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY,
ConfigValueFactory.fromAnyRef("0.8"));
TestYarnService yarnService = new TestYarnService(modifiedConfig, "testApp2", "appId2", this.clusterConf,
FileSystem.getLocal(new Configuration()), this.eventBus);
ContainerId containerId = ContainerId
.newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 0), 0), 0);
Resource resource = Resource.newInstance(2048, 1);
Container container = Container.newInstance(containerId, null, null, resource, null, null);
String command = yarnService.buildContainerCommand(container, "helixInstance1");
// 1628 is from 2048 * 0.8 - 10
Assert.assertTrue(command.contains("-Xmx1628"));
}
From source file:org.apache.ignite.yarn.IgniteApplicationMasterSelfTest.java
License:Apache License
/** * @param host Host./* w w w. j av a 2 s . c o m*/ * @param cpu Cpu count. * @param mem Memory. * @return Container. */ private Container createContainer(String host, int cpu, int mem) { return Container.newInstance( ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0), ThreadLocalRandom.current().nextLong()), NodeId.newInstance(host, 0), "example.com", new MockResource(mem, cpu), Priority.newInstance(0), null); }
From source file:org.apache.myriad.TestObjectFactory.java
License:Apache License
/** * Returns a new RMContainer corresponding to the RMNode and RMContext. The RMContainer is the * ResourceManager's view of an application container per the Hadoop docs * /*from w ww. ja v a 2 s . c o m*/ * @param node * @param context * @param appId * @param cores * @param memory * @return RMContainer */ public static RMContainer getRMContainer(RMNode node, RMContext context, int appId, int cores, int memory) { ContainerId containerId = ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(123456789, 1), 1), appId); Container container = Container.newInstance(containerId, node.getNodeID(), node.getHttpAddress(), Resources.createResource(memory, cores), null, null); return new RMContainerImpl(container, containerId.getApplicationAttemptId(), node.getNodeID(), "user1", context); }
From source file:org.apache.samza.job.yarn.TestYarnClusterResourceManager.java
License:Apache License
@Test public void testErrorInStartContainerShouldUpdateState() { // create mocks final int samzaContainerId = 1; YarnConfiguration yarnConfiguration = mock(YarnConfiguration.class); SamzaAppMasterMetrics metrics = mock(SamzaAppMasterMetrics.class); Config config = mock(Config.class); AMRMClientAsync asyncClient = mock(AMRMClientAsync.class); YarnAppState yarnAppState = new YarnAppState(0, mock(ContainerId.class), "host", 8080, 8081); SamzaYarnAppMasterLifecycle lifecycle = mock(SamzaYarnAppMasterLifecycle.class); SamzaYarnAppMasterService service = mock(SamzaYarnAppMasterService.class); NMClientAsync asyncNMClient = mock(NMClientAsync.class); ClusterResourceManager.Callback callback = mock(ClusterResourceManager.Callback.class); // start the cluster manager YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient, callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config); yarnAppState.pendingProcessors.put(String.valueOf(samzaContainerId), new YarnContainer(Container.newInstance( ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(10000l, 1), 1), 1), NodeId.newInstance("host1", 8088), "http://host1", Resource.newInstance(1024, 1), Priority.newInstance(1), Token.newInstance("id".getBytes(), "read", "password".getBytes(), "service")))); yarnClusterResourceManager.start();//w w w .java 2s. c om assertEquals(1, yarnAppState.pendingProcessors.size()); yarnClusterResourceManager.onStartContainerError( ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(10000l, 1), 1), 1), new Exception()); assertEquals(0, yarnAppState.pendingProcessors.size()); verify(callback, times(1)).onStreamProcessorLaunchFailure(anyObject(), any(Exception.class)); }
From source file:org.apache.samza.webapp.TestApplicationMasterRestClient.java
License:Apache License
private YarnAppState createYarnAppState(ContainerId containerId) throws MalformedURLException { YarnAppState yarnAppState = new YarnAppState(2, containerId, AM_HOST_NAME, AM_RPC_PORT, AM_HTTP_PORT); yarnAppState.rpcUrl = new URL(new HttpHost(AM_HOST_NAME, AM_RPC_PORT).toURI()); yarnAppState.runningProcessors.put("0", new YarnContainer(Container.newInstance(ConverterUtils.toContainerId(YARN_CONTAINER_ID_2), ConverterUtils.toNodeIdWithDefaultPort("dummyNodeHost1"), "dummyNodeHttpHost1", null, null, null)));/*from w w w . j ava 2s.c om*/ yarnAppState.runningProcessors.put("1", new YarnContainer(Container.newInstance(ConverterUtils.toContainerId(YARN_CONTAINER_ID_3), ConverterUtils.toNodeIdWithDefaultPort("dummyNodeHost2"), "dummyNodeHttpHost2", null, null, null))); return yarnAppState; }
From source file:org.apache.tez.dag.app.rm.container.TestAMContainerMap.java
License:Apache License
private Container mockContainer(ContainerId containerId) { NodeId nodeId = NodeId.newInstance("localhost", 43255); Container container = Container.newInstance(containerId, nodeId, "localhost:33333", Resource.newInstance(1024, 1), Priority.newInstance(1), mock(Token.class)); return container; }
From source file:org.apache.tez.dag.app.rm.ContainerFactory.java
License:Apache License
public Container createContainer(Resource capability, Priority priority, String hostname, int port) { ContainerId containerId = ContainerId.newContainerId(customAppAttemptId, nextId.getAndIncrement()); NodeId nodeId = NodeId.newInstance(hostname, port); String nodeHttpAddress = "hostname:0"; // TODO: include UI ports Container container = Container.newInstance(containerId, nodeId, nodeHttpAddress, capability, priority, null);//ww w . j a v a 2 s. c o m return container; }
From source file:org.apache.tez.dag.app.rm.TestContainerReuse.java
License:Apache License
private Container createContainer(int id, String host, Resource resource, Priority priority) { ContainerId containerID = ContainerId .newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), id); NodeId nodeID = NodeId.newInstance(host, 0); Container container = Container.newInstance(containerID, nodeID, host + ":0", resource, priority, null); return container; }