List of usage examples for org.apache.hadoop.yarn.client.api NMTokenCache setToken
@Public @Evolving public void setToken(String nodeAddr, Token token)
From source file:org.apache.hama.bsp.JobImpl.java
License:Apache License
@Override public JobState startJob() throws Exception { this.allocatedContainers = new ArrayList<Container>(numBSPTasks); NMTokenCache nmTokenCache = new NMTokenCache(); while (allocatedContainers.size() < numBSPTasks) { AllocateRequest req = AllocateRequest.newInstance(lastResponseID, 0.0f, createBSPTaskRequest(numBSPTasks - allocatedContainers.size(), taskMemoryInMb, priority), releasedContainers, null); AllocateResponse allocateResponse = resourceManager.allocate(req); for (NMToken token : allocateResponse.getNMTokens()) { nmTokenCache.setToken(token.getNodeId().toString(), token.getToken()); }/*from ww w.j av a 2 s . c om*/ LOG.info("Got response ID: " + allocateResponse.getResponseId() + " with num of containers: " + allocateResponse.getAllocatedContainers().size() + " and following resources: " + allocateResponse.getAvailableResources().getMemory() + "mb"); this.lastResponseID = allocateResponse.getResponseId(); this.allocatedContainers.addAll(allocateResponse.getAllocatedContainers()); LOG.info("Waiting to allocate " + (numBSPTasks - allocatedContainers.size()) + " more containers..."); Thread.sleep(1000l); } LOG.info("Got " + allocatedContainers.size() + " containers!"); int id = 0; for (Container allocatedContainer : allocatedContainers) { LOG.info("Launching task on a new container." + ", containerId=" + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory" + allocatedContainer.getResource().getMemory()); // Connect to ContainerManager on the allocated container String user = conf.get("bsp.user.name"); if (user == null) { user = System.getenv(ApplicationConstants.Environment.USER.name()); } ContainerManagementProtocol cm = null; try { cm = getContainerManagementProtocolProxy(yarnRPC, nmTokenCache.getToken(allocatedContainer.getNodeId().toString()), allocatedContainer.getNodeId(), user); } catch (Exception e) { LOG.error("Failed to create ContainerManager..."); if (cm != null) yarnRPC.stopProxy(cm, conf); e.printStackTrace(); } BSPTaskLauncher runnableLaunchContainer = new BSPTaskLauncher(id, allocatedContainer, cm, conf, jobFile, jobId); launchers.put(id, runnableLaunchContainer); runnableLaunchContainer.start(); completionQueue.add(runnableLaunchContainer); id++; } LOG.info("Waiting for tasks to finish..."); state = JobState.RUNNING; int completed = 0; List<Integer> cleanupTasks = new ArrayList<Integer>(); while (completed != numBSPTasks) { for (BSPTaskLauncher task : completionQueue) { BSPTaskStatus returnedTask = task.poll(); // if our task returned with a finished state if (returnedTask != null) { if (returnedTask.getExitStatus() != 0) { LOG.error("Task with id \"" + returnedTask.getId() + "\" failed!"); cleanupTask(returnedTask.getId()); state = JobState.FAILED; return state; } else { LOG.info("Task \"" + returnedTask.getId() + "\" sucessfully finished!"); completed++; LOG.info("Waiting for " + (numBSPTasks - completed) + " tasks to finish!"); } cleanupTasks.add(returnedTask.getId()); } } Thread.sleep(1000L); } for (Integer stopId : cleanupTasks) { cleanupTask(stopId); } state = JobState.SUCCESS; return state; }