List of usage examples for org.apache.hadoop.yarn.api.records Container getNodeHttpAddress
@Public @Stable public abstract String getNodeHttpAddress();
From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java
License:Apache License
/** * For each container successfully allocated, attempt to set up and launch a Guagua worker/master task. * //from w w w . j a va 2s. co m * @param allocatedContainers * the containers we have currently allocated. */ private void startContainerLaunchingThreads(final List<Container> allocatedContainers) { Map<String, List<Container>> hostContainterMap = getHostContainersMap(allocatedContainers); int size = allocatedContainers.size(); while (size > 0) { int currentPartition = getCurrentPartition(); if (currentPartition == -1) { LOG.warn("Request too many resources. TODO, remove containers no needed."); for (Container container : allocatedContainers) { GuaguaAppMaster.this.getAmRMClient().releaseAssignedContainer(container.getId()); } break; } Container container = getDataLocalityContainer(hostContainterMap, currentPartition); if (container == null) { container = allocatedContainers.get(0); } allocatedContainers.remove(container); LOG.info( "Launching command on a new container., containerId={}, containerNode={}, containerPort={}, containerNodeURI={}, containerResourceMemory={}", container.getId(), container.getNodeId().getHost(), container.getNodeId().getPort(), container.getNodeHttpAddress(), container.getResource().getMemory()); List<Container> list = this.partitionContainerMap.get(currentPartition); if (list == null) { list = new ArrayList<Container>(); } list.add(container); this.partitionContainerMap.put(currentPartition, list); this.containerPartitionMap.put(container.getId().toString(), currentPartition); this.partitionStatusMap.put(currentPartition, PartitionStatus.INIT); LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(container, getContainerListener(), currentPartition); getExecutor().execute(runnableLaunchContainer); size = allocatedContainers.size(); } }
From source file:org.apache.giraph.yarn.GiraphApplicationMaster.java
License:Apache License
/** * For each container successfully allocated, attempt to set up and launch * a Giraph worker/master task./*www. j a v a 2 s. com*/ * @param allocatedContainers the containers we have currently allocated. */ private void startContainerLaunchingThreads(final List<Container> allocatedContainers) { for (Container allocatedContainer : allocatedContainers) { LOG.info("Launching command on a new container." + ", containerId=" + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory=" + allocatedContainer.getResource().getMemory()); // Launch and start the container on a separate thread to keep the main // thread unblocked as all containers may not be allocated at one go. LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(allocatedContainer, containerListener); executor.execute(runnableLaunchContainer); } }
From source file:org.apache.hama.bsp.JobImpl.java
License:Apache License
@Override public JobState startJob() throws Exception { this.allocatedContainers = new ArrayList<Container>(numBSPTasks); NMTokenCache nmTokenCache = new NMTokenCache(); while (allocatedContainers.size() < numBSPTasks) { AllocateRequest req = AllocateRequest.newInstance(lastResponseID, 0.0f, createBSPTaskRequest(numBSPTasks - allocatedContainers.size(), taskMemoryInMb, priority), releasedContainers, null); AllocateResponse allocateResponse = resourceManager.allocate(req); for (NMToken token : allocateResponse.getNMTokens()) { nmTokenCache.setToken(token.getNodeId().toString(), token.getToken()); }/*from ww w . j a va 2s .c om*/ LOG.info("Got response ID: " + allocateResponse.getResponseId() + " with num of containers: " + allocateResponse.getAllocatedContainers().size() + " and following resources: " + allocateResponse.getAvailableResources().getMemory() + "mb"); this.lastResponseID = allocateResponse.getResponseId(); this.allocatedContainers.addAll(allocateResponse.getAllocatedContainers()); LOG.info("Waiting to allocate " + (numBSPTasks - allocatedContainers.size()) + " more containers..."); Thread.sleep(1000l); } LOG.info("Got " + allocatedContainers.size() + " containers!"); int id = 0; for (Container allocatedContainer : allocatedContainers) { LOG.info("Launching task on a new container." + ", containerId=" + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory" + allocatedContainer.getResource().getMemory()); // Connect to ContainerManager on the allocated container String user = conf.get("bsp.user.name"); if (user == null) { user = System.getenv(ApplicationConstants.Environment.USER.name()); } ContainerManagementProtocol cm = null; try { cm = getContainerManagementProtocolProxy(yarnRPC, nmTokenCache.getToken(allocatedContainer.getNodeId().toString()), allocatedContainer.getNodeId(), user); } catch (Exception e) { LOG.error("Failed to create ContainerManager..."); if (cm != null) yarnRPC.stopProxy(cm, conf); e.printStackTrace(); } BSPTaskLauncher runnableLaunchContainer = new BSPTaskLauncher(id, allocatedContainer, cm, conf, jobFile, jobId); launchers.put(id, runnableLaunchContainer); runnableLaunchContainer.start(); completionQueue.add(runnableLaunchContainer); id++; } LOG.info("Waiting for tasks to finish..."); state = JobState.RUNNING; int completed = 0; List<Integer> cleanupTasks = new ArrayList<Integer>(); while (completed != numBSPTasks) { for (BSPTaskLauncher task : completionQueue) { BSPTaskStatus returnedTask = task.poll(); // if our task returned with a finished state if (returnedTask != null) { if (returnedTask.getExitStatus() != 0) { LOG.error("Task with id \"" + returnedTask.getId() + "\" failed!"); cleanupTask(returnedTask.getId()); state = JobState.FAILED; return state; } else { LOG.info("Task \"" + returnedTask.getId() + "\" sucessfully finished!"); completed++; LOG.info("Waiting for " + (numBSPTasks - completed) + " tasks to finish!"); } cleanupTasks.add(returnedTask.getId()); } } Thread.sleep(1000L); } for (Integer stopId : cleanupTasks) { cleanupTask(stopId); } state = JobState.SUCCESS; return state; }
From source file:org.apache.helix.provisioning.yarn.RMCallbackHandler.java
License:Apache License
@Override public void onContainersAllocated(List<Container> allocatedContainers) { GenericApplicationMaster.LOG// w ww .j ava 2 s .c o m .info("Got response from RM for container ask, allocatedCnt=" + allocatedContainers.size()); for (Container allocatedContainer : allocatedContainers) { GenericApplicationMaster.LOG.info("Allocated new container." + ", containerId=" + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory" + allocatedContainer.getResource().getMemory()); for (ContainerRequest containerRequest : _genericApplicationMaster.containerRequestMap.keySet()) { if (containerRequest.getCapability().getMemory() == allocatedContainer.getResource().getMemory()) { SettableFuture<ContainerAskResponse> future = _genericApplicationMaster.containerRequestMap .remove(containerRequest); ContainerAskResponse response = new ContainerAskResponse(); response.setContainer(allocatedContainer); _genericApplicationMaster.allocatedContainerSet.add(allocatedContainer.getId()); future.set(response); break; } } } }
From source file:org.apache.hoya.tools.HoyaUtils.java
License:Apache License
public static String containerToString(Container container) { if (container == null) { return "null container"; }//from ww w . j av a 2 s . co m return String.format(Locale.ENGLISH, "ContainerID=%s nodeID=%s http=%s priority=%s", container.getId(), container.getNodeId(), container.getNodeHttpAddress(), container.getPriority()); }
From source file:org.apache.hoya.yarn.appmaster.state.RoleInstance.java
License:Apache License
public RoleInstance(Container container) { this.container = container; if (container == null) { throw new NullPointerException("Null container"); }//from w w w.j av a 2 s . c o m if (container.getId() == null) { throw new NullPointerException("Null container ID"); } id = container.getId().toString(); if (container.getNodeId() != null) { host = container.getNodeId().getHost(); } if (container.getNodeHttpAddress() != null) { hostURL = "http://" + container.getNodeHttpAddress(); } }
From source file:org.apache.metron.maas.service.callback.ContainerRequestListener.java
License:Apache License
@Override public void onContainersAllocated(List<Container> allocatedContainers) { LOG.info("Got response from RM for container ask, allocatedCnt=" + allocatedContainers.size()); for (Container allocatedContainer : allocatedContainers) { containers.put(allocatedContainer.getId(), allocatedContainer); state.registerContainer(allocatedContainer.getResource(), allocatedContainer); LOG.info("Launching shell command on a new container." + ", containerId=" + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory=" + allocatedContainer.getResource().getMemory() + ", containerResourceVirtualCores=" + allocatedContainer.getResource().getVirtualCores()); }/* w ww . j a va 2 s.c om*/ }
From source file:org.apache.samza.job.yarn.ContainerRequestState.java
License:Apache License
/** * This method is called every time the RM returns an allocated container. * Adds the allocated container resource to the correct allocatedContainers buffer * @param container Container resource that was returned by the RM */// w w w. j a v a2 s .co m public synchronized void addContainer(Container container) { if (hostAffinityEnabled) { String hostName = container.getNodeHttpAddress().split(":")[0]; AtomicInteger requestCount = requestsToCountMap.get(hostName); // Check if this host was requested for any of the containers if (requestCount == null || requestCount.get() == 0) { log.debug( "Request count for the allocatedContainer on {} is null or 0. This means that the host was not requested " + "for running containers.Hence, saving the container {} in the buffer for ANY_HOST", hostName, container.getId()); addToAllocatedContainerList(ANY_HOST, container); } else { int requestCountOnThisHost = requestCount.get(); List<Container> allocatedContainersOnThisHost = allocatedContainers.get(hostName); if (requestCountOnThisHost > 0) { if (allocatedContainersOnThisHost == null) { log.debug("Saving the container {} in the buffer for {}", container.getId(), hostName); addToAllocatedContainerList(hostName, container); } else { if (allocatedContainersOnThisHost.size() < requestCountOnThisHost) { log.debug("Saving the container {} in the buffer for {}", container.getId(), hostName); addToAllocatedContainerList(hostName, container); } else { /** * The RM may allocate more containers on a given host than requested. In such a case, even though the * requestCount != 0, it will be greater than the total request count for that host. Hence, it should be * assigned to ANY_HOST */ log.debug("The number of containers already allocated on {} is greater than what was " + "requested, which is {}. Hence, saving the container {} in the buffer for ANY_HOST", new Object[] { hostName, requestCountOnThisHost, container.getId() }); addToAllocatedContainerList(ANY_HOST, container); } } } else { log.debug( "This host was never requested. Hence, saving the container {} in the buffer for ANY_HOST", new Object[] { hostName, requestCountOnThisHost, container.getId() }); addToAllocatedContainerList(ANY_HOST, container); } } } else { log.debug("Saving the container {} in the buffer for ANY_HOST", container.getId()); addToAllocatedContainerList(ANY_HOST, container); } }
From source file:org.apache.samza.job.yarn.ContainerUtil.java
License:Apache License
public void runContainer(int samzaContainerId, Container container) { String containerIdStr = ConverterUtils.toString(container.getId()); log.info("Got available container ID ({}) for container: {}", samzaContainerId, container); String cmdBuilderClassName;//from w w w . j a v a 2s. co m if (taskConfig.getCommandClass().isDefined()) { cmdBuilderClassName = taskConfig.getCommandClass().get(); } else { cmdBuilderClassName = ShellCommandBuilder.class.getName(); } CommandBuilder cmdBuilder = (CommandBuilder) Util.getObj(cmdBuilderClassName); cmdBuilder.setConfig(config).setId(samzaContainerId).setUrl(state.coordinatorUrl); String command = cmdBuilder.buildCommand(); log.info("Container ID {} using command {}", samzaContainerId, command); log.info("Container ID {} using environment variables: ", samzaContainerId); Map<String, String> env = new HashMap<String, String>(); for (Map.Entry<String, String> entry : cmdBuilder.buildEnvironment().entrySet()) { String escapedValue = Util.envVarEscape(entry.getValue()); env.put(entry.getKey(), escapedValue); log.info("{}={} ", entry.getKey(), escapedValue); } Path path = new Path(yarnConfig.getPackagePath()); log.info("Starting container ID {} using package path {}", samzaContainerId, path); startContainer(path, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR)); if (state.neededContainers.decrementAndGet() == 0) { state.jobHealthy.set(true); } state.runningContainers.put(samzaContainerId, new YarnContainer(container)); log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).", new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr }); log.info("Started container ID {}", samzaContainerId); }
From source file:org.apache.samza.job.yarn.refactor.YarnContainerRunner.java
License:Apache License
/** * Runs a process as specified by the command builder on the container. * @param samzaContainerId id of the samza Container to run (passed as a command line parameter to the process) * @param container the samza container to run. * @param cmdBuilder the command builder that encapsulates the command, and the context * * @throws SamzaContainerLaunchException when there's an exception in submitting the request to the RM. * *///from www . ja v a 2 s . co m //TODO: we don't need samzaContainerId as a param here. public void runContainer(int samzaContainerId, Container container, CommandBuilder cmdBuilder) throws SamzaContainerLaunchException { String containerIdStr = ConverterUtils.toString(container.getId()); log.info("Got available container ID ({}) for container: {}", samzaContainerId, container); // check if we have framework path specified. If yes - use it, if not use default ./__package/ String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries String cmdPath = "./__package/"; String fwkPath = JobConfig.getFwkPath(config); if (fwkPath != null && (!fwkPath.isEmpty())) { cmdPath = fwkPath; jobLib = "export JOB_LIB_DIR=./__package/lib"; } log.info("In runContainer in util: fwkPath= " + fwkPath + ";cmdPath=" + cmdPath + ";jobLib=" + jobLib); cmdBuilder.setCommandPath(cmdPath); String command = cmdBuilder.buildCommand(); log.info("Container ID {} using command {}", samzaContainerId, command); Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder); printContainerEnvironmentVariables(samzaContainerId, env); log.info("Samza FWK path: " + command + "; env=" + env); Path path = new Path(yarnConfig.getPackagePath()); log.info("Starting container ID {} using package path {}", samzaContainerId, path); startContainer(path, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, jobLib, command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR)); log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).", new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr }); log.info("Started container ID {}", samzaContainerId); }