List of usage examples for org.apache.hadoop.yarn.api.records Container getResource
@Public @Stable public abstract Resource getResource();
Resource allocated to the container. From source file:io.amient.yarn1.YarnContainerContext.java
License:Open Source License
public boolean isSatisfiedBy(Container container) { return container.getResource().getMemory() >= capability.getMemory() && container.getResource().getVirtualCores() >= capability.getVirtualCores() && container.getPriority().getPriority() >= priority.getPriority(); }
From source file:io.hops.tensorflow.TimelineHandler.java
License:Apache License
public void publishContainerStartEvent(Container container) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getId().toString()); entity.setEntityType(ApplicationMaster.YarntfEntity.YARNTF_CONTAINER.toString()); entity.setDomainId(domainId);/*from www .j av a2s . com*/ entity.addPrimaryFilter("user", ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(ApplicationMaster.YarntfEvent.YARNTF_CONTAINER_START.toString()); event.addEventInfo("Node", container.getNodeId().toString()); event.addEventInfo("Resources", container.getResource().toString()); entity.addEvent(event); try { ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() { @Override public TimelinePutResponse run() throws Exception { return timelineClient.putEntities(entity); } }); } catch (Exception e) { LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e); } }
From source file:io.hops.util.DBUtility.java
License:Apache License
public static void removeContainersToDecrease( final Collection<org.apache.hadoop.yarn.api.records.Container> containers) throws IOException { long start = System.currentTimeMillis(); AsyncLightWeightRequestHandler removeContainerToDecrease = new AsyncLightWeightRequestHandler( YARNOperationType.TEST) {/*from ww w . java 2s . c o m*/ @Override public Object performTask() throws StorageException { connector.beginTransaction(); connector.writeLock(); ContainerToDecreaseDataAccess ctsDA = (ContainerToDecreaseDataAccess) RMStorageFactory .getDataAccess(ContainerToDecreaseDataAccess.class); List<io.hops.metadata.yarn.entity.Container> containersToDecrease = new ArrayList<io.hops.metadata.yarn.entity.Container>(); for (org.apache.hadoop.yarn.api.records.Container container : containers) { containersToDecrease.add(new io.hops.metadata.yarn.entity.Container( container.getId().toString(), container.getNodeId().toString(), container.getNodeHttpAddress(), container.getPriority().getPriority(), container.getResource().getMemorySize(), container.getResource().getVirtualCores(), container.getResource().getGPUs(), container.getVersion())); } ctsDA.removeAll(containersToDecrease); connector.commit(); return null; } }; removeContainerToDecrease.handle(); long duration = System.currentTimeMillis() - start; if (duration > 10) { LOG.error("too long " + duration); } }
From source file:io.hops.util.DBUtility.java
License:Apache License
public static void addContainersToDecrease(final List<org.apache.hadoop.yarn.api.records.Container> containers) throws IOException { long start = System.currentTimeMillis(); AsyncLightWeightRequestHandler addContainerToDecrease = new AsyncLightWeightRequestHandler( YARNOperationType.TEST) {/* ww w.ja va 2 s . c om*/ @Override public Object performTask() throws StorageException { connector.beginTransaction(); connector.writeLock(); ContainerToDecreaseDataAccess ctsDA = (ContainerToDecreaseDataAccess) RMStorageFactory .getDataAccess(ContainerToDecreaseDataAccess.class); List<io.hops.metadata.yarn.entity.Container> containersToDecrease = new ArrayList<io.hops.metadata.yarn.entity.Container>(); for (org.apache.hadoop.yarn.api.records.Container container : containers) { containersToDecrease.add(new io.hops.metadata.yarn.entity.Container( container.getId().toString(), container.getNodeId().toString(), container.getNodeHttpAddress(), container.getPriority().getPriority(), container.getResource().getMemorySize(), container.getResource().getVirtualCores(), container.getResource().getGPUs(), container.getVersion())); } ctsDA.addAll(containersToDecrease); connector.commit(); return null; } }; addContainerToDecrease.handle(); long duration = System.currentTimeMillis() - start; if (duration > 10) { LOG.error("too long " + duration); } }
From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java
License:Apache License
/** * For each container successfully allocated, attempt to set up and launch a Guagua worker/master task. * /*ww w . j a v a 2 s . com*/ * @param allocatedContainers * the containers we have currently allocated. */ private void startContainerLaunchingThreads(final List<Container> allocatedContainers) { Map<String, List<Container>> hostContainterMap = getHostContainersMap(allocatedContainers); int size = allocatedContainers.size(); while (size > 0) { int currentPartition = getCurrentPartition(); if (currentPartition == -1) { LOG.warn("Request too many resources. TODO, remove containers no needed."); for (Container container : allocatedContainers) { GuaguaAppMaster.this.getAmRMClient().releaseAssignedContainer(container.getId()); } break; } Container container = getDataLocalityContainer(hostContainterMap, currentPartition); if (container == null) { container = allocatedContainers.get(0); } allocatedContainers.remove(container); LOG.info( "Launching command on a new container., containerId={}, containerNode={}, containerPort={}, containerNodeURI={}, containerResourceMemory={}", container.getId(), container.getNodeId().getHost(), container.getNodeId().getPort(), container.getNodeHttpAddress(), container.getResource().getMemory()); List<Container> list = this.partitionContainerMap.get(currentPartition); if (list == null) { list = new ArrayList<Container>(); } list.add(container); this.partitionContainerMap.put(currentPartition, list); this.containerPartitionMap.put(container.getId().toString(), currentPartition); this.partitionStatusMap.put(currentPartition, PartitionStatus.INIT); LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(container, getContainerListener(), currentPartition); getExecutor().execute(runnableLaunchContainer); size = allocatedContainers.size(); } }
From source file:org.apache.drill.yarn.core.DoYUtil.java
License:Apache License
/** * Utility method to display YARN container information in a useful way for * log messages./* w ww .j av a 2 s . c om*/ * * @param container * @return */ public static String describeContainer(Container container) { StringBuilder buf = new StringBuilder().append("[id: ").append(container.getId()).append(", host: ") .append(container.getNodeId().getHost()).append(", priority: ").append(container.getPriority()) .append(", memory: ").append(container.getResource().getMemory()).append(" MB, vcores: ") .append(container.getResource().getVirtualCores()).append("]"); return buf.toString(); }
From source file:org.apache.flink.yarn.YarnResourceManager.java
License:Apache License
@Override public void onContainersAllocated(List<Container> containers) { for (Container container : containers) { numPendingContainerRequests = Math.max(0, numPendingContainerRequests - 1); LOG.info("Received new container: {} - Remaining pending container requests: {}", container.getId(), numPendingContainerRequests); try {//w w w . j a v a2 s . com /** Context information used to start a TaskExecutor Java process */ ContainerLaunchContext taskExecutorLaunchContext = createTaskExecutorLaunchContext( container.getResource(), container.getId().toString(), container.getNodeId().getHost()); nodeManagerClient.startContainer(container, taskExecutorLaunchContext); } catch (Throwable t) { // failed to launch the container, will release the failed one and ask for a new one LOG.error("Could not start TaskManager in container {},", container, t); resourceManagerClient.releaseAssignedContainer(container.getId()); requestYarnContainer(container.getResource(), container.getPriority()); } } if (numPendingContainerRequests <= 0) { resourceManagerClient.setHeartbeatInterval(yarnHeartbeatIntervalMillis); } }
From source file:org.apache.flink.yarn.YarnResourceManagerTest.java
License:Apache License
private static Container mockContainer(String host, int port, int containerId) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); ContainerId mockContainerId = ContainerId.newInstance( ApplicationAttemptId.newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), containerId);//from w ww . jav a 2 s . c o m when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockContainer.getResource()).thenReturn(Resource.newInstance(200, 1)); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; }
From source file:org.apache.giraph.yarn.GiraphApplicationMaster.java
License:Apache License
/** * For each container successfully allocated, attempt to set up and launch * a Giraph worker/master task.//ww w. jav a 2 s . co m * @param allocatedContainers the containers we have currently allocated. */ private void startContainerLaunchingThreads(final List<Container> allocatedContainers) { for (Container allocatedContainer : allocatedContainers) { LOG.info("Launching command on a new container." + ", containerId=" + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory=" + allocatedContainer.getResource().getMemory()); // Launch and start the container on a separate thread to keep the main // thread unblocked as all containers may not be allocated at one go. LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(allocatedContainer, containerListener); executor.execute(runnableLaunchContainer); } }
From source file:org.apache.gobblin.yarn.YarnService.java
License:Apache License
@VisibleForTesting protected String buildContainerCommand(Container container, String helixInstanceName) { String containerProcessName = GobblinYarnTaskRunner.class.getSimpleName(); return new StringBuilder().append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java") .append(" -Xmx") .append((int) (container.getResource().getMemory() * this.jvmMemoryXmxRatio) - this.jvmMemoryOverheadMbs) .append("M").append(" -D").append(GobblinYarnConfigurationKeys.JVM_USER_TIMEZONE_CONFIG).append("=") .append(this.containerTimezone).append(" -D") .append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_DIR_NAME).append("=") .append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(" -D") .append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=") .append(containerProcessName).append(".").append(ApplicationConstants.STDOUT).append(" ") .append(JvmUtils.formatJvmArguments(this.containerJvmArgs)).append(" ") .append(GobblinYarnTaskRunner.class.getName()).append(" --") .append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME).append(" ") .append(this.applicationName).append(" --") .append(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME).append(" ") .append(helixInstanceName).append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR) .append(File.separator).append(containerProcessName).append(".").append(ApplicationConstants.STDOUT) .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator) .append(containerProcessName).append(".").append(ApplicationConstants.STDERR).toString(); }