Java tutorial
/* * Copyright 2016 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.master.startup; import co.cask.cdap.common.conf.CConfiguration; import co.cask.cdap.common.conf.Constants; import com.google.common.base.Joiner; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.YarnClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; /** * Checks that YARN is available and has enough resources to run all system services. */ // class is picked up through classpath examination @SuppressWarnings("unused") class YarnCheck extends AbstractMasterCheck { private static final Logger LOG = LoggerFactory.getLogger(YarnCheck.class); private final Configuration hConf; @Inject private YarnCheck(CConfiguration cConf, Configuration hConf) { super(cConf); this.hConf = hConf; } @Override public void run() { int yarnConnectTimeout = cConf.getInt(Constants.Startup.YARN_CONNECT_TIMEOUT_SECONDS, 60); LOG.info("Checking YARN availability -- may take up to {} seconds.", yarnConnectTimeout); final YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(hConf); List<NodeReport> nodeReports; // if yarn is not up, yarnClient.start() will hang. ExecutorService executorService = Executors .newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("startup-checker").build()); try { Future<List<NodeReport>> result = executorService.submit(new Callable<List<NodeReport>>() { @Override public List<NodeReport> call() throws Exception { yarnClient.start(); return yarnClient.getNodeReports(); } }); nodeReports = result.get(yarnConnectTimeout, TimeUnit.SECONDS); LOG.info(" YARN availability successfully verified."); } catch (Exception e) { throw new RuntimeException("Unable to get status of YARN nodemanagers. " + "Please check that YARN is running " + "and that the correct Hadoop configuration (core-site.xml, yarn-site.xml) and libraries " + "are included in the CDAP master classpath.", e); } finally { try { yarnClient.stop(); } catch (Exception e) { LOG.warn("Error stopping yarn client.", e); } finally { executorService.shutdown(); } } checkResources(nodeReports); } private void checkResources(List<NodeReport> nodeReports) { LOG.info("Checking that YARN has enough resources to run all system services."); int memoryCapacity = 0; int vcoresCapacity = 0; int memoryUsed = 0; int vcoresUsed = 0; int availableNodes = 0; for (NodeReport nodeReport : nodeReports) { NodeId nodeId = nodeReport.getNodeId(); LOG.debug("Got report for node {}", nodeId); if (!nodeReport.getNodeState().isUnusable()) { Resource nodeCapability = nodeReport.getCapability(); Resource nodeUsed = nodeReport.getUsed(); // some versions of hadoop return null, others do not if (nodeCapability != null) { LOG.debug("node {} resource capability: memory = {}, vcores = {}", nodeId, nodeCapability.getMemory(), nodeCapability.getVirtualCores()); memoryCapacity += nodeCapability.getMemory(); vcoresCapacity += nodeCapability.getVirtualCores(); } if (nodeUsed != null) { LOG.debug("node {} resources used: memory = {}, vcores = {}", nodeId, nodeUsed.getMemory(), nodeUsed.getVirtualCores()); memoryUsed += nodeUsed.getMemory(); vcoresUsed += nodeUsed.getVirtualCores(); } availableNodes++; } } LOG.debug("YARN resource capacity: {} MB of memory and {} virtual cores.", memoryCapacity, vcoresCapacity); LOG.debug("YARN resources used: {} MB of memory and {} virtual cores.", memoryUsed, vcoresUsed); // calculate memory and vcores required by CDAP int requiredMemoryMB = 0; int requiredVCores = 0; Set<String> invalidKeys = new HashSet<>(); for (ServiceResourceKeys serviceResourceKeys : systemServicesResourceKeys) { boolean hasConfigError = false; int instances = 0; int memoryMB = 0; int vcores = 0; try { instances = cConf.getInt(serviceResourceKeys.getInstancesKey()); } catch (Exception e) { invalidKeys.add(serviceResourceKeys.getInstancesKey()); hasConfigError = true; } try { memoryMB = cConf.getInt(serviceResourceKeys.getMemoryKey()); } catch (Exception e) { invalidKeys.add(serviceResourceKeys.getMemoryKey()); hasConfigError = true; } try { vcores = cConf.getInt(serviceResourceKeys.getVcoresKey()); } catch (Exception e) { invalidKeys.add(serviceResourceKeys.getVcoresKey()); hasConfigError = true; } if (!hasConfigError) { LOG.debug("Resource settings for system service {}: {}={}, {}={}, {}={}", serviceResourceKeys.getServiceName(), serviceResourceKeys.getInstancesKey(), instances, serviceResourceKeys.getMemoryKey(), memoryMB, serviceResourceKeys.getVcoresKey(), vcores); requiredMemoryMB += memoryMB * instances; requiredVCores += vcores * instances; } } if (!invalidKeys.isEmpty()) { throw new RuntimeException("YARN resources check failed to invalid config settings for keys: " + Joiner.on(',').join(invalidKeys)); } LOG.debug("{} MB of memory and {} virtual cores are required.", requiredMemoryMB, requiredVCores); int availableMemoryMB = memoryCapacity - memoryUsed; int availableVCores = vcoresCapacity - vcoresUsed; boolean memoryOK = requiredMemoryMB <= availableMemoryMB; // if this is negative or zero just assume its not using vcores boolean vcoresOK = vcoresCapacity <= 0 || requiredVCores <= availableVCores; if (!memoryOK && !vcoresOK) { LOG.warn( "Services require {} MB of memory and {} vcores, " + "but the cluster only has {} MB of memory and {} vcores available.", requiredMemoryMB, requiredVCores, availableMemoryMB, availableVCores); } else if (!memoryOK) { LOG.warn("Services require {} MB of memory but the cluster only has {} MB of memory available.", requiredMemoryMB, availableMemoryMB); } else if (!vcoresOK) { LOG.warn("Services require {} vcores but the cluster only has {} vcores available.", requiredVCores, availableVCores); } else { LOG.info(" YARN resources successfully verified."); } } }