List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB
int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB
To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB.
Click Source Link
From source file:com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer.java
License:Open Source License
/** * Analyzes properties of Yarn cluster and Hadoop configurations. *///from w w w .ja v a2 s . com public static void analyzeYarnCluster(YarnClient yarnClient, YarnConfiguration conf, boolean verbose) { try { List<NodeReport> nodesReport = yarnClient.getNodeReports(); if (verbose) System.out.println("There are " + nodesReport.size() + " nodes in the cluster"); if (nodesReport.isEmpty()) throw new YarnException("There are zero available nodes in the yarn cluster"); nodesMaxPhySorted = new ArrayList<Long>(nodesReport.size()); clusterTotalMem = 0; clusterTotalCores = 0; clusterTotalNodes = 0; minimumMRContainerPhyMB = -1; for (NodeReport node : nodesReport) { Resource resource = node.getCapability(); Resource used = node.getUsed(); if (used == null) used = Resource.newInstance(0, 0); int mb = resource.getMemory(); int cores = resource.getVirtualCores(); if (mb <= 0) throw new YarnException("A node has non-positive memory " + mb); int myMinMRPhyMB = mb / cores / CPU_HYPER_FACTOR; if (minimumMRContainerPhyMB < myMinMRPhyMB) minimumMRContainerPhyMB = myMinMRPhyMB; // minimumMRContainerPhyMB needs to be the largest among the mins clusterTotalMem += (long) mb * 1024 * 1024; nodesMaxPhySorted.add((long) mb * 1024 * 1024); clusterTotalCores += cores; clusterTotalNodes++; if (verbose) System.out.println("\t" + node.getNodeId() + " has " + mb + " MB (" + used.getMemory() + " MB used) memory and " + resource.getVirtualCores() + " (" + used.getVirtualCores() + " used) cores"); } Collections.sort(nodesMaxPhySorted, Collections.reverseOrder()); nodesMaxBudgetSorted = new ArrayList<Double>(nodesMaxPhySorted.size()); for (int i = 0; i < nodesMaxPhySorted.size(); i++) nodesMaxBudgetSorted.add(ResourceOptimizer.phyToBudget(nodesMaxPhySorted.get(i))); _remotePar = nodesReport.size(); if (_remotePar == 0) throw new YarnException("There are no available nodes in the yarn cluster"); // Now get the default cluster settings _remoteMRSortMem = (1024 * 1024) * conf.getLong("io.sort.mb", 100); //100MB //handle jvm max mem (map mem budget is relevant for map-side distcache and parfor) //(for robustness we probe both: child and map configuration parameters) String javaOpts1 = conf.get("mapred.child.java.opts"); //internally mapred/mapreduce synonym String javaOpts2 = conf.get("mapreduce.map.java.opts", null); //internally mapred/mapreduce synonym String javaOpts3 = conf.get("mapreduce.reduce.java.opts", null); //internally mapred/mapreduce synonym if (javaOpts2 != null) //specific value overrides generic _remoteJVMMaxMemMap = extractMaxMemoryOpt(javaOpts2); else _remoteJVMMaxMemMap = extractMaxMemoryOpt(javaOpts1); if (javaOpts3 != null) //specific value overrides generic _remoteJVMMaxMemReduce = extractMaxMemoryOpt(javaOpts3); else _remoteJVMMaxMemReduce = extractMaxMemoryOpt(javaOpts1); //HDFS blocksize String blocksize = conf.get(MRConfigurationNames.DFS_BLOCK_SIZE, "134217728"); _blocksize = Long.parseLong(blocksize); minimalPhyAllocate = (long) 1024 * 1024 * conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); maximumPhyAllocate = (long) 1024 * 1024 * conf.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); mrAMPhy = (long) conf.getInt("yarn.app.mapreduce.am.resource.mb", 1536) * 1024 * 1024; } catch (Exception e) { throw new RuntimeException("Unable to analyze yarn cluster ", e); } /* * This is for AppMaster to query available resource in the cluster during heartbeat * AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf); rmClient.start(); AllocateResponse response = rmClient.allocate(0); int nodeCount = response.getNumClusterNodes(); Resource resource = response.getAvailableResources(); List<NodeReport> nodeUpdate = response.getUpdatedNodes(); LOG.info("This is a " + nodeCount + " node cluster with totally " + resource.getMemory() + " memory and " + resource.getVirtualCores() + " cores"); LOG.info(nodereport.size() + " updatedNode reports received"); for (NodeReport node : nodeUpdate) { resource = node.getCapability(); LOG.info(node.getNodeId() + " updated with " + resource.getMemory() + " memory and " + resource.getVirtualCores() + " cores"); }*/ }
From source file:de.huberlin.wbi.hiway.am.WorkflowDriver.java
License:Apache License
/** Parse command line arguments, initialize HDFS, manage environment variables. */ public boolean init(String[] args) throws ParseException, IOException, JSONException { DefaultMetricsSystem.initialize("ApplicationMaster"); Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); opts.addOption("u", "summary", true, "The name of the json summary file. No file is created if this parameter is not specified."); opts.addOption("m", "memory", true, "The amount of memory (in MB) to be allocated per worker container. Overrides settings in hiway-site.xml."); opts.addOption("c", "custom", true, "The name of an (optional) JSON file, in which custom amounts of memory can be specified per task."); opts.addOption("s", "scheduler", true, "The scheduling policy that is to be employed. Valid arguments: " + Arrays.toString(HiWayConfiguration.HIWAY_SCHEDULERS.values()) + ". Overrides settings in hiway-site.xml."); opts.addOption("d", "debug", false, "Provide additional logs and information for debugging"); opts.addOption("v", "verbose", false, "Increase verbosity of output / reporting."); opts.addOption("appid", true, "Id of this Application Master."); opts.addOption("h", "help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { Logger.printUsage(opts);//from w ww . j a v a 2 s. com throw new IllegalArgumentException("No args specified for application master to initialize"); } if (cliParser.getArgs().length == 0) { Logger.printUsage(opts); throw new IllegalArgumentException("No workflow file specified."); } if (!cliParser.hasOption("appid")) { throw new IllegalArgumentException("No id of Application Master specified"); } if (cliParser.hasOption("verbose")) { HiWayConfiguration.verbose = true; } appId = cliParser.getOptionValue("appid"); try { logger.statLog = new BufferedWriter(new FileWriter(appId + ".log")); } catch (IOException e) { e.printStackTrace(System.out); System.exit(-1); } if (cliParser.hasOption("help")) { Logger.printUsage(opts); return false; } if (cliParser.hasOption("debug")) { Logger.dumpOutDebugInfo(); HiWayConfiguration.debug = true; } if (cliParser.hasOption("summary")) { summaryPath = new Path(cliParser.getOptionValue("summary")); } String hdfsBaseDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE, HiWayConfiguration.HIWAY_AM_DIRECTORY_BASE_DEFAULT); String hdfsSandboxDirectoryName = conf.get(HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE, HiWayConfiguration.HIWAY_AM_DIRECTORY_CACHE_DEFAULT); Path hdfsBaseDirectory = new Path(new Path(hdfs.getUri()), hdfsBaseDirectoryName); Data.setHdfsBaseDirectory(hdfsBaseDirectory); Path hdfsSandboxDirectory = new Path(hdfsBaseDirectory, hdfsSandboxDirectoryName); hdfsApplicationDirectory = new Path(hdfsSandboxDirectory, appId); Data.setHdfsApplicationDirectory(hdfsApplicationDirectory); Data.setHdfs(hdfs); if (cliParser.hasOption("custom")) { Data customMemPath = new Data(cliParser.getOptionValue("custom")); customMemPath.stageIn(); StringBuilder sb = new StringBuilder(); try (BufferedReader in = new BufferedReader(new FileReader(customMemPath.getLocalPath().toString()))) { String line; while ((line = in.readLine()) != null) { sb.append(line); } } JSONObject obj = new JSONObject(sb.toString()); Iterator<?> keys = obj.keys(); while (keys.hasNext()) { String key = (String) keys.next(); int minMem = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); int desiredMem = obj.getInt(key); customMemoryMap.put(key, (desiredMem % minMem) == 0 ? desiredMem : (desiredMem / minMem + 1) * minMem); } } Map<String, String> envs = System.getenv(); /* this application's attempt id (combination of attemptId and fail count) */ ApplicationAttemptId appAttemptID; if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HOST.name())) { throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment"); } if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) { throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment"); } if (!envs.containsKey(Environment.NM_PORT.name())) { throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment"); } Logger.writeToStdout("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); String shellEnvs[] = conf.getStrings(HiWayConfiguration.HIWAY_WORKER_SHELL_ENV, HiWayConfiguration.HIWAY_WORKER_SHELL_ENV_DEFAULT); for (String env : shellEnvs) { env = env.trim(); int index = env.indexOf('='); if (index == -1) { shellEnv.put(env, ""); continue; } String key = env.substring(0, index); String val = ""; if (index < (env.length() - 1)) { val = env.substring(index + 1); } shellEnv.put(key, val); } String workflowParam = cliParser.getArgs()[0]; try { workflowPath = new Path(new URI(workflowParam).getPath()); } catch (URISyntaxException e) { workflowPath = new Path(workflowParam); } schedulerEnumValue = HiWayConfiguration.HIWAY_SCHEDULERS.valueOf(conf .get(HiWayConfiguration.HIWAY_SCHEDULER, HiWayConfiguration.HIWAY_SCHEDULER_DEFAULT.toString())); if (cliParser.hasOption("scheduler")) { schedulerEnumValue = HiWayConfiguration.HIWAY_SCHEDULERS.valueOf(cliParser.getOptionValue("scheduler")); } containerMemory = conf.getInt(HiWayConfiguration.HIWAY_WORKER_MEMORY, HiWayConfiguration.HIWAY_WORKER_MEMORY_DEFAULT); if (cliParser.hasOption("memory")) { containerMemory = Integer.parseInt(cliParser.getOptionValue("memory")); } containerCores = conf.getInt(HiWayConfiguration.HIWAY_WORKER_VCORES, HiWayConfiguration.HIWAY_WORKER_VCORES_DEFAULT); requestPriority = conf.getInt(HiWayConfiguration.HIWAY_WORKER_PRIORITY, HiWayConfiguration.HIWAY_WORKER_PRIORITY_DEFAULT); // Create and start the Timeline timelineClient if (conf.getBoolean("yarn.timeline-service.enabled", false)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); Logger.writeToStdout("Started TimeLineClient."); } else { Logger.writeToStdErr("TimeLineClient disabled."); } return true; }
From source file:io.hops.metadata.util.TestHopYarnAPIUtilities.java
License:Apache License
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue, Set<String> tags, boolean unmanaged) { // ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); ContainerLaunchContext amContainerSpec = new ContainerLaunchContextPBImpl(); org.apache.hadoop.yarn.api.records.Resource resource = Resources .createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory .newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); submissionContext.setApplicationName(name); submissionContext.setQueue(queue);//from w w w. ja va 2s. c om submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(submissionContext); return submitRequest; }
From source file:org.apache.sysml.yarn.ropt.YarnClusterAnalyzer.java
License:Apache License
/** * Analyzes properties of Yarn cluster and Hadoop configurations. * /* w w w . java 2s.c om*/ * @param yarnClient hadoop yarn client * @param conf hadoop yarn configuration * @param verbose output info to standard output */ public static void analyzeYarnCluster(YarnClient yarnClient, YarnConfiguration conf, boolean verbose) { try { List<NodeReport> nodesReport = yarnClient.getNodeReports(); if (verbose) System.out.println("There are " + nodesReport.size() + " nodes in the cluster"); if (nodesReport.isEmpty()) throw new YarnException("There are zero available nodes in the yarn cluster"); nodesMaxPhySorted = new ArrayList<>(nodesReport.size()); clusterTotalMem = 0; clusterTotalCores = 0; clusterTotalNodes = 0; minimumMRContainerPhyMB = -1; for (NodeReport node : nodesReport) { Resource resource = node.getCapability(); Resource used = node.getUsed(); if (used == null) used = Resource.newInstance(0, 0); int mb = resource.getMemory(); int cores = resource.getVirtualCores(); if (mb <= 0) throw new YarnException("A node has non-positive memory " + mb); int myMinMRPhyMB = mb / cores / CPU_HYPER_FACTOR; if (minimumMRContainerPhyMB < myMinMRPhyMB) minimumMRContainerPhyMB = myMinMRPhyMB; // minimumMRContainerPhyMB needs to be the largest among the mins clusterTotalMem += (long) mb * 1024 * 1024; nodesMaxPhySorted.add((long) mb * 1024 * 1024); clusterTotalCores += cores; clusterTotalNodes++; if (verbose) System.out.println("\t" + node.getNodeId() + " has " + mb + " MB (" + used.getMemory() + " MB used) memory and " + resource.getVirtualCores() + " (" + used.getVirtualCores() + " used) cores"); } Collections.sort(nodesMaxPhySorted, Collections.reverseOrder()); nodesMaxBudgetSorted = new ArrayList<>(nodesMaxPhySorted.size()); for (int i = 0; i < nodesMaxPhySorted.size(); i++) nodesMaxBudgetSorted.add(ResourceOptimizer.phyToBudget(nodesMaxPhySorted.get(i))); _remotePar = nodesReport.size(); if (_remotePar == 0) throw new YarnException("There are no available nodes in the yarn cluster"); // Now get the default cluster settings _remoteMRSortMem = (1024 * 1024) * conf.getLong(MRConfigurationNames.MR_TASK_IO_SORT_MB, 100); //100MB //handle jvm max mem (map mem budget is relevant for map-side distcache and parfor) //(for robustness we probe both: child and map configuration parameters) String javaOpts1 = conf.get(MRConfigurationNames.MR_CHILD_JAVA_OPTS); //internally mapred/mapreduce synonym String javaOpts2 = conf.get(MRConfigurationNames.MR_MAP_JAVA_OPTS, null); //internally mapred/mapreduce synonym String javaOpts3 = conf.get(MRConfigurationNames.MR_REDUCE_JAVA_OPTS, null); //internally mapred/mapreduce synonym if (javaOpts2 != null) //specific value overrides generic _remoteJVMMaxMemMap = extractMaxMemoryOpt(javaOpts2); else _remoteJVMMaxMemMap = extractMaxMemoryOpt(javaOpts1); if (javaOpts3 != null) //specific value overrides generic _remoteJVMMaxMemReduce = extractMaxMemoryOpt(javaOpts3); else _remoteJVMMaxMemReduce = extractMaxMemoryOpt(javaOpts1); //HDFS blocksize String blocksize = conf.get(MRConfigurationNames.DFS_BLOCKSIZE, "134217728"); _blocksize = Long.parseLong(blocksize); minimalPhyAllocate = (long) 1024 * 1024 * conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); maximumPhyAllocate = (long) 1024 * 1024 * conf.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); mrAMPhy = (long) conf.getInt(MRConfigurationNames.YARN_APP_MR_AM_RESOURCE_MB, 1536) * 1024 * 1024; } catch (Exception e) { throw new RuntimeException("Unable to analyze yarn cluster ", e); } /* * This is for AppMaster to query available resource in the cluster during heartbeat * AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf); rmClient.start(); AllocateResponse response = rmClient.allocate(0); int nodeCount = response.getNumClusterNodes(); Resource resource = response.getAvailableResources(); List<NodeReport> nodeUpdate = response.getUpdatedNodes(); LOG.info("This is a " + nodeCount + " node cluster with totally " + resource.getMemory() + " memory and " + resource.getVirtualCores() + " cores"); LOG.info(nodereport.size() + " updatedNode reports received"); for (NodeReport node : nodeUpdate) { resource = node.getCapability(); LOG.info(node.getNodeId() + " updated with " + resource.getMemory() + " memory and " + resource.getVirtualCores() + " cores"); }*/ }