Java tutorial
/** * (C) Copyright IBM Corp. 2010, 2015 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package com.ibm.bi.dml.yarn; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.yarn.api.records.ApplicationId; import com.ibm.bi.dml.api.DMLScript; import com.ibm.bi.dml.conf.DMLConfig; import com.ibm.bi.dml.hops.HopsException; import com.ibm.bi.dml.hops.OptimizerUtils; import com.ibm.bi.dml.hops.OptimizerUtils.OptimizationLevel; import com.ibm.bi.dml.lops.Lop; import com.ibm.bi.dml.lops.LopsException; import com.ibm.bi.dml.runtime.DMLRuntimeException; import com.ibm.bi.dml.runtime.DMLUnsupportedOperationException; import com.ibm.bi.dml.runtime.controlprogram.Program; import com.ibm.bi.dml.runtime.controlprogram.ProgramBlock; import com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; import com.ibm.bi.dml.yarn.ropt.ResourceConfig; import com.ibm.bi.dml.yarn.ropt.ResourceOptimizer; import com.ibm.bi.dml.yarn.ropt.YarnClusterConfig; public class DMLAppMasterUtils { private static ResourceConfig _rc = null; private static HashMap<ProgramBlock, Long> _rcMap = null; /** * * @param conf * @param appId * @return */ public static String constructHDFSWorkingDir(DMLConfig conf, ApplicationId appId) { StringBuilder sb = new StringBuilder(); sb.append(conf.getTextValue(DMLConfig.SCRATCH_SPACE)); sb.append(Lop.FILE_SEPARATOR); sb.append(appId); sb.append(Lop.FILE_SEPARATOR); return sb.toString(); } /** * * @param conf * @throws DMLRuntimeException */ public static void setupConfigRemoteMaxMemory(DMLConfig conf) throws DMLRuntimeException { //set remote max memory (if in yarn appmaster context) if (DMLScript.isActiveAM()) { //set optimization level (for awareness of resource optimization) OptimizerUtils.setOptimizationLevel(conf.getIntValue(DMLConfig.OPTIMIZATION_LEVEL)); if (isResourceOptimizerEnabled()) { //handle optimized memory (mr memory budget per program block) //ensure cluster has been analyzed InfrastructureAnalyzer.getRemoteMaxMemoryMap(); String memStr = conf.getTextValue(DMLConfig.YARN_MAPREDUCEMEM); ResourceConfig rc = ResourceConfig.deserialize(memStr); _rc = rc; //keep resource config for later program mapping } else { //handle user configuration if (conf.getIntValue(DMLConfig.YARN_MAPREDUCEMEM) > 0) { //ensure cluster has been analyzed InfrastructureAnalyzer.getRemoteMaxMemoryMap(); //set max map and reduce memory (to be used by the compiler) //see GMR and parfor EMR and DPEMR for runtime configuration long mem = ((long) conf.getIntValue(DMLConfig.YARN_MAPREDUCEMEM)) * 1024 * 1024; InfrastructureAnalyzer.setRemoteMaxMemoryMap(mem); InfrastructureAnalyzer.setRemoteMaxMemoryReduce(mem); } } } } /** * * @param prog * @throws DMLRuntimeException * @throws HopsException * @throws LopsException * @throws DMLUnsupportedOperationException * @throws IOException */ public static void setupProgramMappingRemoteMaxMemory(Program prog) throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException { if (DMLScript.isActiveAM() && isResourceOptimizerEnabled()) { ArrayList<ProgramBlock> pbProg = getRuntimeProgramBlocks(prog); ArrayList<ProgramBlock> B = ResourceOptimizer.compileProgram(pbProg, _rc); _rcMap = new HashMap<ProgramBlock, Long>(); for (int i = 0; i < B.size(); i++) { _rcMap.put(B.get(i), _rc.getMRResources(i)); } } } /** * * @param sb */ public static void setupProgramBlockRemoteMaxMemory(ProgramBlock pb) { if (DMLScript.isActiveAM() && isResourceOptimizerEnabled()) { if (_rcMap != null && _rcMap.containsKey(pb)) { //set max map and reduce memory (to be used by the compiler) long mem = _rcMap.get(pb); InfrastructureAnalyzer.setRemoteMaxMemoryMap(mem); InfrastructureAnalyzer.setRemoteMaxMemoryReduce(mem); OptimizerUtils.setDefaultSize(); } } } /** * * @param job * @param conf */ public static void setupMRJobRemoteMaxMemory(JobConf job, DMLConfig conf) { if (DMLScript.isActiveAM() && conf.getBooleanValue(DMLConfig.YARN_APPMASTER)) { int memMB = -1; //obtain the current configuation (optimized or user-specified) if (isResourceOptimizerEnabled()) memMB = (int) (InfrastructureAnalyzer.getRemoteMaxMemoryMap() / (1024 * 1024)); else memMB = conf.getIntValue(DMLConfig.YARN_MAPREDUCEMEM); //set the memory configuration into the job conf if (memMB > 0) { //ignored if negative String memOpts = "-Xmx" + memMB + "m -Xms" + memMB + "m -Xmn" + (int) (memMB / 10) + "m"; //set mapper heapsizes job.set("mapreduce.map.java.opts", memOpts); job.set("mapreduce.map.memory.mb", String.valueOf(DMLYarnClient.computeMemoryAllocation(memMB))); //set reducer heapsizes job.set("mapreduce.reduce.java.opts", memOpts); job.set("mapreduce.reduce.memory.mb", String.valueOf(DMLYarnClient.computeMemoryAllocation(memMB))); } } } /** * * @return */ public static boolean isResourceOptimizerEnabled() { return (DMLYarnClientProxy.RESOURCE_OPTIMIZER || OptimizerUtils.isOptLevel(OptimizationLevel.O3_LOCAL_RESOURCE_TIME_MEMORY)); } /** * * @param args * @return * @throws DMLException */ protected static ArrayList<ProgramBlock> getRuntimeProgramBlocks(Program prog) throws DMLRuntimeException { //construct single list of all program blocks including functions ArrayList<ProgramBlock> ret = new ArrayList<ProgramBlock>(); ret.addAll(prog.getProgramBlocks()); ret.addAll(prog.getFunctionProgramBlocks().values()); return ret; } /** * * @param cc */ protected static void setupRemoteParallelTasks(YarnClusterConfig cc) { int pmap = (int) cc.getNumCores(); int preduce = (int) cc.getNumCores() / 2; InfrastructureAnalyzer.setRemoteParallelMapTasks(pmap); InfrastructureAnalyzer.setRemoteParallelReduceTasks(preduce); } }