List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration()
From source file:oz.hadoop.yarn.api.YarnAssemblyTests.java
License:Apache License
@Test public void validateJavaBasedContainerWithShell() { YarnConfiguration yarnConfig = new YarnConfiguration(); ApplicationMasterSpecBuilder<DataProcessor> applicationMasterBuilder = YarnAssembly .forApplicationContainer(DslTestApplicationContainer.class, "java").memory(28).containerCount(4) .virtualCores(34).priority(3).withApplicationMaster(yarnConfig); YarnApplication<DataProcessor> yarnApplication = applicationMasterBuilder.maxAttempts(12).priority(34) .queueName("foo").virtualCores(35).memory(1024).build("my-application"); PrimitiveImmutableTypeMap specMap = (PrimitiveImmutableTypeMap) yarnApplication .getApplicationSpecification(); assertEquals(35, specMap.getInt(YayaConstants.VIRTUAL_CORES)); assertEquals(34, specMap.getInt(YayaConstants.PRIORITY)); assertEquals(12, specMap.getInt(YayaConstants.MAX_ATTEMPTS)); assertEquals("foo", specMap.getString(YayaConstants.QUEUE_NAME)); assertEquals(1024, specMap.getInt(YayaConstants.MEMORY)); assertFalse(specMap.getBoolean(YayaConstants.YARN_EMULATOR)); assertEquals("my-application", specMap.getString(YayaConstants.APPLICATION_NAME)); PrimitiveImmutableTypeMap containerSpecMap = (PrimitiveImmutableTypeMap) specMap .get(YayaConstants.CONTAINER_SPEC); assertEquals(4, containerSpecMap.getInt(YayaConstants.CONTAINER_COUNT)); assertEquals(3, containerSpecMap.getInt(YayaConstants.PRIORITY)); assertEquals(28, containerSpecMap.getInt(YayaConstants.MEMORY)); assertEquals(34, containerSpecMap.getInt(YayaConstants.VIRTUAL_CORES)); assertNull(containerSpecMap.get(YayaConstants.CONTAINER_ARG)); assertEquals(DslTestApplicationContainer.class.getName(), containerSpecMap.getString(YayaConstants.CONTAINER_IMPL)); }
From source file:probos.TestEndToEnd.java
License:Open Source License
@Before public void setupCluster() throws Exception { System.setProperty("probos.home", System.getProperty("user.dir")); (probosJobDir = new File(System.getProperty("user.dir"), "probos")).mkdir(); String name = "mycluster"; int noOfNodeManagers = 1; int numLocalDirs = 1; int numLogDirs = 1; YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs); miniCluster.init(conf);/*w w w.j a v a2 s . com*/ miniCluster.start(); //once the cluster is created, you can get its configuration //with the binding details to the cluster added from the minicluster YarnConfiguration appConf = new YarnConfiguration(miniCluster.getConfig()); cs = new ControllerServer(appConf); cs.startAndWait(); Thread.sleep(1000); new pbsnodes().run(new String[0]); }
From source file:probos.TestProblem.java
License:Open Source License
@Before public void setupCluster() throws Exception { String name = "mycluster"; int noOfNodeManagers = 1; int numLocalDirs = 1; int numLogDirs = 1; YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs); miniCluster.init(conf);// w ww .ja v a2 s . c o m miniCluster.start(); //once the cluster is created, you can get its configuration //with the binding details to the cluster added from the minicluster yConf = new YarnConfiguration(miniCluster.getConfig()); }
From source file:runtime.starter.MPJAppMaster.java
License:Open Source License
public MPJAppMaster() { conf = new YarnConfiguration(); opts = new Options(); opts.addOption("np", true, "Number of Processes"); opts.addOption("serverName", true, "Hostname required for Server Socket"); opts.addOption("ioServerPort", true, "Port required for a socket" + " redirecting IO"); opts.addOption("wireUpPort", true, "Port forwarded to Wrappers for " + "sharing read,write ports and ranks"); opts.addOption("deviceName", true, "Specifies the MPJ device name"); opts.addOption("className", true, "Main Class name"); opts.addOption("wdir", true, "Specifies the current working directory"); opts.addOption("psl", true, "Specifies the Protocol Switch Limit"); opts.addOption("wrapperPath", true, "Specifies the wrapper jar path in hdfs"); opts.addOption("userJarPath", true, "Specifies the user jar path in hdfs"); opts.addOption("appArgs", true, "Specifies the User Application args"); opts.getOption("appArgs").setArgs(Option.UNLIMITED_VALUES); opts.addOption("containerMem", true, "Specifies mpj containers memory"); opts.addOption("containerCores", true, "Specifies mpj containers v-cores"); opts.addOption("mpjContainerPriority", true, "Specifies the prioirty of" + "containers running MPI processes"); opts.addOption("debugYarn", false, "Specifies the debug flag"); }
From source file:runtime.starter.MPJYarnClient.java
License:Open Source License
public MPJYarnClient() { logger = LogFactory.getLog(MPJYarnClient.class); conf = new YarnConfiguration(); opts = new Options(); opts.addOption("np", true, "Number of Processes"); opts.addOption("server", true, "Hostname required for Server Socket"); opts.addOption("serverPort", true, "Port required for Server Socket"); opts.addOption("dev", true, "Specifies the MPJ device name"); opts.addOption("className", true, "Main Class name"); opts.addOption("wdir", true, "Specifies the current working directory"); opts.addOption("psl", true, "Specifies the Protocol Switch Limit"); opts.addOption("jarPath", true, "Specifies the Path to user's Jar File"); opts.addOption("appArgs", true, "Specifies the User Application args"); opts.getOption("appArgs").setArgs(Option.UNLIMITED_VALUES); opts.addOption("amMem", true, "Specifies AM container memory"); opts.addOption("amCores", true, "Specifies AM container virtual cores"); opts.addOption("containerMem", true, "Specifies mpj containers memory"); opts.addOption("containerCores", true, "Specifies mpj containers v-cores"); opts.addOption("yarnQueue", true, "Specifies the yarn queue"); opts.addOption("appName", true, "Specifies the application name"); opts.addOption("amPriority", true, "Specifies AM container priority"); opts.addOption("mpjContainerPriority", true, "Specifies the prioirty of" + "containers running MPI processes"); opts.addOption("hdfsFolder", true, "Specifies the HDFS folder where AM," + "Wrapper and user code jar files will be uploaded"); opts.addOption("debugYarn", false, "Specifies the debug flag"); }
From source file:runtime.starter.MPJYarnClient.java
License:Open Source License
public void run() throws Exception { Map<String, String> map = System.getenv(); try {/*from w w w .j a v a 2 s . co m*/ mpjHomeDir = map.get("MPJ_HOME"); if (mpjHomeDir == null) { throw new Exception("[MPJRun.java]:MPJ_HOME environment found.."); } } catch (Exception exc) { System.out.println("[MPJRun.java]:" + exc.getMessage()); exc.printStackTrace(); return; } // Copy the application master jar to HDFS // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); /* Path dataset = new Path(fs.getHomeDirectory(),"/dataset"); FileStatus datasetFile = fs.getFileStatus(dataset); BlockLocation myBlocks [] = fs.getFileBlockLocations(datasetFile,0,datasetFile.getLen()); for(BlockLocation b : myBlocks){ System.out.println("\n--------------------"); System.out.println("Length "+b.getLength()); for(String host : b.getHosts()){ System.out.println("host "+host); } } */ Path source = new Path(mpjHomeDir + "/lib/mpj-app-master.jar"); String pathSuffix = hdfsFolder + "mpj-app-master.jar"; Path dest = new Path(fs.getHomeDirectory(), pathSuffix); if (debugYarn) { logger.info("Uploading mpj-app-master.jar to: " + dest.toString()); } fs.copyFromLocalFile(false, true, source, dest); FileStatus destStatus = fs.getFileStatus(dest); Path wrapperSource = new Path(mpjHomeDir + "/lib/mpj-yarn-wrapper.jar"); String wrapperSuffix = hdfsFolder + "mpj-yarn-wrapper.jar"; Path wrapperDest = new Path(fs.getHomeDirectory(), wrapperSuffix); if (debugYarn) { logger.info("Uploading mpj-yarn-wrapper.jar to: " + wrapperDest.toString()); } fs.copyFromLocalFile(false, true, wrapperSource, wrapperDest); Path userJar = new Path(jarPath); String userJarSuffix = hdfsFolder + "user-code.jar"; Path userJarDest = new Path(fs.getHomeDirectory(), userJarSuffix); if (debugYarn) { logger.info("Uploading user-code.jar to: " + userJarDest.toString()); } fs.copyFromLocalFile(false, true, userJar, userJarDest); YarnConfiguration conf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); if (debugYarn) { YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics(); logger.info("\nNodes Information"); logger.info("Number of NM: " + metrics.getNumNodeManagers() + "\n"); List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); for (NodeReport n : nodeReports) { logger.info("NodeId: " + n.getNodeId()); logger.info("RackName: " + n.getRackName()); logger.info("Total Memory: " + n.getCapability().getMemory()); logger.info("Used Memory: " + n.getUsed().getMemory()); logger.info("Total vCores: " + n.getCapability().getVirtualCores()); logger.info("Used vCores: " + n.getUsed().getVirtualCores() + "\n"); } } logger.info("Creating server socket at HOST " + serverName + " PORT " + serverPort + " \nWaiting for " + np + " processes to connect..."); // Creating a server socket for incoming connections try { servSock = new ServerSocket(serverPort); infoSock = new ServerSocket(); TEMP_PORT = findPort(infoSock); } catch (Exception e) { e.printStackTrace(); } // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); int maxMem = appResponse.getMaximumResourceCapability().getMemory(); if (debugYarn) { logger.info("Max memory capability resources in cluster: " + maxMem); } if (amMem > maxMem) { amMem = maxMem; logger.info("AM memory specified above threshold of cluster " + "Using maximum memory for AM container: " + amMem); } int maxVcores = appResponse.getMaximumResourceCapability().getVirtualCores(); if (debugYarn) { logger.info("Max vCores capability resources in cluster: " + maxVcores); } if (amCores > maxVcores) { amCores = maxVcores; logger.info("AM virtual cores specified above threshold of cluster " + "Using maximum virtual cores for AM container: " + amCores); } // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); List<String> commands = new ArrayList<String>(); commands.add("$JAVA_HOME/bin/java"); commands.add("-Xmx" + amMem + "m"); commands.add("runtime.starter.MPJAppMaster"); commands.add("--np"); commands.add(String.valueOf(np)); commands.add("--serverName"); commands.add(serverName); //server name commands.add("--ioServerPort"); commands.add(Integer.toString(serverPort)); //server port commands.add("--deviceName"); commands.add(deviceName); //device name commands.add("--className"); commands.add(className); //class name commands.add("--wdir"); commands.add(workingDirectory); //wdir commands.add("--psl"); commands.add(Integer.toString(psl)); //protocol switch limit commands.add("--wireUpPort"); commands.add(String.valueOf(TEMP_PORT)); //for sharing ports & rank commands.add("--wrapperPath"); commands.add(wrapperDest.toString());//MPJYarnWrapper.jar HDFS path commands.add("--userJarPath"); commands.add(userJarDest.toString());//User Jar File HDFS path commands.add("--mpjContainerPriority"); commands.add(mpjContainerPriority);// priority for mpj containers commands.add("--containerMem"); commands.add(containerMem); commands.add("--containerCores"); commands.add(containerCores); if (debugYarn) { commands.add("--debugYarn"); } if (appArgs != null) { commands.add("--appArgs"); for (int i = 0; i < appArgs.length; i++) { commands.add(appArgs[i]); } } amContainer.setCommands(commands); //set commands // Setup local Resource for ApplicationMaster LocalResource appMasterJar = Records.newRecord(LocalResource.class); appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(dest)); appMasterJar.setSize(destStatus.getLen()); appMasterJar.setTimestamp(destStatus.getModificationTime()); appMasterJar.setType(LocalResourceType.ARCHIVE); appMasterJar.setVisibility(LocalResourceVisibility.APPLICATION); amContainer.setLocalResources(Collections.singletonMap("mpj-app-master.jar", appMasterJar)); // Setup CLASSPATH for ApplicationMaster // Setting up the environment Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMem); capability.setVirtualCores(amCores); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(appName); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue(yarnQueue); // queue Priority priority = Priority.newInstance(amPriority); appContext.setPriority(priority); ApplicationId appId = appContext.getApplicationId(); //Adding ShutDown Hook Runtime.getRuntime().addShutdownHook(new KillYarnApp(appId, yarnClient)); // Submit application System.out.println("Submitting Application: " + appContext.getApplicationName() + "\n"); try { isRunning = true; yarnClient.submitApplication(appContext); } catch (Exception exp) { System.err.println("Error Submitting Application"); exp.printStackTrace(); } // np = number of processes , + 1 for Application Master container IOMessagesThread[] ioThreads = new IOMessagesThread[np + 1]; peers = new String[np]; socketList = new Vector<Socket>(); int wport = 0; int rport = 0; int rank = 0; // np + 1 IOThreads for (int i = 0; i < (np + 1); i++) { try { sock = servSock.accept(); //start IO thread to read STDOUT and STDERR from wrappers IOMessagesThread io = new IOMessagesThread(sock); ioThreads[i] = io; ioThreads[i].start(); } catch (Exception e) { System.err.println("Error accepting connection from peer socket.."); e.printStackTrace(); } } // Loop to read port numbers from Wrapper.java processes // and to create WRAPPER_INFO (containing all IPs and ports) String WRAPPER_INFO = "#Peer Information"; for (int i = np; i > 0; i--) { try { sock = infoSock.accept(); DataOutputStream out = new DataOutputStream(sock.getOutputStream()); DataInputStream in = new DataInputStream(sock.getInputStream()); if (in.readUTF().startsWith("Sending Info")) { wport = in.readInt(); rport = in.readInt(); rank = in.readInt(); peers[rank] = ";" + sock.getInetAddress().getHostAddress() + "@" + rport + "@" + wport + "@" + rank; socketList.add(sock); } } catch (Exception e) { System.err.println("[MPJYarnClient.java]: Error accepting" + " connection from peer socket!"); e.printStackTrace(); } } for (int i = 0; i < np; i++) { WRAPPER_INFO += peers[i]; } // Loop to broadcast WRAPPER_INFO to all Wrappers for (int i = np; i > 0; i--) { try { sock = socketList.get(np - i); DataOutputStream out = new DataOutputStream(sock.getOutputStream()); out.writeUTF(WRAPPER_INFO); out.flush(); sock.close(); } catch (Exception e) { System.err.println("[MPJYarnClient.java]: Error closing" + " connection from peer socket.."); e.printStackTrace(); } } try { infoSock.close(); } catch (IOException exp) { exp.printStackTrace(); } // wait for all IO Threads to complete for (int i = 0; i < (np + 1); i++) { ioThreads[i].join(); } isRunning = true; System.out.println("\nApplication Statistics!"); while (true) { appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); fStatus = appReport.getFinalApplicationStatus(); if (appState == YarnApplicationState.FINISHED) { isRunning = false; if (fStatus == FinalApplicationStatus.SUCCEEDED) { System.out.println("State: " + fStatus); } else { System.out.println("State: " + fStatus); } break; } else if (appState == YarnApplicationState.KILLED) { isRunning = false; System.out.println("State: " + appState); break; } else if (appState == YarnApplicationState.FAILED) { isRunning = false; System.out.println("State: " + appState); break; } Thread.sleep(100); } try { if (debugYarn) { logger.info("Cleaning the files from hdfs: "); logger.info("1) " + dest.toString()); logger.info("2) " + wrapperDest.toString()); logger.info("3) " + userJarDest.toString()); } fs.delete(dest); fs.delete(wrapperDest); fs.delete(userJarDest); } catch (IOException exp) { exp.printStackTrace(); } System.out.println("Application ID: " + appId + "\n" + "Application User: " + appReport.getUser() + "\n" + "RM Queue: " + appReport.getQueue() + "\n" + "Start Time: " + appReport.getStartTime() + "\n" + "Finish Time: " + appReport.getFinishTime()); }
From source file:wwh.exec.Main.java
private static YarnConfiguration createClusterConfiguration(String ip) { //create yarn configuration for c0 //A YarnConfiguration object is part of hadoop Yarn and //is allows us to connect to a specific cluster and send application //masters into it. YarnConfiguration configuration = new YarnConfiguration(); //configure the resource manager host name to submit jobs into configuration.set("yarn.resourcemanager.hostname", ip); //configure the hdfs to use configuration.set("fs.defaultFS", "hdfs://" + ip + ":9000"); // configuration.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, ip + ":2181"); return configuration; }
From source file:x10.x10rt.yarn.ApplicationMaster.java
License:Open Source License
public ApplicationMaster(String[] args) throws Exception { this.conf = new YarnConfiguration(); Map<String, String> envs = System.getenv(); ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId()); initialNumPlaces = Integer.parseInt(envs.get(X10_NPLACES)); coresPerPlace = Integer.parseInt(envs.get(X10_NTHREADS)); nthreads = coresPerPlace;//from w ww . j av a 2 s .c o m links = new HashMap<Integer, ApplicationMaster.CommunicationLink>(initialNumPlaces); places = new HashMap<ContainerId, Integer>(initialNumPlaces); pendingReads = new HashMap<SocketChannel, ByteBuffer>(); selector = Selector.open(); this.args = args; this.appName = System.getProperty(X10_YARN_MAIN); // look for max memory argument this.memoryPerPlaceInMb = 4024; // default of 1Gb per place for (int i = 0; i < args.length; i++) { try { if (args[i].startsWith("-Xmx")) { if (args[i].toLowerCase().endsWith("g")) this.memoryPerPlaceInMb = Integer.parseInt(args[i].substring(4, args[i].length() - 1)) * 1024; else if (args[i].toLowerCase().endsWith("m")) this.memoryPerPlaceInMb = Integer.parseInt(args[i].substring(4, args[i].length() - 1)); else if (args[i].toLowerCase().endsWith("k")) this.memoryPerPlaceInMb = Integer.parseInt(args[i].substring(4, args[i].length() - 1)) / 1024; else this.memoryPerPlaceInMb = Integer.parseInt(args[i].substring(4)) / 1024 / 1024; break; } } catch (NumberFormatException e) { // ignore, use default value e.printStackTrace(); } } if (envs.containsKey(X10YARNENV_ + X10_YARN_KILL)) { placeKiller = new ScheduledThreadPoolExecutor(1); // things to kill takes the form place:delayInSeconds,place:delayInSeconds,etc. e.g. "2:2,3:3" will kill place 2 after 2 seconds, 3 after 3 seconds String thingsToKill = System.getenv(X10YARNENV_ + X10_YARN_KILL); // TODO: format error checking String[] sets = thingsToKill.split(","); pendingKills = new HashMap<Integer, Integer>(sets.length); for (String set : sets) { String[] place_delay = set.split(":"); int place = Integer.parseInt(place_delay[0]); int delay = Integer.parseInt(place_delay[1]); pendingKills.put(place, delay); } } else { placeKiller = null; pendingKills = null; } }
From source file:x10.x10rt.yarn.Client.java
License:Open Source License
public Client(String[] args) { this.conf = new YarnConfiguration(); //this.conf.set(YarnConfiguration.RM_HOSTNAME, args[0]); //this.conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 1000); //this.conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1000); //this.conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "hdfs://"+args[0]); this.appMasterMainClass = ApplicationMaster.class.getName(); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(this.conf); this.args = args; // find the original classpath argument from the x10 script String prefix = "-Djava.class.path="; for (int classPathArg = 0; classPathArg < args.length; classPathArg++) { if (args[classPathArg].startsWith(prefix)) { classPath = args[classPathArg].substring(prefix.length()); this.classPathArg = classPathArg; break; }//from w w w . j av a 2 s . c o m } // find the first non-jvm argument, which is the main class name int mainClassArg = 0; for (; mainClassArg < args.length; mainClassArg++) { if (args[mainClassArg].charAt(0) != '-') break; } this.mainClassArg = mainClassArg; int lastslash = args[mainClassArg].lastIndexOf('/'); if (lastslash == -1) this.appName = args[mainClassArg]; else this.appName = args[mainClassArg].substring(lastslash + 1); }
From source file:yarn.demo.CommandBasedYarnApplicationClusterDemo.java
License:Apache License
/** * If running in Mini-Cluster (see yarn-test-cluster project), make sure you start it * by executing StartMiniCluster.java first. *//* www . ja v a2 s . c om*/ public static void main(String[] args) throws Exception { MiniClusterUtils.startMiniCluster(); ConfigUtils.addToClasspath(new File("mini-cluster-config")); YarnApplication<Void> yarnApplication = YarnAssembly.forApplicationContainer("ping -c 4 google.com") .containerCount(4).withApplicationMaster(new YarnConfiguration()) .build("CommandBasedYarnApplicationDemo"); yarnApplication.launch(); yarnApplication.awaitFinish(); MiniClusterUtils.stoptMiniCluster(); }