List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration()
From source file:com.hazelcast.yarn.ApplicationMaster.java
License:Open Source License
public ApplicationMaster(String[] args) throws Exception { if (args.length < 1) { throw new IllegalStateException("Invalid arguments number"); }//from w ww . jav a 2 s . co m this.yarnConfiguration = new YarnConfiguration(); this.hdfs = FileSystem.get(this.yarnConfiguration); this.hazelcastPath = new Path(args[0]); String yarnConfigPath = args.length > 1 ? args[1] : ""; this.properties = new HazelcastYarnProperties(yarnConfigPath); LOG.log(Level.INFO, "HazelcastUri: {0}. YarnConfig: {1}.", new Object[] { this.hazelcastPath.toUri(), yarnConfigPath }); }
From source file:com.hazelcast.yarn.HazelcastYarnClient.java
License:Open Source License
public HazelcastYarnClient(String[] args) throws Exception { System.out.println("Starting yarn client..."); this.conf = new YarnConfiguration(); this.fs = FileSystem.get(this.conf); this.pathToYarnConfig = getYarnConfig(args); this.properties = new HazelcastYarnProperties(this.pathToYarnConfig); this.pathToAppJar = getAppMasterJarPath(args); this.hazelcastUri = setUpHazelcast(args).toUri(); }
From source file:com.ibm.bi.dml.yarn.DMLAppMaster.java
License:Open Source License
/** * /*from w ww. ja v a 2s . c o m*/ * @param args * @throws YarnException * @throws IOException */ public void runApplicationMaster(String[] args) throws YarnException, IOException { _conf = new YarnConfiguration(); //obtain application ID String containerIdString = System.getenv(Environment.CONTAINER_ID.name()); ContainerId containerId = ConverterUtils.toContainerId(containerIdString); _appId = containerId.getApplicationAttemptId().getApplicationId(); LOG.info("SystemML appplication master (applicationID: " + _appId + ")"); //initialize clients to ResourceManager AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(_conf); rmClient.start(); //register with ResourceManager rmClient.registerApplicationMaster("", 0, ""); //host, port for rm communication LOG.debug("Registered the SystemML application master with resource manager"); //start status reporter to ResourceManager DMLAppMasterStatusReporter reporter = new DMLAppMasterStatusReporter(rmClient, 10000); reporter.start(); LOG.debug("Started status reporter (heartbeat to resource manager)"); //set DMLscript app master context DMLScript.setActiveAM(); //parse input arguments String[] otherArgs = new GenericOptionsParser(_conf, args).getRemainingArgs(); //run SystemML CP FinalApplicationStatus status = null; try { //core dml script execution (equivalent to non-AM runtime) boolean success = DMLScript.executeScript(_conf, otherArgs); if (success) status = FinalApplicationStatus.SUCCEEDED; else status = FinalApplicationStatus.FAILED; } catch (DMLScriptException ex) { LOG.error(DMLYarnClient.APPMASTER_NAME + ": Failed to executed DML script due to stop call:\n\t" + ex.getMessage()); status = FinalApplicationStatus.FAILED; writeMessageToHDFSWorkingDir(ex.getMessage()); } catch (Exception ex) { LOG.error(DMLYarnClient.APPMASTER_NAME + ": Failed to executed DML script.", ex); status = FinalApplicationStatus.FAILED; } finally { //stop periodic status reports reporter.stopStatusReporter(); LOG.debug("Stopped status reporter"); //unregister resource manager client rmClient.unregisterApplicationMaster(status, "", ""); LOG.debug("Unregistered the SystemML application master"); } }
From source file:com.ibm.bi.dml.yarn.DMLYarnClient.java
License:Open Source License
/** * Method to launch the dml yarn app master and execute the given dml script * with the given configuration and jar file. * /* w w w. ja va 2 s .c o m*/ * NOTE: on launching the yarn app master, we do not explicitly probe if we * are running on a yarn or MR1 cluster. In case of MR1, already the class * YarnConfiguration will not be found and raise a classnotfound. In case of any * exception we fall back to run CP directly in the client process. * * @return true if dml program successfully executed as yarn app master * @throws IOException */ protected boolean launchDMLYarnAppmaster() throws IOException, DMLScriptException { boolean ret = false; String hdfsWD = null; try { Timing time = new Timing(true); // load yarn configuration YarnConfiguration yconf = new YarnConfiguration(); // create yarn client YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yconf); yarnClient.start(); // create application and get the ApplicationID YarnClientApplication app = yarnClient.createApplication(); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); LOG.debug("Created application (applicationID: " + appId + ")"); // prepare hdfs working directory via ApplicationID // copy script, config, jar file to hdfs hdfsWD = DMLAppMasterUtils.constructHDFSWorkingDir(_dmlConfig, appId); copyResourcesToHdfsWorkingDir(yconf, hdfsWD); //construct command line argument String command = constructAMCommand(_args, _dmlConfig); LOG.debug("Constructed application master command: \n" + command); // set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); amContainer.setCommands(Collections.singletonList(command)); amContainer.setLocalResources(constructLocalResourceMap(yconf)); amContainer.setEnvironment(constructEnvionmentMap(yconf)); // Set up resource type requirements for ApplicationMaster int memHeap = _dmlConfig.getIntValue(DMLConfig.YARN_APPMASTERMEM); int memAlloc = (int) computeMemoryAllocation(memHeap); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memAlloc); capability.setVirtualCores(NUM_CORES); LOG.debug("Requested application resources: memory=" + memAlloc + ", vcores=" + NUM_CORES); // Finally, set-up ApplicationSubmissionContext for the application String qname = _dmlConfig.getTextValue(DMLConfig.YARN_APPQUEUE); appContext.setApplicationName(APPMASTER_NAME); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue(qname); // queue LOG.debug("Configured application meta data: name=" + APPMASTER_NAME + ", queue=" + qname); // submit application (non-blocking) yarnClient.submitApplication(appContext); // Check application status periodically (and output web ui address) ApplicationReport appReport = yarnClient.getApplicationReport(appId); LOG.info("Application tracking-URL: " + appReport.getTrackingUrl()); YarnApplicationState appState = appReport.getYarnApplicationState(); YarnApplicationState oldState = appState; LOG.info("Application state: " + appState); while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) { Thread.sleep(APP_STATE_INTERVAL); //wait for 200ms appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); if (appState != oldState) { oldState = appState; LOG.info("Application state: " + appState); } } //check final status (failed or succeeded) FinalApplicationStatus finalState = appReport.getFinalApplicationStatus(); LOG.info("Application final status: " + finalState); //show application and total runtime double appRuntime = (double) (appReport.getFinishTime() - appReport.getStartTime()) / 1000; LOG.info("Application runtime: " + appRuntime + " sec."); LOG.info("Total runtime: " + String.format("%.3f", time.stop() / 1000) + " sec."); //raised script-level error in case of failed final status if (finalState != FinalApplicationStatus.SUCCEEDED) { //propagate script-level stop call message String stop_msg = readMessageToHDFSWorkingDir(_dmlConfig, yconf, appId); if (stop_msg != null) throw new DMLScriptException(stop_msg); //generic failure message throw new DMLRuntimeException( "DML yarn app master finished with final status: " + finalState + "."); } ret = true; } catch (DMLScriptException ex) { //rethrow DMLScriptException to propagate stop call throw ex; } catch (Exception ex) { LOG.error("Failed to run DML yarn app master.", ex); ret = false; } finally { //cleanup working directory if (hdfsWD != null) MapReduceTool.deleteFileIfExistOnHDFS(hdfsWD); } return ret; }
From source file:com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer.java
License:Open Source License
public static void analyzeYarnCluster(boolean verbose) { YarnConfiguration conf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf);/*from w w w . j av a 2 s . c o m*/ yarnClient.start(); analyzeYarnCluster(yarnClient, conf, verbose); }
From source file:com.ibm.bi.dml.yarn.ropt.YarnClusterAnalyzer.java
License:Open Source License
/** * /* ww w.j av a 2s. c o m*/ * @return */ private static YarnClient createYarnClient() { YarnConfiguration conf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); return yarnClient; }
From source file:com.inforefiner.hdata.SubmitClient.java
License:Apache License
/** */ public SubmitClient() throws Exception { this(new YarnConfiguration()); }
From source file:com.microsoft.canberra.tf.util.DoubleMatrixTextIO.java
License:Open Source License
@Inject public DoubleMatrixTextIO() throws IOException { final YarnConfiguration yarnConf = new YarnConfiguration(); yarnConf.set("fs.hdfs.impl", DistributedFileSystem.class.getName()); yarnConf.set("fs.file.impl", LocalFileSystem.class.getName()); this.fileSystem = FileSystem.newInstance(yarnConf); }
From source file:com.msd.gin.halyard.common.HBaseServerTestInstance.java
License:Apache License
public static synchronized Configuration getInstanceConfig() throws Exception { if (conf == null) { File zooRoot = File.createTempFile("hbase-zookeeper", ""); zooRoot.delete();/*from w ww .j a va 2s. c om*/ ZooKeeperServer zookeper = new ZooKeeperServer(zooRoot, zooRoot, 2000); ServerCnxnFactory factory = ServerCnxnFactory.createFactory(new InetSocketAddress("localhost", 0), 5000); factory.startup(zookeper); YarnConfiguration yconf = new YarnConfiguration(); String argLine = System.getProperty("argLine"); if (argLine != null) { yconf.set("yarn.app.mapreduce.am.command-opts", argLine.replace("jacoco.exec", "jacocoMR.exec")); } yconf.setBoolean(MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false); yconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); MiniMRYarnCluster miniCluster = new MiniMRYarnCluster("testCluster"); miniCluster.init(yconf); yconf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true); miniCluster.start(); File hbaseRoot = File.createTempFile("hbase-root", ""); hbaseRoot.delete(); conf = HBaseConfiguration.create(miniCluster.getConfig()); conf.set(HConstants.HBASE_DIR, hbaseRoot.toURI().toURL().toString()); conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, factory.getLocalPort()); conf.set("hbase.master.hostname", "localhost"); conf.set("hbase.regionserver.hostname", "localhost"); conf.setInt("hbase.master.info.port", -1); conf.set("hbase.fs.tmp.dir", new File(System.getProperty("java.io.tmpdir")).toURI().toURL().toString()); LocalHBaseCluster cluster = new LocalHBaseCluster(conf); cluster.startup(); } return conf; }
From source file:com.qubole.rubix.hadoop2.hadoop2CM.Hadoop2ClusterManager.java
License:Apache License
@Override public void initialize(Configuration conf) { super.initialize(conf); yconf = new YarnConfiguration(); this.address = yconf.get(addressConf, address); this.serverAddress = address.substring(0, address.indexOf(":")); this.serverPort = Integer.parseInt(address.substring(address.indexOf(":") + 1)); ExecutorService executor = Executors.newSingleThreadExecutor(); nodesCache = CacheBuilder.newBuilder().refreshAfterWrite(getNodeRefreshTime(), TimeUnit.SECONDS) .build(CacheLoader.asyncReloading(new CacheLoader<String, List<String>>() { @Override// w w w . ja va 2 s .co m public List<String> load(String s) throws Exception { if (!isMaster) { // First time all nodes start assuming themselves as master and down the line figure out their role // Next time onwards, only master will be fetching the list of nodes return ImmutableList.of(); } try { StringBuffer response = new StringBuffer(); URL obj = getNodeURL(); HttpURLConnection httpcon = (HttpURLConnection) obj.openConnection(); httpcon.setRequestMethod("GET"); log.debug("Sending 'GET' request to URL: " + obj.toString()); int responseCode = httpcon.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { BufferedReader in = new BufferedReader( new InputStreamReader(httpcon.getInputStream())); String inputLine; while ((inputLine = in.readLine()) != null) { response.append(inputLine); } in.close(); httpcon.disconnect(); } else { log.info("/ws/v1/cluster/nodes failed due to " + responseCode + ". Setting this node as worker."); isMaster = false; httpcon.disconnect(); return ImmutableList.of(); } Gson gson = new Gson(); Type type = new TypeToken<Nodes>() { }.getType(); Nodes nodes = gson.fromJson(response.toString(), type); List<Elements> allNodes = nodes.getNodes().getNode(); Set<String> hosts = new HashSet<>(); for (Elements node : allNodes) { String state = node.getState(); log.debug("Hostname: " + node.getNodeHostName() + "State: " + state); //keep only healthy data nodes if (state.equalsIgnoreCase("Running") || state.equalsIgnoreCase("New") || state.equalsIgnoreCase("Rebooted")) { hosts.add(node.getNodeHostName()); } } if (hosts.isEmpty()) { throw new Exception("No healthy data nodes found."); } List<String> hostList = Lists.newArrayList(hosts.toArray(new String[0])); Collections.sort(hostList); log.debug("Hostlist: " + hostList.toString()); return hostList; } catch (Exception e) { throw Throwables.propagate(e); } } }, executor)); }