List of usage examples for org.apache.hadoop.yarn.api ApplicationConstants LOG_DIR_EXPANSION_VAR
String LOG_DIR_EXPANSION_VAR
To view the source code for org.apache.hadoop.yarn.api ApplicationConstants LOG_DIR_EXPANSION_VAR.
Click Source Link
From source file:com.flyhz.avengers.framework.AvengersAppMaster.java
License:Apache License
private void crawl() { LOG.info("App crawl"); initCommon();/*from w w w. ja v a 2 s . com*/ currentProcess = "crawl"; numTotalContainers = domainRootForCrawlSet.size(); LOG.info("initCrawl CLASSPATH -> " + System.getenv("CLASSPATH")); LOG.info("domainRootForCrawlSet.size() > {}", domainRootForCrawlSet.size()); if (numTotalContainers > 0) { for (String root : domainRootForCrawlSet) { Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); // Set class name vargs.add("-url " + root); vargs.add("-batchId " + version); // Add log redirect params vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/Crawl.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/Crawl.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); cmds.add(command.toString()); } for (int i = 0; i < numTotalContainers; ++i) { ContainerRequest containerAsk = setupContainerAskForRM(); amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainers); while (!done && (numCompletedContainers.get() != numTotalContainers)) { try { Thread.sleep(200); } catch (InterruptedException ex) { } } } }
From source file:com.flyhz.avengers.framework.AvengersClient.java
License:Apache License
private List<String> getInitEnvCmd() { // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); String appTempDir = conf.get("hadoop.tmp.dir"); vargs.add("/bin/mkdir -p"); vargs.add(appTempDir + "/" + batchId); vargs.add("&&"); vargs.add(Environment.HADOOP_YARN_HOME.$() + "/bin/hadoop"); vargs.add("fs"); vargs.add("-copyToLocal"); vargs.add(hdfsPath);// ww w. j a v a2 s.c om vargs.add(appTempDir + "/" + batchId); vargs.add("&&"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); vargs.add("-classpath"); vargs.add("$CLASSPATH:" + appTempDir + "/" + batchId + "/avengers.jar"); vargs.add("-Xmx" + amMemory + "m"); vargs.add(InitEnvApplication.class.getName()); vargs.add("-batchId"); vargs.add(String.valueOf(this.batchId)); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/InitEnvApplication.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/InitEnvApplication.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app client command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); return commands; }
From source file:com.flyhz.avengers.framework.AvengersClient.java
License:Apache License
private List<String> getCrawlCmd() { // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app crawl command"); String appTempDir = conf.get("hadoop.tmp.dir"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); vargs.add("-Xmx" + amMemory + "m"); vargs.add("-classpath"); vargs.add(Environment.CLASSPATH.$() + ":" + appTempDir + "/" + batchId + "/avengers.jar"); vargs.add(CrawlApplication.class.getName()); vargs.add("-batchId"); vargs.add(String.valueOf(this.batchId)); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/CrawlApplication.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/CrawlApplication.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); }/*w ww.ja v a2s . c o m*/ LOG.info("Completed setting up app client command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); return commands; }
From source file:com.flyhz.avengers.framework.AvengersClient.java
License:Apache License
private List<String> getFetchCmd() { // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app fetch command"); String appTempDir = conf.get("hadoop.tmp.dir"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); vargs.add("-Xmx" + amMemory + "m"); vargs.add("-classpath"); vargs.add(Environment.CLASSPATH.$() + ":" + appTempDir + "/" + batchId + "/avengers.jar"); vargs.add(FetchApplication.class.getName()); vargs.add("-batchId"); vargs.add(String.valueOf(this.batchId)); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/FetchApplication.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/FetchApplication.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); }/*w ww .j av a 2 s .c o m*/ LOG.info("Completed setting up app client command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); return commands; }
From source file:com.flyhz.avengers.framework.AvengersClient.java
License:Apache License
private List<String> runAnalyze() { // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app analyze command"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); vargs.add("-Xmx" + amMemory + "m"); String appTempDir = conf.get("hadoop.tmp.dir"); vargs.add("-classpath"); vargs.add(Environment.CLASSPATH.$() + ":" + appTempDir + "/" + batchId + "/avengers.jar"); vargs.add(Analyze.class.getName()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AnalyzeApplication.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AnalyzeApplication.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); }// w ww.j a v a 2s . co m LOG.info("Completed setting up app client command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); return commands; }
From source file:com.github.hdl.tensorflow.yarn.app.TFAmContainer.java
License:Apache License
public StringBuilder makeCommands(long amMemory, String appMasterMainClass, int containerMemory, int containerVirtualCores, int workerNumContainers, int psNumContainers, String jarDfsPath, Vector<CharSequence> containerRetryOptions, String jniSoDfsPath) { // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass);/*from w w w . java2s .c om*/ // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); vargs.add(TFApplication.makeOption(TFApplication.OPT_TF_WORKER_NUM, String.valueOf(workerNumContainers))); vargs.add(TFApplication.makeOption(TFApplication.OPT_TF_PS_NUM, String.valueOf(psNumContainers))); vargs.add("--" + TFApplication.OPT_TF_SERVER_JAR + " " + String.valueOf(jarDfsPath)); if (jniSoDfsPath != null && !jniSoDfsPath.equals("")) { vargs.add(TFApplication.makeOption(TFApplication.OPT_TF_JNI_SO, jniSoDfsPath)); } vargs.addAll(containerRetryOptions); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } return command; }
From source file:com.github.hdl.tensorflow.yarn.app.TFContainer.java
License:Apache License
public StringBuilder makeCommands(long containerMemory, String clusterSpec, String jobName, int taskIndex) { // Set the necessary command to execute on the allocated container Vector<CharSequence> vargs = new Vector<CharSequence>(5); vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"); //vargs.add("-Xmx" + containerMemory + "m"); vargs.add("-Xmx" + containerMemory + "m"); String containerClassName = TFServerLauncher.class.getName(); vargs.add(containerClassName);//from w w w . j a v a 2 s.com vargs.add("--" + TFServerLauncher.OPT_CS + " " + clusterSpec); vargs.add("--" + TFServerLauncher.OPT_JN + " " + jobName); vargs.add("--" + TFServerLauncher.OPT_TI + " " + taskIndex); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerLauncher." + ApplicationConstants.STDOUT); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerLauncher." + ApplicationConstants.STDERR); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } return command; }
From source file:com.github.sakserv.minicluster.simpleyarnapp.Client.java
License:Apache License
public void run(String[] args) throws Exception { final String command = args[0]; final int n = Integer.valueOf(args[1]); final Path jarPath = new Path(args[2]); final String resourceManagerAddress = args[3]; final String resourceManagerHostname = args[4]; final String resourceManagerSchedulerAddress = args[5]; final String resourceManagerResourceTrackerAddress = args[6]; // Create yarnClient YarnConfiguration conf = new YarnConfiguration(); conf.set("yarn.resourcemanager.address", resourceManagerAddress); conf.set("yarn.resourcemanager.hostname", resourceManagerHostname); conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress); conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf);//from w w w . j av a2 s .c o m yarnClient.start(); // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); amContainer.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M" + " com.hortonworks.simpleyarnapp.ApplicationMaster" + " " + command + " " + String.valueOf(n) + " " + resourceManagerAddress + " " + resourceManagerHostname + " " + resourceManagerSchedulerAddress + " " + resourceManagerResourceTrackerAddress + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); // Setup jar for ApplicationMaster LocalResource appMasterJar = Records.newRecord(LocalResource.class); setupAppMasterJar(jarPath, appMasterJar); amContainer.setLocalResources(Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar)); // Setup CLASSPATH for ApplicationMaster Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); capability.setVirtualCores(1); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("simple-yarn-app"); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue("default"); // queue // Submit application ApplicationId appId = appContext.getApplicationId(); System.out.println("Submitting application " + appId); yarnClient.submitApplication(appContext); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) { Thread.sleep(100); appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); } System.out.println("Application " + appId + " finished with" + " state " + appState + " at " + appReport.getFinishTime()); }
From source file:com.gpiskas.yarn.AppMaster.java
License:Open Source License
private ContainerLaunchContext initContainer() { try {//from ww w.j a v a 2s.co m // Create Container Context ContainerLaunchContext cCLC = Records.newRecord(ContainerLaunchContext.class); cCLC.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M" + " com.gpiskas.yarn.Container" + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); // Set Container jar LocalResource jar = Records.newRecord(LocalResource.class); Utils.setUpLocalResource(Utils.YARNAPP_JAR_PATH, jar, conf); cCLC.setLocalResources(Collections.singletonMap(Utils.YARNAPP_JAR_NAME, jar)); // Set Container CLASSPATH Map<String, String> env = new HashMap<String, String>(); Utils.setUpEnv(env, conf); cCLC.setEnvironment(env); return cCLC; } catch (Exception ex) { ex.printStackTrace(); return null; } }
From source file:com.gpiskas.yarn.Client.java
License:Open Source License
private void run() throws Exception { conf = new YarnConfiguration(); // Create Yarn Client YarnClient client = YarnClient.createYarnClient(); client.init(conf);/*from www . j ava 2 s . c o m*/ client.start(); // Create Application YarnClientApplication app = client.createApplication(); // Create AM Container ContainerLaunchContext amCLC = Records.newRecord(ContainerLaunchContext.class); amCLC.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M" + " com.gpiskas.yarn.AppMaster" + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); // Set AM jar LocalResource jar = Records.newRecord(LocalResource.class); Utils.setUpLocalResource(Utils.YARNAPP_JAR_PATH, jar, conf); amCLC.setLocalResources(Collections.singletonMap(Utils.YARNAPP_JAR_NAME, jar)); // Set AM CLASSPATH Map<String, String> env = new HashMap<String, String>(); Utils.setUpEnv(env, conf); amCLC.setEnvironment(env); // Set AM resources Resource res = Records.newRecord(Resource.class); res.setMemory(256); res.setVirtualCores(1); // Create ApplicationSubmissionContext ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("YARNAPP"); appContext.setQueue("default"); appContext.setAMContainerSpec(amCLC); appContext.setResource(res); // Submit Application ApplicationId id = appContext.getApplicationId(); System.out.println("Client: Submitting " + id); client.submitApplication(appContext); ApplicationReport appReport = client.getApplicationReport(id); YarnApplicationState appState = appReport.getYarnApplicationState(); while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) { Thread.sleep(1000); appReport = client.getApplicationReport(id); appState = appReport.getYarnApplicationState(); } System.out.println("Client: Finished " + id + " with state " + appState); }