List of usage examples for org.apache.hadoop.yarn.client.api YarnClient createYarnClient
@Public public static YarnClient createYarnClient()
From source file:org.apache.tez.tests.TestExternalTezServicesErrors.java
License:Apache License
private void testFatalError(String methodName, Vertex.VertexExecutionContext lhsExecutionContext, String dagNameSuffix, List<String> expectedDiagMessages) throws IOException, TezException, YarnException, InterruptedException { TezConfiguration tezClientConf = new TezConfiguration(extServiceTestHelper.getConfForJobs()); TezClient tezClient = TezClient//from ww w .j a v a 2s. c o m .newBuilder(TestExternalTezServicesErrors.class.getSimpleName() + methodName + "_session", tezClientConf) .setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build(); ApplicationId appId = null; try { tezClient.start(); LOG.info("TezSessionStarted for " + methodName); tezClient.waitTillReady(); LOG.info("TezSession ready for submission for " + methodName); JoinValidateConfigured joinValidate = new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsExecutionContext, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, dagNameSuffix); DAG dag = joinValidate.createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()), HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH, 3); DAGClient dagClient = tezClient.submitDAG(dag); DAGStatus dagStatus = dagClient .waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS)); assertEquals(DAGStatus.State.ERROR, dagStatus.getState()); boolean foundDiag = false; for (String diag : dagStatus.getDiagnostics()) { foundDiag = checkDiag(diag, expectedDiagMessages); if (foundDiag) { break; } } appId = tezClient.getAppMasterApplicationId(); assertTrue(foundDiag); } catch (InterruptedException e) { e.printStackTrace(); } finally { tezClient.stop(); } // Verify the state of the application. if (appId != null) { YarnClient yarnClient = YarnClient.createYarnClient(); try { yarnClient.init(tezClientConf); yarnClient.start(); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); while (!EnumSet .of(YarnApplicationState.FINISHED, YarnApplicationState.FAILED, YarnApplicationState.KILLED) .contains(appState)) { Thread.sleep(200L); appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); } // TODO Workaround for YARN-4554. AppReport does not provide diagnostics - need to fetch them from ApplicationAttemptReport ApplicationAttemptId appAttemptId = appReport.getCurrentApplicationAttemptId(); ApplicationAttemptReport appAttemptReport = yarnClient.getApplicationAttemptReport(appAttemptId); String diag = appAttemptReport.getDiagnostics(); assertEquals(FinalApplicationStatus.FAILED, appReport.getFinalApplicationStatus()); assertEquals(YarnApplicationState.FINISHED, appReport.getYarnApplicationState()); checkDiag(diag, expectedDiagMessages); } finally { yarnClient.stop(); } } }
From source file:org.conan.myhadoop02.mr.yarntest.Client.java
License:Apache License
Client(String appMasterMainClass, Configuration conf) {
this.conf = conf;
this.appMasterMainClass = appMasterMainClass;
yarnClient = YarnClient.createYarnClient();
yarnClient.init(conf);//from ww w . j a va2 s .c o m
opts = new Options();
opts.addOption("appname", true, "Application Name. Default value - DistributedShell");
opts.addOption("priority", true, "Application Priority. Default 0");
opts.addOption("queue", true, "RM Queue in which this application is to be submitted");
opts.addOption("timeout", true, "Application timeout in milliseconds");
opts.addOption("master_memory", true,
"Amount of memory in MB to be requested to run the application master");
opts.addOption("master_vcores", true,
"Amount of virtual cores to be requested to run the application master");
opts.addOption("jar", true, "Jar file containing the application master");
opts.addOption("shell_command", true, "Shell command to be executed by "
+ "the Application Master. Can only specify either --shell_command " + "or --shell_script");
opts.addOption("shell_script", true, "Location of the shell script to be "
+ "executed. Can only specify either --shell_command or --shell_script");
opts.addOption("shell_args", true,
"Command line args for the shell script." + "Multiple args can be separated by empty space.");
opts.getOption("shell_args").setArgs(Option.UNLIMITED_VALUES);
opts.addOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs");
opts.addOption("shell_cmd_priority", true, "Priority for the shell command containers");
opts.addOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command");
opts.addOption("container_vcores", true,
"Amount of virtual cores to be requested to run the shell command");
opts.addOption("num_containers", true, "No. of containers on which the shell command needs to be executed");
opts.addOption("log_properties", true, "log4j.properties file");
opts.addOption("keep_containers_across_application_attempts", false,
"Flag to indicate whether to keep containers across application attempts."
+ " If the flag is true, running containers will not be killed when"
+ " application attempt fails and these containers will be retrieved by"
+ " the new application attempt ");
opts.addOption("attempt_failures_validity_interval", true,
"when attempt_failures_validity_interval in milliseconds is set to > 0,"
+ "the failure number will not take failures which happen out of "
+ "the validityInterval into failure count. "
+ "If failure count reaches to maxAppAttempts, " + "the application will be failed.");
opts.addOption("debug", false, "Dump out debug information");
opts.addOption("domain", true, "ID of the timeline domain where the " + "timeline entities will be put");
opts.addOption("view_acls", true,
"Users and groups that allowed to " + "view the timeline entities in the given domain");
opts.addOption("modify_acls", true,
"Users and groups that allowed to " + "modify the timeline entities in the given domain");
opts.addOption("create", false,
"Flag to indicate whether to create the " + "domain specified with -domain.");
opts.addOption("help", false, "Print usage");
opts.addOption("node_label_expression", true,
"Node label expression to determine the nodes" + " where all the containers of this application"
+ " will be allocated, \"\" means containers"
+ " can be allocated anywhere, if you don't specify the option,"
+ " default node_label_expression of queue will be used.");
}
From source file:org.dknight.app.ApplicationMaster.java
License:Apache License
/** * Main run function for the application master * * @throws YarnException/*from www.ja va2s . c om*/ * @throws IOException */ @SuppressWarnings({ "unchecked" }) public boolean run() throws YarnException, IOException { LOG.info("Starting ApplicationMaster"); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); // Setup local RPC Server to accept status requests directly from clients // TODO need to setup a protocol for client to be able to communicate to // the RPC server // TODO use the rpc port info to register with the RM for the client to // send requests to this app master // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); // Dump out information about cluster capability as seen by the // resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (containerMemory > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); containerMemory = maxMem; } // Setup ask for containers from RM // Send request for containers to RM // Until we get our fully allocated quota, we keep on polling RM for // containers // Keep looping until all the containers are launched and shell script // executed on them ( regardless of success/failure). List<NodeReport> nodeReports = null; if (yarnClient != null) { try { nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); for (NodeReport node : nodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } } catch (YarnException e) { e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. } catch (IOException e) { e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. } } for (int i = 0; i < numTotalContainers; ++i) { ContainerRequest containerAsk = null; if (CollectionUtils.isNotEmpty(nodeReports)) { NodeReport requestNode = nodeReports.get(Math.min(i, nodeReports.size() - 1)); String[] requsetHosts = new String[] { requestNode.getNodeId().getHost() }; LOG.info("Ask container at the host list:" + requsetHosts); containerAsk = setupContainerAskForRM(requsetHosts); } else { LOG.info("Ask container with ANY host"); containerAsk = setupContainerAskForRM(null); } amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainers); while (!done && (numCompletedContainers.get() != numTotalContainers)) { try { Thread.sleep(200); } catch (InterruptedException ex) { } } finish(); return success; }
From source file:org.elasticsearch.hadoop.yarn.client.ClientRpc.java
License:Apache License
public void start() { if (client != null) { return;//w w w .j a v a 2s. co m } UserGroupInformation.setConfiguration(cfg); client = YarnClient.createYarnClient(); client.init(cfg); client.start(); }
From source file:org.etosha.cumulusonyarn.CumulusRDFRunner.java
License:Apache License
/** *//* w ww . ja v a2s . c o m*/ public CumulusRDFRunner(Configuration conf) throws Exception { this.conf = conf; yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); opts = new Options(); opts.addOption("appname", true, "Application Name. Default value - JBoss on YARN"); opts.addOption("priority", true, "Application Priority. Default 0"); opts.addOption("queue", true, "RM Queue in which this application is to be submitted"); opts.addOption("timeout", true, "Application timeout in milliseconds"); opts.addOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"); opts.addOption("jar", true, "JAR file containing the application master"); opts.addOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command"); opts.addOption("num_containers", true, "No. of containers on which the shell command needs to be executed"); opts.addOption("admin_user", true, "User id for initial administrator user"); opts.addOption("admin_password", true, "Password for initial administrator user"); opts.addOption("log_properties", true, "log4j.properties file"); opts.addOption("debug", false, "Dump out debug information"); opts.addOption("help", false, "Print usage"); }
From source file:org.hdl.caffe.yarn.app.Client.java
License:Apache License
Client(String appMasterMainClass, Configuration conf) {
this.conf = conf;
this.appMasterMainClass = appMasterMainClass;
yarnClient = YarnClient.createYarnClient();
yarnClient.init(conf);/* w ww .j a v a 2 s . com*/
opts = new Options();
opts.addOption(CaffeApplication.OPT_CAFFE_APPNAME, true, "Application Name, Default value: caffe");
opts.addOption(CaffeApplication.OPT_CAFFE_PRIORITY, true, "Application Priority. Default 0");
opts.addOption(CaffeApplication.OPT_CAFFE_QUEUE, true,
"RM Queue in which this application is to be submitted");
opts.addOption("jar", true, "Jar file containing the applicationMaster");
opts.addOption(CaffeApplication.OPT_CAFFE_MASTER_MEMORY, true,
"Amount of memory in MB to be requested to run the application master");
opts.addOption(CaffeApplication.OPT_CAFFE_MASTER_VCORES, true,
"Amount of virtual cores to be requested to run the application master");
opts.addOption(CaffeApplication.OPT_CAFFE_CONTAINER_MEMORY, true,
"Amount of memory in MB to be requested to run a tensorflow worker");
opts.addOption(CaffeApplication.OPT_CAFFE_CONTAINER_VCORES, true,
"Amount of virtual cores to be requested to run a tensorflow worker");
opts.addOption(CaffeApplication.OPT_CAFFE_LOG_PROPERTIES, true, "log4j.properties file");
opts.addOption(CaffeApplication.OPT_CAFFE_ATTEMPT_FAILURES_VALIDITY_INTERVAL, true,
"when attempt_failures_validity_interval in milliseconds is set to > 0,"
+ "the failure number will not take failures which happen out of "
+ "the validityInterval into failure count. "
+ "If failure count reaches to maxAppAttempts, " + "the application will be failed.");
opts.addOption(CaffeApplication.OPT_CAFFE_NODE_LABEL_EXPRESSION, true,
"Node label expression to determine the nodes" + " where all the containers of this application"
+ " will be allocated, \"\" means containers"
+ " can be allocated anywhere, if you don't specify the option,"
+ " default node_label_expression of queue will be used.");
opts.addOption(CaffeApplication.OPT_CAFFE_CONTAINER_RETRY_POLICY, true,
"Retry policy when container fails to run, " + "0: NEVER_RETRY, 1: RETRY_ON_ALL_ERRORS, "
+ "2: RETRY_ON_SPECIFIC_ERROR_CODES");
opts.addOption(CaffeApplication.OPT_CAFFE_CONTAINER_RETRY_ERROR_CODES, true,
"When retry policy is set to RETRY_ON_SPECIFIC_ERROR_CODES, error "
+ "codes is specified with this option, " + "e.g. --container_retry_error_codes 1,2,3");
opts.addOption(CaffeApplication.OPT_CAFFE_CONTAINER_MAX_RETRIES, true,
"If container could retry, it specifies max retires");
opts.addOption(CaffeApplication.OPT_CAFFE_CONTAINER_RETRY_INTERVAL, true,
"Interval between each retry, unit is milliseconds");
opts.addOption(CaffeApplication.OPT_CAFFE_PROCESSOR_NUM, true, "worker quantity of caffe");
opts.addOption(CaffeApplication.OPT_CAFFE_PROCESSOR_SOLVER, true, "solver_configuration");
opts.addOption(CaffeApplication.OPT_CAFFE_PROCESSOR_TRAIN, true, "training_mode");
opts.addOption(CaffeApplication.OPT_CAFFE_PROCESSOR_FEATURES, true, "name_of_output_blobs");
opts.addOption(CaffeApplication.OPT_CAFFE_PROCESSOR_LABEL, true,
"name of label blobs to be included in features");
opts.addOption(CaffeApplication.OPT_CAFFE_PROCESSOR_MODEL, true, "model path");
opts.addOption(CaffeApplication.OPT_CAFFE_PROCESSOR_OUTPUT, true, "output path");
opts.addOption(CaffeApplication.OPT_CAFFE_PROCESSOR_CONNECTION, true, "network mode");
}
From source file:org.hdl.tensorflow.yarn.client.Client.java
License:Apache License
public Client() { this(new YarnConfiguration(), YarnClient.createYarnClient()); }
From source file:org.hortonworks.dovetail.client.Client.java
License:Apache License
/** *//*w w w .j a va2 s. c om*/ public Client(DovetailConfiguration conf) throws Exception { this.conf = conf; yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); }
From source file:org.Microsoft.Telemetry.ServiceInformation.java
/** * This method Initializes all objects of the Telemetry service. * * @throws IOException ,YarnException//from ww w . ja va2 s. c o m */ public void initialization(Configuration conf) throws YarnException, IOException { telemetryconfig = TelemetryConfiguration.getActive(); client = YarnClient.createYarnClient(); Ikey = conf.get(IKEY_NAME_PROPERTY_CONFIG); dimension_from_config = conf.getValByRegex(PREFIX_CUSTOM_DIMENSINS + "*"); LOG.info(PATTERN_LOG_INFO + String.format("Updating %d dimensions from Configuration file ", dimension_from_config.size())); LOG2.info(PATTERN_LOG_INFO + String.format("Updating %d dimensions from Configuration file ", dimension_from_config.size())); if (!Ikey.equals("")) { telemetryconfig.setInstrumentationKey(Ikey); telemetry = new TelemetryClient(telemetryconfig); LOG.info(PATTERN_LOG_INFO + "Instrumentation Key initialized successfully....!"); LOG2.info(PATTERN_LOG_INFO + "Instrumentation Key initialized successfully....!"); } else { LOG.error(PATTERN_LOG_ERROR + "Instrumentation Key is not initialized Because not provided Ikey or failing to read from config file "); } }
From source file:org.starschema.hadoop.yarn.applications.distributedshell.Client.java
License:Apache License
Client(String appMasterMainClass, Configuration conf) {
this.conf = conf;
this.appMasterMainClass = appMasterMainClass;
yarnClient = YarnClient.createYarnClient();
yarnClient.init(conf);//ww w . j av a2s . co m
opts = new Options();
opts.addOption("appname", true, "Application Name. Default value - DistributedShell");
opts.addOption("priority", true, "Application Priority. Default 0");
opts.addOption("queue", true, "RM Queue in which this application is to be submitted");
opts.addOption("timeout", true, "Application timeout in milliseconds");
opts.addOption("master_memory", true,
"Amount of memory in MB to be requested to run the application master");
opts.addOption("master_vcores", true,
"Amount of virtual cores to be requested to run the application master");
opts.addOption("jar", true, "Jar file containing the application master");
opts.addOption("shell_command", true, "Shell command to be executed by "
+ "the Application Master. Can only specify either --shell_command " + "or --shell_script");
opts.addOption("shell_script", true, "Location of the shell script to be "
+ "executed. Can only specify either --shell_command or --shell_script");
opts.addOption("hazelcast_zip", true, "Location of the hazelcast zip.");
opts.addOption("shell_args", true,
"Command line args for the shell script." + "Multiple args can be separated by empty space.");
opts.getOption("shell_args").setArgs(Option.UNLIMITED_VALUES);
opts.addOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs");
opts.addOption("shell_cmd_priority", true, "Priority for the shell command containers");
opts.addOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command");
opts.addOption("container_vcores", true,
"Amount of virtual cores to be requested to run the shell command");
opts.addOption("num_containers", true, "No. of containers on which the shell command needs to be executed");
opts.addOption("log_properties", true, "log4j.properties file");
opts.addOption("keep_containers_across_application_attempts", false,
"Flag to indicate whether to keep containers across application attempts."
+ " If the flag is true, running containers will not be killed when"
+ " application attempt fails and these containers will be retrieved by"
+ " the new application attempt ");
opts.addOption("attempt_failures_validity_interval", true,
"when attempt_failures_validity_interval in milliseconds is set to > 0,"
+ "the failure number will not take failures which happen out of "
+ "the validityInterval into failure count. "
+ "If failure count reaches to maxAppAttempts, " + "the application will be failed.");
opts.addOption("debug", false, "Dump out debug information");
opts.addOption("domain", true, "ID of the timeline domain where the " + "timeline entities will be put");
opts.addOption("view_acls", true,
"Users and groups that allowed to " + "view the timeline entities in the given domain");
opts.addOption("modify_acls", true,
"Users and groups that allowed to " + "modify the timeline entities in the given domain");
opts.addOption("create", false,
"Flag to indicate whether to create the " + "domain specified with -domain.");
opts.addOption("help", false, "Print usage");
opts.addOption("node_label_expression", true,
"Node label expression to determine the nodes" + " where all the containers of this application"
+ " will be allocated, \"\" means containers"
+ " can be allocated anywhere, if you don't specify the option,"
+ " default node_label_expression of queue will be used.");
}