List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration()
From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncherTest.java
License:Apache License
/** * Test that the job cleanup call is called *//*www . ja v a 2 s . c o m*/ @Test public void testJobCleanup() throws Exception { ContainerId containerId = ContainerId .newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0), 0); TestApplicationMaster appMaster = Mockito .spy(new TestApplicationMaster("testApp", containerId, config, new YarnConfiguration())); GobblinHelixMultiManager mockMultiManager = Mockito.mock(GobblinHelixMultiManager.class); appMaster.setMultiManager(mockMultiManager); appMaster.start(); Mockito.verify(mockMultiManager, times(1)).cleanUpJobs(); }
From source file:org.apache.gobblin.yarn.YarnServiceTest.java
License:Apache License
@BeforeClass public void setUp() throws Exception { // Set java home in environment since it isn't set on some systems String javaHome = System.getProperty("java.home"); setEnv("JAVA_HOME", javaHome); this.clusterConf = new YarnConfiguration(); this.clusterConf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "100"); this.clusterConf.set(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, "10000"); this.clusterConf.set(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, "60000"); this.yarnCluster = this.closer.register(new MiniYARNCluster("YarnServiceTestCluster", 4, 1, 1)); this.yarnCluster.init(this.clusterConf); this.yarnCluster.start(); // YARN client should not be started before the Resource Manager is up AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() { @Override/*w w w. ja v a2 s .co m*/ public boolean apply(Void input) { return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0"); } }, "Waiting for RM"); this.yarnClient = this.closer.register(YarnClient.createYarnClient()); this.yarnClient.init(this.clusterConf); this.yarnClient.start(); URL url = YarnServiceTest.class.getClassLoader() .getResource(YarnServiceTest.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); this.config = ConfigFactory.parseURL(url).resolve(); // Start a dummy application manager so that the YarnService can use the AM-RM token. startApp(); // create and start the test yarn service this.yarnService = new TestYarnService(this.config, "testApp", "appId", this.clusterConf, FileSystem.getLocal(new Configuration()), this.eventBus); this.yarnService.startUp(); }
From source file:org.apache.hama.bsp.ApplicationMaster.java
License:Apache License
public ApplicationMaster() { // Set up the configuration this.localConf = new YarnConfiguration(); }
From source file:org.apache.hama.bsp.BSPApplicationMaster.java
License:Apache License
private BSPApplicationMaster(String[] args) throws Exception { if (args.length != 1) { throw new IllegalArgumentException(); }// w w w.ja v a2 s. c om this.jobFile = args[0]; this.localConf = new YarnConfiguration(); this.jobConf = getSubmitConfiguration(jobFile); fs = FileSystem.get(jobConf); this.applicationName = jobConf.get("bsp.job.name", "<no bsp job name defined>"); if (applicationName.isEmpty()) { this.applicationName = "<no bsp job name defined>"; } this.appAttemptId = getApplicationAttemptId(); this.yarnRPC = YarnRPC.create(localConf); this.clock = new SystemClock(); this.startTime = clock.getTime(); this.jobId = new BSPJobID(appAttemptId.toString(), 0); this.hostname = BSPNetUtils.getCanonicalHostname(); this.clientPort = BSPNetUtils.getFreePort(12000); // start our synchronization service startSyncServer(); startRPCServers(); /* * Make sure that this executes after the start the RPC servers, because we * are readjusting the configuration. */ rewriteSubmitConfiguration(jobFile, jobConf); String jobSplit = jobConf.get("bsp.job.split.file"); splits = null; if (jobSplit != null) { DataInputStream splitFile = fs.open(new Path(jobSplit)); try { splits = BSPJobClient.readSplitFile(splitFile); } finally { splitFile.close(); } } this.amrmRPC = getYarnRPCConnection(localConf); registerApplicationMaster(amrmRPC, hostname, clientPort, "http://localhost:8080"); }
From source file:org.apache.helix.provisioning.yarn.AppLauncher.java
License:Apache License
private void init() throws Exception { _applicationSpec = _applicationSpecFactory.fromYaml(new FileInputStream(_yamlConfigFile)); _appMasterConfig = new AppMasterConfig(); appMasterArchive = new File(_applicationSpec.getAppMasterPackage()); yarnClient = YarnClient.createYarnClient(); _conf = new YarnConfiguration(); yarnClient.init(_conf);/* w ww . j a v a 2 s. co m*/ }
From source file:org.apache.helix.provisioning.yarn.AppMasterLauncher.java
License:Apache License
public static void main(String[] args) throws Exception { Map<String, String> env = System.getenv(); LOG.info("Starting app master with the following environment variables"); for (String key : env.keySet()) { LOG.info(key + "\t\t=" + env.get(key)); }/* w w w. j a v a 2 s .com*/ Options opts; opts = new Options(); opts.addOption("num_containers", true, "Number of containers"); // START ZOOKEEPER String dataDir = "dataDir"; String logDir = "logDir"; IDefaultNameSpace defaultNameSpace = new IDefaultNameSpace() { @Override public void createDefaultNameSpace(ZkClient zkClient) { } }; try { FileUtils.deleteDirectory(new File(dataDir)); FileUtils.deleteDirectory(new File(logDir)); } catch (IOException e) { LOG.error(e); } final ZkServer server = new ZkServer(dataDir, logDir, defaultNameSpace); server.start(); // start Generic AppMaster that interacts with Yarn RM AppMasterConfig appMasterConfig = new AppMasterConfig(); String containerIdStr = appMasterConfig.getContainerId(); ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId(); String configFile = AppMasterConfig.AppEnvironment.APP_SPEC_FILE.toString(); String className = appMasterConfig.getApplicationSpecFactory(); GenericApplicationMaster genericApplicationMaster = new GenericApplicationMaster(appAttemptID); try { genericApplicationMaster.start(); } catch (Exception e) { LOG.error("Unable to start application master: ", e); } ApplicationSpecFactory factory = HelixYarnUtil.createInstance(className); // TODO: Avoid setting static variable. YarnProvisioner.applicationMaster = genericApplicationMaster; YarnProvisioner.applicationMasterConfig = appMasterConfig; ApplicationSpec applicationSpec = factory.fromYaml(new FileInputStream(configFile)); YarnProvisioner.applicationSpec = applicationSpec; String zkAddress = appMasterConfig.getZKAddress(); String clusterName = appMasterConfig.getAppName(); // CREATE CLUSTER and setup the resources // connect ZkHelixConnection connection = new ZkHelixConnection(zkAddress); connection.connect(); // create the cluster ClusterId clusterId = ClusterId.from(clusterName); ClusterAccessor clusterAccessor = connection.createClusterAccessor(clusterId); StateModelDefinition statelessService = new StateModelDefinition( StateModelConfigGenerator.generateConfigForStatelessService()); StateModelDefinition taskStateModel = new StateModelDefinition( StateModelConfigGenerator.generateConfigForTaskStateModel()); clusterAccessor.createCluster(new ClusterConfig.Builder(clusterId).addStateModelDefinition(statelessService) .addStateModelDefinition(taskStateModel).build()); for (String service : applicationSpec.getServices()) { String resourceName = service; // add the resource with the local provisioner ResourceId resourceId = ResourceId.from(resourceName); ServiceConfig serviceConfig = applicationSpec.getServiceConfig(resourceName); serviceConfig.setSimpleField("service_name", service); int numContainers = serviceConfig.getIntField("num_containers", 1); YarnProvisionerConfig provisionerConfig = new YarnProvisionerConfig(resourceId); provisionerConfig.setNumContainers(numContainers); AutoRebalanceModeISBuilder idealStateBuilder = new AutoRebalanceModeISBuilder(resourceId); idealStateBuilder.setStateModelDefId(statelessService.getStateModelDefId()); idealStateBuilder.add(PartitionId.from(resourceId, "0")); idealStateBuilder.setNumReplica(1); ResourceConfig.Builder resourceConfigBuilder = new ResourceConfig.Builder( ResourceId.from(resourceName)); ResourceConfig resourceConfig = resourceConfigBuilder.provisionerConfig(provisionerConfig) .idealState(idealStateBuilder.build()) // .build(); clusterAccessor.addResource(resourceConfig); } // start controller ControllerId controllerId = ControllerId.from("controller1"); HelixController controller = connection.createController(clusterId, controllerId); controller.start(); // Start any pre-specified jobs List<TaskConfig> taskConfigs = applicationSpec.getTaskConfigs(); if (taskConfigs != null) { YarnConfiguration conf = new YarnConfiguration(); FileSystem fs; fs = FileSystem.get(conf); for (TaskConfig taskConfig : taskConfigs) { URI yamlUri = taskConfig.getYamlURI(); if (yamlUri != null && taskConfig.name != null) { InputStream is = readFromHDFS(fs, taskConfig.name, yamlUri, applicationSpec, appAttemptID.getApplicationId()); Workflow workflow = Workflow.parse(is); TaskDriver taskDriver = new TaskDriver(new ZKHelixManager(controller)); taskDriver.start(workflow); } } } Thread shutdownhook = new Thread(new Runnable() { @Override public void run() { server.shutdown(); } }); Runtime.getRuntime().addShutdownHook(shutdownhook); Thread.sleep(10000); }
From source file:org.apache.helix.provisioning.yarn.GenericApplicationMaster.java
License:Apache License
public GenericApplicationMaster(ApplicationAttemptId appAttemptID) { this.appAttemptID = appAttemptID; // Set up the configuration conf = new YarnConfiguration(); }
From source file:org.apache.helix.provisioning.yarn.YarnProvisioner.java
License:Apache License
private ContainerLaunchContext createLaunchContext(ContainerId containerId, Container container, Participant participant) throws Exception { ContainerLaunchContext participantContainer = Records.newRecord(ContainerLaunchContext.class); // Map<String, String> envs = System.getenv(); String appName = applicationMasterConfig.getAppName(); int appId = applicationMasterConfig.getAppId(); String serviceName = _resourceConfig.getId().stringify(); String serviceClasspath = applicationMasterConfig.getClassPath(serviceName); String mainClass = applicationMasterConfig.getMainClass(serviceName); String zkAddress = applicationMasterConfig.getZKAddress(); // set the localresources needed to launch container Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LocalResource servicePackageResource = Records.newRecord(LocalResource.class); YarnConfiguration conf = new YarnConfiguration(); FileSystem fs;/*from ww w . j a v a2 s . c om*/ fs = FileSystem.get(conf); String pathSuffix = appName + "/" + appId + "/" + serviceName + ".tar"; Path dst = new Path(fs.getHomeDirectory(), pathSuffix); FileStatus destStatus = fs.getFileStatus(dst); // Set the type of resource - file or archive // archives are untarred at destination // we don't need the jar file to be untarred for now servicePackageResource.setType(LocalResourceType.ARCHIVE); // Set visibility of the resource // Setting to most private option servicePackageResource.setVisibility(LocalResourceVisibility.APPLICATION); // Set the resource to be copied over servicePackageResource.setResource(ConverterUtils.getYarnUrlFromPath(dst)); // Set timestamp and length of file so that the framework // can do basic sanity checks for the local resource // after it has been copied over to ensure it is the same // resource the client intended to use with the application servicePackageResource.setTimestamp(destStatus.getModificationTime()); servicePackageResource.setSize(destStatus.getLen()); LOG.info("Setting local resource:" + servicePackageResource + " for service" + serviceName); localResources.put(serviceName, servicePackageResource); // Set local resource info into app master container launch context participantContainer.setLocalResources(localResources); // Set the necessary security tokens as needed // amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); env.put(serviceName, dst.getName()); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar) .append("./*"); classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(serviceClasspath); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties"); LOG.info("Setting classpath for service:\n" + classPathEnv.toString()); env.put("CLASSPATH", classPathEnv.toString()); participantContainer.setEnvironment(env); if (applicationMaster.allTokens != null) { LOG.info("Setting tokens: " + applicationMaster.allTokens); participantContainer.setTokens(applicationMaster.allTokens); } // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + 4096 + "m"); // Set class name vargs.add(ParticipantLauncher.class.getCanonicalName()); // Set params for container participant vargs.add("--zkAddress " + zkAddress); vargs.add("--cluster " + appName); vargs.add("--participantId " + participant.getId().stringify()); vargs.add("--participantClass " + mainClass); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up container launch command " + command.toString() + " with arguments \n" + vargs); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); participantContainer.setCommands(commands); return participantContainer; }
From source file:org.apache.hoya.tools.HoyaUtils.java
License:Apache License
/** * Create a configuration with Hoya-specific tuning. * This is done rather than doing custom configs. * @return the config//ww w . j ava 2s .com */ public static YarnConfiguration createConfiguration() { YarnConfiguration conf = new YarnConfiguration(); patchConfiguration(conf); return conf; }
From source file:org.apache.hoya.yarn.client.HoyaClient.java
License:Apache License
/** * Constructor/*ww w . jav a 2s. co m*/ */ public HoyaClient() { // make sure all the yarn configs get loaded new YarnConfiguration(); log.debug("Hoya constructed"); }