List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration()
From source file:org.apache.flink.yarn.FlinkYarnClient.java
License:Apache License
public FlinkYarnClient() { conf = new YarnConfiguration(); if (this.yarnClient == null) { // Create yarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf);/*w w w.j a va2 s . c o m*/ yarnClient.start(); } // for unit tests only if (System.getenv("IN_TESTS") != null) { try { conf.addResource(new File(System.getenv("YARN_CONF_DIR") + "/yarn-site.xml").toURI().toURL()); } catch (Throwable t) { throw new RuntimeException("Error", t); } } }
From source file:org.apache.flink.yarn.FlinkYarnClientBase.java
License:Apache License
public FlinkYarnClientBase() { conf = new YarnConfiguration(); if (this.yarnClient == null) { // Create yarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf);/* w ww. j a va2 s . c om*/ yarnClient.start(); } // for unit tests only if (System.getenv("IN_TESTS") != null) { try { conf.addResource(new File(System.getenv("YARN_CONF_DIR") + "/yarn-site.xml").toURI().toURL()); } catch (Throwable t) { throw new RuntimeException("Error", t); } } }
From source file:org.apache.flink.yarn.UtilsTest.java
License:Apache License
@Test public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Exception { new JavaTestKit(system) { {//from w w w . j a v a 2s .com final Deadline deadline = new FiniteDuration(3, TimeUnit.MINUTES).fromNow(); Configuration flinkConfig = new Configuration(); YarnConfiguration yarnConfig = new YarnConfiguration(); TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService(); String applicationMasterHostName = "localhost"; String webInterfaceURL = "foobar"; ContaineredTaskManagerParameters taskManagerParameters = new ContaineredTaskManagerParameters(1l, 1l, 1l, 1, new HashMap<String, String>()); ContainerLaunchContext taskManagerLaunchContext = mock(ContainerLaunchContext.class); int yarnHeartbeatIntervalMillis = 1000; int maxFailedContainers = 10; int numInitialTaskManagers = 5; final YarnResourceManagerCallbackHandler callbackHandler = new YarnResourceManagerCallbackHandler(); AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = mock(AMRMClientAsync.class); NMClient nodeManagerClient = mock(NMClient.class); UUID leaderSessionID = UUID.randomUUID(); final List<Container> containerList = new ArrayList<>(); for (int i = 0; i < numInitialTaskManagers; i++) { containerList.add(new TestingContainer("container_" + i, "localhost")); } doAnswer(new Answer() { int counter = 0; @Override public Object answer(InvocationOnMock invocation) throws Throwable { if (counter < containerList.size()) { callbackHandler .onContainersAllocated(Collections.singletonList(containerList.get(counter++))); } return null; } }).when(resourceManagerClient).addContainerRequest(Matchers.any(AMRMClient.ContainerRequest.class)); ActorRef resourceManager = null; ActorRef leader1; try { leader1 = system.actorOf(Props.create(TestingUtils.ForwardingActor.class, getRef(), Option.apply(leaderSessionID))); resourceManager = system.actorOf(Props.create(TestingYarnFlinkResourceManager.class, flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers, callbackHandler, resourceManagerClient, nodeManagerClient)); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); final AkkaActorGateway leader1Gateway = new AkkaActorGateway(leader1, leaderSessionID); final AkkaActorGateway resourceManagerGateway = new AkkaActorGateway(resourceManager, leaderSessionID); doAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { Container container = (Container) invocation.getArguments()[0]; resourceManagerGateway.tell(new NotifyResourceStarted( YarnFlinkResourceManager.extractResourceID(container)), leader1Gateway); return null; } }).when(nodeManagerClient).startContainer(Matchers.any(Container.class), Matchers.any(ContainerLaunchContext.class)); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.EMPTY_LIST)); for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> taskManagerRegisteredFuture = resourceManagerGateway .ask(new NotifyWhenResourcesRegistered(numInitialTaskManagers), deadline.timeLeft()); Await.ready(taskManagerRegisteredFuture, deadline.timeLeft()); leaderRetrievalService.notifyListener(null, null); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.EMPTY_LIST)); for (Container container : containerList) { resourceManagerGateway.tell( new NotifyResourceStarted(YarnFlinkResourceManager.extractResourceID(container)), leader1Gateway); } for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> numberOfRegisteredResourcesFuture = resourceManagerGateway .ask(RequestNumberOfRegisteredResources.Instance, deadline.timeLeft()); int numberOfRegisteredResources = (Integer) Await.result(numberOfRegisteredResourcesFuture, deadline.timeLeft()); assertEquals(numInitialTaskManagers, numberOfRegisteredResources); } finally { if (resourceManager != null) { resourceManager.tell(PoisonPill.getInstance(), ActorRef.noSender()); } } } }; }
From source file:org.apache.flink.yarn.YarnApplicationMasterRunner.java
License:Apache License
/** * The main work method, must run as a privileged action. * // w w w . j av a2 s . c o m * @return The return code for the Java process. */ protected int runApplicationMaster() { ActorSystem actorSystem = null; WebMonitor webMonitor = null; try { // ------- (1) load and parse / validate all configurations ------- // loading all config values here has the advantage that the program fails fast, if any // configuration problem occurs final String currDir = ENV.get(Environment.PWD.key()); require(currDir != null, "Current working directory variable (%s) not set", Environment.PWD.key()); // Note that we use the "appMasterHostname" given by YARN here, to make sure // we use the hostnames given by YARN consistently throughout akka. // for akka "localhost" and "localhost.localdomain" are different actors. final String appMasterHostname = ENV.get(Environment.NM_HOST.key()); require(appMasterHostname != null, "ApplicationMaster hostname variable %s not set", Environment.NM_HOST.key()); LOG.info("YARN assigned hostname for application master: {}", appMasterHostname); // Flink configuration final Map<String, String> dynamicProperties = CliFrontend .getDynamicProperties(ENV.get(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES)); LOG.debug("YARN dynamic properties: {}", dynamicProperties); final Configuration config = createConfiguration(currDir, dynamicProperties); // Hadoop/Yarn configuration (loads config data automatically from classpath files) final YarnConfiguration yarnConfig = new YarnConfiguration(); final int taskManagerContainerMemory; final int numInitialTaskManagers; final int slotsPerTaskManager; try { taskManagerContainerMemory = Integer.parseInt(ENV.get(YarnConfigKeys.ENV_TM_MEMORY)); } catch (NumberFormatException e) { throw new RuntimeException( "Invalid value for " + YarnConfigKeys.ENV_TM_MEMORY + " : " + e.getMessage()); } try { numInitialTaskManagers = Integer.parseInt(ENV.get(YarnConfigKeys.ENV_TM_COUNT)); } catch (NumberFormatException e) { throw new RuntimeException( "Invalid value for " + YarnConfigKeys.ENV_TM_COUNT + " : " + e.getMessage()); } try { slotsPerTaskManager = Integer.parseInt(ENV.get(YarnConfigKeys.ENV_SLOTS)); } catch (NumberFormatException e) { throw new RuntimeException( "Invalid value for " + YarnConfigKeys.ENV_SLOTS + " : " + e.getMessage()); } final ContaineredTaskManagerParameters taskManagerParameters = ContaineredTaskManagerParameters .create(config, taskManagerContainerMemory, slotsPerTaskManager); LOG.info("TaskManagers will be created with {} task slots", taskManagerParameters.numSlots()); LOG.info( "TaskManagers will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB", taskManagerParameters.taskManagerTotalMemoryMB(), taskManagerParameters.taskManagerHeapSizeMB(), taskManagerParameters.taskManagerDirectMemoryLimitMB()); // ----------------- (2) start the actor system ------------------- // try to start the actor system, JobManager and JobManager actor system // using the port range definition from the config. final String amPortRange = config.getString(ConfigConstants.YARN_APPLICATION_MASTER_PORT, ConfigConstants.DEFAULT_YARN_JOB_MANAGER_PORT); actorSystem = BootstrapTools.startActorSystem(config, appMasterHostname, amPortRange, LOG); final String akkaHostname = AkkaUtils.getAddress(actorSystem).host().get(); final int akkaPort = (Integer) AkkaUtils.getAddress(actorSystem).port().get(); LOG.info("Actor system bound to hostname {}.", akkaHostname); // ---- (3) Generate the configuration for the TaskManagers final Configuration taskManagerConfig = BootstrapTools.generateTaskManagerConfiguration(config, akkaHostname, akkaPort, slotsPerTaskManager, TASKMANAGER_REGISTRATION_TIMEOUT); LOG.debug("TaskManager configuration: {}", taskManagerConfig); final ContainerLaunchContext taskManagerContext = createTaskManagerContext(config, yarnConfig, ENV, taskManagerParameters, taskManagerConfig, currDir, getTaskManagerClass(), LOG); // ---- (4) start the actors and components in this order: // 1) JobManager & Archive (in non-HA case, the leader service takes this) // 2) Web Monitor (we need its port to register) // 3) Resource Master for YARN // 4) Process reapers for the JobManager and Resource Master // 1: the JobManager LOG.debug("Starting JobManager actor"); // we start the JobManager with its standard name ActorRef jobManager = JobManager .startJobManagerActors(config, actorSystem, new scala.Some<>(JobManager.JOB_MANAGER_NAME()), scala.Option.<String>empty(), getJobManagerClass(), getArchivistClass()) ._1(); // 2: the web monitor LOG.debug("Starting Web Frontend"); webMonitor = BootstrapTools.startWebMonitorIfConfigured(config, actorSystem, jobManager, LOG); final String webMonitorURL = webMonitor == null ? null : "http://" + appMasterHostname + ":" + webMonitor.getServerPort(); // 3: Flink's Yarn ResourceManager LOG.debug("Starting YARN Flink Resource Manager"); // we need the leader retrieval service here to be informed of new // leader session IDs, even though there can be only one leader ever LeaderRetrievalService leaderRetriever = LeaderRetrievalUtils.createLeaderRetrievalService(config, jobManager); Props resourceMasterProps = YarnFlinkResourceManager.createActorProps(getResourceManagerClass(), config, yarnConfig, leaderRetriever, appMasterHostname, webMonitorURL, taskManagerParameters, taskManagerContext, numInitialTaskManagers, LOG); ActorRef resourceMaster = actorSystem.actorOf(resourceMasterProps); // 4: Process reapers // The process reapers ensure that upon unexpected actor death, the process exits // and does not stay lingering around unresponsive LOG.debug("Starting process reapers for JobManager and YARN Application Master"); actorSystem.actorOf(Props.create(ProcessReaper.class, resourceMaster, LOG, ACTOR_DIED_EXIT_CODE), "YARN_Resource_Master_Process_Reaper"); actorSystem.actorOf(Props.create(ProcessReaper.class, jobManager, LOG, ACTOR_DIED_EXIT_CODE), "JobManager_Process_Reaper"); } catch (Throwable t) { // make sure that everything whatever ends up in the log LOG.error("YARN Application Master initialization failed", t); if (actorSystem != null) { try { actorSystem.shutdown(); } catch (Throwable tt) { LOG.error("Error shutting down actor system", tt); } } if (webMonitor != null) { try { webMonitor.stop(); } catch (Throwable ignored) { LOG.warn("Failed to stop the web frontend", t); } } return INIT_ERROR_EXIT_CODE; } // everything started, we can wait until all is done or the process is killed LOG.info("YARN Application Master started"); // wait until everything is done actorSystem.awaitTermination(); // if we get here, everything work out jolly all right, and we even exited smoothly if (webMonitor != null) { try { webMonitor.stop(); } catch (Throwable t) { LOG.error("Failed to stop the web frontend", t); } } return 0; }
From source file:org.apache.flink.yarn.YarnFlinkResourceManagerTest.java
License:Apache License
@Test public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Exception { new JavaTestKit(system) { {/*from ww w . ja v a2s.c o m*/ final Deadline deadline = new FiniteDuration(3, TimeUnit.MINUTES).fromNow(); Configuration flinkConfig = new Configuration(); YarnConfiguration yarnConfig = new YarnConfiguration(); SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(null, null); String applicationMasterHostName = "localhost"; String webInterfaceURL = "foobar"; ContaineredTaskManagerParameters taskManagerParameters = new ContaineredTaskManagerParameters(1L, 1L, 1L, 1, new HashMap<String, String>()); ContainerLaunchContext taskManagerLaunchContext = mock(ContainerLaunchContext.class); int yarnHeartbeatIntervalMillis = 1000; int maxFailedContainers = 10; int numInitialTaskManagers = 5; final YarnResourceManagerCallbackHandler callbackHandler = new YarnResourceManagerCallbackHandler(); AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = mock(AMRMClientAsync.class); NMClient nodeManagerClient = mock(NMClient.class); UUID leaderSessionID = UUID.randomUUID(); final List<Container> containerList = new ArrayList<>(); for (int i = 0; i < numInitialTaskManagers; i++) { Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(ContainerId.newInstance(ApplicationAttemptId .newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), i)); when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance("container", 1234)); containerList.add(mockContainer); } doAnswer(new Answer() { int counter = 0; @Override public Object answer(InvocationOnMock invocation) throws Throwable { if (counter < containerList.size()) { callbackHandler .onContainersAllocated(Collections.singletonList(containerList.get(counter++))); } return null; } }).when(resourceManagerClient).addContainerRequest(Matchers.any(AMRMClient.ContainerRequest.class)); final CompletableFuture<AkkaActorGateway> resourceManagerFuture = new CompletableFuture<>(); final CompletableFuture<AkkaActorGateway> leaderGatewayFuture = new CompletableFuture<>(); doAnswer((InvocationOnMock invocation) -> { Container container = (Container) invocation.getArguments()[0]; resourceManagerFuture.thenCombine(leaderGatewayFuture, (resourceManagerGateway, leaderGateway) -> { resourceManagerGateway.tell( new NotifyResourceStarted( YarnFlinkResourceManager.extractResourceID(container)), leaderGateway); return null; }); return null; }).when(nodeManagerClient).startContainer(Matchers.any(Container.class), Matchers.any(ContainerLaunchContext.class)); ActorRef resourceManager = null; ActorRef leader1; try { leader1 = system.actorOf(Props.create(TestingUtils.ForwardingActor.class, getRef(), Option.apply(leaderSessionID))); resourceManager = system.actorOf(Props.create(TestingYarnFlinkResourceManager.class, flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers, callbackHandler, resourceManagerClient, nodeManagerClient)); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); final AkkaActorGateway leader1Gateway = new AkkaActorGateway(leader1, leaderSessionID); final AkkaActorGateway resourceManagerGateway = new AkkaActorGateway(resourceManager, leaderSessionID); leaderGatewayFuture.complete(leader1Gateway); resourceManagerFuture.complete(resourceManagerGateway); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> taskManagerRegisteredFuture = resourceManagerGateway .ask(new NotifyWhenResourcesRegistered(numInitialTaskManagers), deadline.timeLeft()); Await.ready(taskManagerRegisteredFuture, deadline.timeLeft()); leaderRetrievalService.notifyListener(null, null); leaderRetrievalService.notifyListener(leader1.path().toString(), leaderSessionID); expectMsgClass(deadline.timeLeft(), RegisterResourceManager.class); resourceManagerGateway .tell(new RegisterResourceManagerSuccessful(leader1, Collections.emptyList())); for (Container container : containerList) { resourceManagerGateway.tell( new NotifyResourceStarted(YarnFlinkResourceManager.extractResourceID(container)), leader1Gateway); } for (int i = 0; i < containerList.size(); i++) { expectMsgClass(deadline.timeLeft(), Acknowledge.class); } Future<Object> numberOfRegisteredResourcesFuture = resourceManagerGateway .ask(RequestNumberOfRegisteredResources.INSTANCE, deadline.timeLeft()); int numberOfRegisteredResources = (Integer) Await.result(numberOfRegisteredResourcesFuture, deadline.timeLeft()); assertEquals(numInitialTaskManagers, numberOfRegisteredResources); } finally { if (resourceManager != null) { resourceManager.tell(PoisonPill.getInstance(), ActorRef.noSender()); } } } }; }
From source file:org.apache.flink.yarn.YarnResourceManager.java
License:Apache License
public YarnResourceManager(Configuration flinkConfig, Map<String, String> env, RpcService rpcService, ResourceManagerConfiguration resourceManagerConfiguration, HighAvailabilityServices highAvailabilityServices, SlotManagerFactory slotManagerFactory, MetricRegistry metricRegistry, JobLeaderIdService jobLeaderIdService, FatalErrorHandler fatalErrorHandler) { super(rpcService, resourceManagerConfiguration, highAvailabilityServices, slotManagerFactory, metricRegistry, jobLeaderIdService, fatalErrorHandler); this.flinkConfig = flinkConfig; this.yarnConfig = new YarnConfiguration(); this.ENV = env; final int yarnHeartbeatIntervalMS = flinkConfig.getInteger(ConfigConstants.YARN_HEARTBEAT_DELAY_SECONDS, DEFAULT_YARN_HEARTBEAT_INTERVAL_MS / 1000) * 1000; final long yarnExpiryIntervalMS = yarnConfig.getLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS); if (yarnHeartbeatIntervalMS >= yarnExpiryIntervalMS) { log.warn(/*from w w w. ja v a2s .com*/ "The heartbeat interval of the Flink Application master ({}) is greater " + "than YARN's expiry interval ({}). The application is likely to be killed by YARN.", yarnHeartbeatIntervalMS, yarnExpiryIntervalMS); } yarnHeartbeatIntervalMillis = yarnHeartbeatIntervalMS; numPendingContainerRequests = 0; }
From source file:org.apache.fluo.cluster.runner.YarnAppRunner.java
License:Apache License
private synchronized TwillRunnerService getTwillRunner(FluoConfiguration config) { if (!twillRunners.containsKey(config.getApplicationName())) { YarnConfiguration yarnConfig = new YarnConfiguration(); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/core-site.xml")); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/yarn-site.xml")); TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, config.getAppZookeepers() + ZookeeperPath.TWILL); twillRunner.start();/* w w w . j a va 2 s .c o m*/ twillRunners.put(config.getApplicationName(), twillRunner); // sleep to give twill time to retrieve state from zookeeper try { Thread.sleep(1000); } catch (InterruptedException e) { throw new IllegalStateException(e); } } return twillRunners.get(config.getApplicationName()); }
From source file:org.apache.gobblin.azkaban.AzkabanGobblinYarnAppLauncher.java
License:Apache License
public AzkabanGobblinYarnAppLauncher(String jobId, Properties props) throws IOException { super(jobId, LOGGER); Config gobblinConfig = ConfigUtils.propertiesToConfig(props); outputConfigToFile(gobblinConfig);//from ww w .j a v a2 s . co m this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(gobblinConfig, new YarnConfiguration()); }
From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncherTest.java
License:Apache License
@BeforeClass public void setUp() throws Exception { // Set java home in environment since it isn't set on some systems String javaHome = System.getProperty("java.home"); setEnv("JAVA_HOME", javaHome); final YarnConfiguration clusterConf = new YarnConfiguration(); clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000"); MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1)); miniYARNCluster.init(clusterConf);//w ww .j a va2s .c o m miniYARNCluster.start(); // YARN client should not be started before the Resource Manager is up AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() { @Override public boolean apply(Void input) { return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0"); } }, "Waiting for RM"); this.yarnClient = this.closer.register(YarnClient.createYarnClient()); this.yarnClient.init(clusterConf); this.yarnClient.start(); // Use a random ZK port TestingServer testingZKServer = this.closer.register(new TestingServer(-1)); LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString()); // the zk port is dynamically configured try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH)) { File dir = new File("target/dummydir"); // dummy directory specified in configuration dir.mkdir(); pw.println("gobblin.cluster.zk.connection.string=\"" + testingZKServer.getConnectString() + "\""); pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\""); } // YARN config is dynamic and needs to be passed to other processes try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) { clusterConf.writeXml(os); } this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer); URL url = GobblinYarnAppLauncherTest.class.getClassLoader() .getResource(GobblinYarnAppLauncherTest.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); this.config = ConfigFactory.parseURL(url).withValue("gobblin.cluster.zk.connection.string", ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())).resolve(); String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); this.helixManager = HelixManagerFactory.getZKHelixManager( this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString); this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf); }
From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncherTest.java
License:Apache License
/** * Test that the dynamic config is added to the config specified when the {@link GobblinApplicationMaster} * is instantiated./*from www. j a va 2s.c o m*/ */ @Test public void testDynamicConfig() throws Exception { Config config = this.config.withFallback(ConfigFactory.parseMap(ImmutableMap.of( ConfigurationKeys.DYNAMIC_CONFIG_GENERATOR_CLASS_KEY, TestDynamicConfigGenerator.class.getName()))); ContainerId containerId = ContainerId .newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0), 0); TestApplicationMaster appMaster = new TestApplicationMaster("testApp", containerId, config, new YarnConfiguration()); Assert.assertEquals(appMaster.getConfig().getString("dynamicKey1"), "dynamicValue1"); Assert.assertEquals(appMaster.getConfig().getString(ConfigurationKeys.DYNAMIC_CONFIG_GENERATOR_CLASS_KEY), TestDynamicConfigGenerator.class.getName()); ServiceBasedAppLauncher appLauncher = appMaster.getAppLauncher(); Field servicesField = ServiceBasedAppLauncher.class.getDeclaredField("services"); servicesField.setAccessible(true); List<Service> services = (List<Service>) servicesField.get(appLauncher); Optional<Service> yarnServiceOptional = services.stream().filter(e -> e instanceof YarnService).findFirst(); Assert.assertTrue(yarnServiceOptional.isPresent()); YarnService yarnService = (YarnService) yarnServiceOptional.get(); Field configField = YarnService.class.getDeclaredField("config"); configField.setAccessible(true); Config yarnServiceConfig = (Config) configField.get(yarnService); Assert.assertEquals(yarnServiceConfig.getString("dynamicKey1"), "dynamicValue1"); Assert.assertEquals(yarnServiceConfig.getString(ConfigurationKeys.DYNAMIC_CONFIG_GENERATOR_CLASS_KEY), TestDynamicConfigGenerator.class.getName()); }