List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration()
From source file:com.cfets.door.yarn.jboss.JBossApplicationMaster.java
License:Apache License
public JBossApplicationMaster() throws Exception { conf = new YarnConfiguration(); }
From source file:com.cfets.door.yarn.jboss.JBossClient.java
License:Apache License
/** */ public JBossClient() throws Exception { this(new YarnConfiguration()); }
From source file:com.cloudera.llama.am.LlamaAMServer.java
License:Apache License
@Override protected void startService() { startHttpServer();//ww w . ja v a 2 s . c o m try { Security.loginToHadoop(getServerConf()); Class<? extends NodeMapper> klass = getServerConf().getNodeMappingClass(); nodeMapper = ReflectionUtils.newInstance(klass, getConf()); clientNotificationService = new ClientNotificationService(getServerConf(), nodeMapper, getMetricRegistry()); clientNotificationService.addListener(this); clientNotificationService.start(); clientNotificationService.addListener(restData); // For mapping reservations to queues and checking queue ACLs YarnConfiguration yarnConf = new YarnConfiguration(); // Check the token renew interval and set the default here so that we can // renew the RMConnectors properly. long renewInterval = yarnConf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); LlamaAM.RM_CONNECTOR_RECYCLE_INTERVAL_DEFAULT = renewInterval * 3 / 4; allocConf = new AtomicReference<AllocationConfiguration>(); allocsLoader = new AllocationFileLoaderService(); allocsLoader.init(yarnConf); allocsLoader.setReloadListener(new AllocationFileLoaderService.Listener() { @Override public void onReload(AllocationConfiguration allocs) { allocConf.set(allocs); } }); try { allocsLoader.reloadAllocations(); allocsLoader.start(); } catch (Exception ex) { LOG.warn("Failed to load queue allocations"); } if (allocConf.get() == null) { allocConf.set(new AllocationConfiguration(yarnConf)); } getConf().set(YarnRMConnector.ADVERTISED_HOSTNAME_KEY, ThriftEndPoint.getServerAddress(getServerConf())); getConf().setInt(YarnRMConnector.ADVERTISED_PORT_KEY, ThriftEndPoint.getServerPort(getServerConf())); getConf().set(YarnRMConnector.ADVERTISED_TRACKING_URL_KEY, getHttpLlamaUI()); llamaAm = LlamaAM.create(getConf()); asyncListener = new AsyncLlamaAMListener(restData); asyncListener.setMetricRegistry(getMetricRegistry()); asyncListener.start(); llamaAm.addListener(asyncListener); llamaAm.setMetricRegistry(getMetricRegistry()); llamaAm.start(); } catch (Exception ex) { throw new RuntimeException(ex); } }
From source file:com.cloudera.llama.am.MiniLlama.java
License:Apache License
public void start() throws Exception { if (useExternalHadoop) { hadoopConf = new YarnConfiguration(); } else {//from w ww.j a v a 2s.c o m hadoopConf = startMiniHadoop(); if (writeHdfsConfig != null) { FileOutputStream fos = new FileOutputStream(new File(writeHdfsConfig)); hadoopConf.writeXml(fos); fos.close(); } } server.getConf().setClass(S_CONF.getPropertyName(ServerConfiguration.NODE_NAME_MAPPING_CLASS_KEY), MiniClusterNodeMapper.class, NodeMapper.class); Map<String, String> mapping = getDataNodeNodeManagerMapping(hadoopConf); MiniClusterNodeMapper.addMapping(server.getConf(), mapping); for (Map.Entry entry : hadoopConf) { server.getConf().set((String) entry.getKey(), (String) entry.getValue()); } dataNodes = new ArrayList<String>(mapping.keySet()); dataNodes = Collections.unmodifiableList(dataNodes); server.start(); }
From source file:com.cloudera.llama.am.MiniLlama.java
License:Apache License
private Configuration startMiniHadoop() throws Exception { int clusterNodes = getConf().getInt(MINI_CLUSTER_NODES_KEY, 1); if (System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA) == null) { String testBuildData = new File("target").getAbsolutePath(); System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, testBuildData); }/*ww w .java 2 s . co m*/ //to trigger hdfs-site.xml registration as default resource new HdfsConfiguration(); Configuration conf = new YarnConfiguration(); String llamaProxyUser = System.getProperty("user.name"); conf.set("hadoop.security.authentication", "simple"); conf.set("hadoop.proxyuser." + llamaProxyUser + ".hosts", "*"); conf.set("hadoop.proxyuser." + llamaProxyUser + ".groups", "*"); String[] userGroups = new String[] { "g" }; UserGroupInformation.createUserForTesting(llamaProxyUser, userGroups); int hdfsPort = 0; String fsUri = conf.get("fs.defaultFS"); if (fsUri != null && !fsUri.equals("file:///")) { int i = fsUri.lastIndexOf(":"); if (i > -1) { try { hdfsPort = Integer.parseInt(fsUri.substring(i + 1)); } catch (Exception ex) { throw new RuntimeException( "Could not parse port from Hadoop's " + "'fs.defaultFS property: " + fsUri); } } } miniHdfs = new MiniDFSCluster(hdfsPort, conf, clusterNodes, !skipDfsFormat, true, null, null); miniHdfs.waitActive(); conf = miniHdfs.getConfiguration(0); miniYarn = new MiniYARNCluster("minillama", clusterNodes, 1, 1); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); miniYarn.init(conf); miniYarn.start(); conf = miniYarn.getConfig(); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); return conf; }
From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java
License:Apache License
private Configuration createMiniYarnConfig(boolean usePortInName) throws Exception { Configuration conf = new YarnConfiguration(); conf.set("yarn.scheduler.fair.allocation.file", "test-fair-scheduler.xml"); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, FairScheduler.class); //proxy user config String llamaProxyUser = System.getProperty("user.name"); conf.set("hadoop.security.authentication", "simple"); conf.set("hadoop.proxyuser." + llamaProxyUser + ".hosts", "*"); conf.set("hadoop.proxyuser." + llamaProxyUser + ".groups", "*"); String[] userGroups = new String[] { "g" }; UserGroupInformation.createUserForTesting(llamaProxyUser, userGroups); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, usePortInName); return conf;//from w w w . ja va 2s . co m }
From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java
License:Apache License
@Override public void setConf(Configuration conf) { this.conf = conf; includePortInNodeName = getConf().getBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME); appType = getConf().get(LlamaAM.CLUSTER_ID, LlamaAM.CLUSTER_ID_DEFAULT); yarnConf = new YarnConfiguration(); for (Map.Entry entry : getConf()) { yarnConf.set((String) entry.getKey(), (String) entry.getValue()); }/*from w w w. j av a 2 s . c om*/ }
From source file:com.cloudera.llama.nm.TestLlamaNMAuxiliaryService.java
License:Apache License
private Configuration createMiniYarnConfig() throws Exception { Configuration conf = new YarnConfiguration(); conf.set("yarn.nodemanager.aux-services", "llama_nm_plugin"); conf.setClass("yarn.nodemanager.aux-services.llama_nm_plugin.class", MyLlamaNMAuxiliaryService.class, AuxiliaryService.class); injectLlamaNMConfiguration(conf);/* w ww. j av a 2 s. c o m*/ return conf; }
From source file:com.continuuity.weave.internal.appmaster.ApplicationMasterMain.java
License:Apache License
/** * Starts the application master./* w ww. j a va 2s. co m*/ */ public static void main(String[] args) throws Exception { String zkConnect = System.getenv(EnvKeys.WEAVE_ZK_CONNECT); File weaveSpec = new File(Constants.Files.WEAVE_SPEC); RunId runId = RunIds.fromString(System.getenv(EnvKeys.WEAVE_RUN_ID)); ZKClientService zkClientService = ZKClientServices.delegate( ZKClients.reWatchOnExpire(ZKClients.retryOnFailure(ZKClientService.Builder.of(zkConnect).build(), RetryStrategies.fixDelay(1, TimeUnit.SECONDS)))); Configuration conf = new YarnConfiguration(); Service service = new ApplicationMasterService(runId, zkClientService, weaveSpec, conf, new VersionDetectYarnAMClientFactory(conf)); new ApplicationMasterMain(String.format("%s/%s/kafka", zkConnect, runId.getId())).doMain(zkClientService, service); }
From source file:com.continuuity.weave.internal.appmaster.ApplicationMasterService.java
License:Open Source License
public ApplicationMasterService(RunId runId, ZKClient zkClient, File weaveSpecFile) throws IOException { this.runId = runId; this.weaveSpec = WeaveSpecificationAdapter.create().fromJson(weaveSpecFile); this.runnableArgs = decodeRunnableArgs(); this.yarnConf = new YarnConfiguration(); this.zkClient = zkClient; // Get the container ID and convert it to ApplicationAttemptId masterContainerId = System.getenv().get(ApplicationConstants.AM_CONTAINER_ID_ENV); Preconditions.checkArgument(masterContainerId != null, "Missing %s from environment", ApplicationConstants.AM_CONTAINER_ID_ENV); amrmClient = new AMRMClientImpl(ConverterUtils.toContainerId(masterContainerId).getApplicationAttemptId()); runningContainers = new RunningContainers(); serviceDelegate = new ZKServiceDecorator(zkClient, runId, createLiveNodeDataSupplier(), new ServiceDelegate()); instanceCounts = initInstanceCounts(weaveSpec, Maps.<String, Integer>newConcurrentMap()); }