List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration()
From source file:com.continuuity.weave.yarn.ApplicationMasterService.java
License:Open Source License
public ApplicationMasterService(RunId runId, String zkConnectStr, File weaveSpecFile) throws IOException { this.runId = runId; this.zkConnectStr = zkConnectStr; this.weaveSpecFile = weaveSpecFile; this.weaveSpec = WeaveSpecificationAdapter.create().fromJson(weaveSpecFile); this.runnableArgs = decodeRunnableArgs(); this.yarnConf = new YarnConfiguration(); this.zkClientService = ZKClientServices.delegate(ZKClients.reWatchOnExpire(ZKClients.retryOnFailure( ZKClientService.Builder.of(zkConnectStr).build(), RetryStrategies.fixDelay(1, TimeUnit.SECONDS)))); this.serviceDelegate = new ZKServiceDecorator(zkClientService, runId, createLiveNodeDataSupplier(), new ServiceDelegate()); // Get the container ID and convert it to ApplicationAttemptId masterContainerId = System.getenv().get(ApplicationConstants.AM_CONTAINER_ID_ENV); Preconditions.checkArgument(masterContainerId != null, "Missing %s from environment", ApplicationConstants.AM_CONTAINER_ID_ENV); amrmClient = new AMRMClientImpl(ConverterUtils.toContainerId(masterContainerId).getApplicationAttemptId()); runningContainers = new RunningContainers(); }
From source file:com.datatorrent.stram.cli.ApexCli.java
License:Apache License
public void preImpersonationInit(String[] args) throws IOException { Signal.handle(new Signal("INT"), new SignalHandler() { @Override//from w w w . j ava2 s . c o m public void handle(Signal sig) { System.out.println("^C"); if (commandThread != null) { commandThread.interrupt(); mainThread.interrupt(); } else { System.out.print(prompt); System.out.flush(); } } }); consolePresent = (System.console() != null); Options options = new Options(); options.addOption("e", true, "Commands are read from the argument"); options.addOption("v", false, "Verbose mode level 1"); options.addOption("vv", false, "Verbose mode level 2"); options.addOption("vvv", false, "Verbose mode level 3"); options.addOption("vvvv", false, "Verbose mode level 4"); options.addOption("r", false, "JSON Raw mode"); options.addOption("p", true, "JSONP padding function"); options.addOption("h", false, "Print this help"); options.addOption("f", true, "Use the specified prompt at all time"); options.addOption("kp", true, "Use the specified kerberos principal"); options.addOption("kt", true, "Use the specified kerberos keytab"); CommandLineParser parser = new BasicParser(); try { CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("v")) { verboseLevel = 1; } if (cmd.hasOption("vv")) { verboseLevel = 2; } if (cmd.hasOption("vvv")) { verboseLevel = 3; } if (cmd.hasOption("vvvv")) { verboseLevel = 4; } if (cmd.hasOption("r")) { raw = true; } if (cmd.hasOption("e")) { commandsToExecute = cmd.getOptionValues("e"); consolePresent = false; } if (cmd.hasOption("p")) { jsonp = cmd.getOptionValue("p"); } if (cmd.hasOption("f")) { forcePrompt = cmd.getOptionValue("f"); } if (cmd.hasOption("h")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(ApexCli.class.getSimpleName(), options); System.exit(0); } if (cmd.hasOption("kp")) { kerberosPrincipal = cmd.getOptionValue("kp"); } if (cmd.hasOption("kt")) { kerberosKeyTab = cmd.getOptionValue("kt"); } } catch (ParseException ex) { System.err.println("Invalid argument: " + ex); System.exit(1); } if (kerberosPrincipal == null && kerberosKeyTab != null) { System.err.println( "Kerberos key tab is specified but not the kerberos principal. Please specify it using the -kp option."); System.exit(1); } if (kerberosPrincipal != null && kerberosKeyTab == null) { System.err.println( "Kerberos principal is specified but not the kerberos key tab. Please specify it using the -kt option."); System.exit(1); } Level logLevel; switch (verboseLevel) { case 0: logLevel = Level.OFF; break; case 1: logLevel = Level.ERROR; break; case 2: logLevel = Level.WARN; break; case 3: logLevel = Level.INFO; break; default: logLevel = Level.DEBUG; break; } for (org.apache.log4j.Logger logger : new org.apache.log4j.Logger[] { org.apache.log4j.Logger.getRootLogger(), org.apache.log4j.Logger.getLogger(ApexCli.class) }) { /* * Override logLevel specified by user, the same logLevel would be inherited by all * appenders related to logger. */ logger.setLevel(logLevel); @SuppressWarnings("unchecked") Enumeration<Appender> allAppenders = logger.getAllAppenders(); while (allAppenders.hasMoreElements()) { Appender appender = allAppenders.nextElement(); if (appender instanceof ConsoleAppender) { ((ConsoleAppender) appender).setThreshold(logLevel); } } } if (commandsToExecute != null) { for (String command : commandsToExecute) { LOG.debug("Command to be executed: {}", command); } } if (kerberosPrincipal != null && kerberosKeyTab != null) { StramUserLogin.authenticate(kerberosPrincipal, kerberosKeyTab); } else { Configuration config = new YarnConfiguration(); StramClientUtils.addDTLocalResources(config); StramUserLogin.attemptAuthentication(config); } }
From source file:com.datatorrent.stram.cli.ApexCli.java
License:Apache License
public void init() throws IOException { conf = StramClientUtils.addDTSiteResources(new YarnConfiguration()); SecurityUtils.init(conf);//from ww w .java 2s .c om fs = StramClientUtils.newFileSystemInstance(conf); stramAgent = new StramAgent(fs, conf); yarnClient.init(conf); yarnClient.start(); LOG.debug("Yarn Client initialized and started"); String socks = conf.get(CommonConfigurationKeysPublic.HADOOP_SOCKS_SERVER_KEY); if (socks != null) { int colon = socks.indexOf(':'); if (colon > 0) { LOG.info("Using socks proxy at {}", socks); System.setProperty("socksProxyHost", socks.substring(0, colon)); System.setProperty("socksProxyPort", socks.substring(colon + 1)); } } }
From source file:com.datatorrent.stram.cli.DTCli.java
License:Apache License
public void preImpersonationInit(String[] args) throws IOException { Signal.handle(new Signal("INT"), new SignalHandler() { @Override/*from www . j av a 2s . c om*/ public void handle(Signal sig) { System.out.println("^C"); if (commandThread != null) { commandThread.interrupt(); mainThread.interrupt(); } else { System.out.print(prompt); System.out.flush(); } } }); consolePresent = (System.console() != null); Options options = new Options(); options.addOption("e", true, "Commands are read from the argument"); options.addOption("v", false, "Verbose mode level 1"); options.addOption("vv", false, "Verbose mode level 2"); options.addOption("vvv", false, "Verbose mode level 3"); options.addOption("vvvv", false, "Verbose mode level 4"); options.addOption("r", false, "JSON Raw mode"); options.addOption("p", true, "JSONP padding function"); options.addOption("h", false, "Print this help"); options.addOption("f", true, "Use the specified prompt at all time"); options.addOption("kp", true, "Use the specified kerberos principal"); options.addOption("kt", true, "Use the specified kerberos keytab"); CommandLineParser parser = new BasicParser(); try { CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("v")) { verboseLevel = 1; } if (cmd.hasOption("vv")) { verboseLevel = 2; } if (cmd.hasOption("vvv")) { verboseLevel = 3; } if (cmd.hasOption("vvvv")) { verboseLevel = 4; } if (cmd.hasOption("r")) { raw = true; } if (cmd.hasOption("e")) { commandsToExecute = cmd.getOptionValues("e"); consolePresent = false; } if (cmd.hasOption("p")) { jsonp = cmd.getOptionValue("p"); } if (cmd.hasOption("f")) { forcePrompt = cmd.getOptionValue("f"); } if (cmd.hasOption("h")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(DTCli.class.getSimpleName(), options); System.exit(0); } if (cmd.hasOption("kp")) { kerberosPrincipal = cmd.getOptionValue("kp"); } if (cmd.hasOption("kt")) { kerberosKeyTab = cmd.getOptionValue("kt"); } } catch (ParseException ex) { System.err.println("Invalid argument: " + ex); System.exit(1); } if (kerberosPrincipal == null && kerberosKeyTab != null) { System.err.println( "Kerberos key tab is specified but not the kerberos principal. Please specify it using the -kp option."); System.exit(1); } if (kerberosPrincipal != null && kerberosKeyTab == null) { System.err.println( "Kerberos principal is specified but not the kerberos key tab. Please specify it using the -kt option."); System.exit(1); } Level logLevel; switch (verboseLevel) { case 0: logLevel = Level.OFF; break; case 1: logLevel = Level.ERROR; break; case 2: logLevel = Level.WARN; break; case 3: logLevel = Level.INFO; break; default: logLevel = Level.DEBUG; break; } for (org.apache.log4j.Logger logger : new org.apache.log4j.Logger[] { org.apache.log4j.Logger.getRootLogger(), org.apache.log4j.Logger.getLogger(DTCli.class) }) { @SuppressWarnings("unchecked") Enumeration<Appender> allAppenders = logger.getAllAppenders(); while (allAppenders.hasMoreElements()) { Appender appender = allAppenders.nextElement(); if (appender instanceof ConsoleAppender) { ((ConsoleAppender) appender).setThreshold(logLevel); } } } if (commandsToExecute != null) { for (String command : commandsToExecute) { LOG.debug("Command to be executed: {}", command); } } if (kerberosPrincipal != null && kerberosKeyTab != null) { StramUserLogin.authenticate(kerberosPrincipal, kerberosKeyTab); } else { Configuration config = new YarnConfiguration(); StramClientUtils.addDTLocalResources(config); StramUserLogin.attemptAuthentication(config); } }
From source file:com.datatorrent.stram.cli.DTCli.java
License:Apache License
public void init() throws IOException { conf = StramClientUtils.addDTSiteResources(new YarnConfiguration()); fs = StramClientUtils.newFileSystemInstance(conf); stramAgent = new StramAgent(fs, conf); yarnClient.init(conf);/*from w w w . j av a 2 s. c o m*/ yarnClient.start(); LOG.debug("Yarn Client initialized and started"); String socks = conf.get(CommonConfigurationKeysPublic.HADOOP_SOCKS_SERVER_KEY); if (socks != null) { int colon = socks.indexOf(':'); if (colon > 0) { LOG.info("Using socks proxy at {}", socks); System.setProperty("socksProxyHost", socks.substring(0, colon)); System.setProperty("socksProxyPort", socks.substring(colon + 1)); } } }
From source file:com.datatorrent.stram.engine.StreamingContainer.java
License:Apache License
public void heartbeatLoop() throws Exception { umbilical.log(containerId, "[" + containerId + "] Entering heartbeat loop.."); logger.debug("Entering heartbeat loop (interval is {} ms)", this.heartbeatIntervalMillis); final YarnConfiguration conf = new YarnConfiguration(); long tokenLifeTime = (long) (containerContext.getValue(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR) * containerContext.getValue(LogicalPlan.HDFS_TOKEN_LIFE_TIME)); long expiryTime = System.currentTimeMillis(); final Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); logger.debug("token: {}", token); }// w ww.j a va 2s. co m String hdfsKeyTabFile = containerContext.getValue(LogicalPlan.KEY_TAB_FILE); while (!exitHeartbeatLoop) { if (UserGroupInformation.isSecurityEnabled() && System.currentTimeMillis() >= expiryTime && hdfsKeyTabFile != null) { expiryTime = StramUserLogin.refreshTokens(tokenLifeTime, "." + File.separator + "tmp", containerId, conf, hdfsKeyTabFile, credentials, null, false); } synchronized (this.heartbeatTrigger) { try { this.heartbeatTrigger.wait(heartbeatIntervalMillis); } catch (InterruptedException e1) { logger.warn("Interrupted in heartbeat loop, exiting.."); break; } } long currentTime = System.currentTimeMillis(); ContainerHeartbeat msg = new ContainerHeartbeat(); msg.jvmName = jvmName; if (this.bufferServerAddress != null) { msg.bufferServerHost = this.bufferServerAddress.getHostName(); msg.bufferServerPort = this.bufferServerAddress.getPort(); if (bufferServer != null && !eventloop.isActive()) { logger.warn("Requesting restart due to terminated event loop"); msg.restartRequested = true; } } msg.memoryMBFree = ((int) (Runtime.getRuntime().freeMemory() / (1024 * 1024))); garbageCollectorMXBeans = ManagementFactory.getGarbageCollectorMXBeans(); for (GarbageCollectorMXBean bean : garbageCollectorMXBeans) { msg.gcCollectionTime += bean.getCollectionTime(); msg.gcCollectionCount += bean.getCollectionCount(); } ContainerHeartbeatResponse rsp; do { ContainerStats stats = new ContainerStats(containerId); // gather heartbeat info for all operators for (Map.Entry<Integer, Node<?>> e : nodes.entrySet()) { OperatorHeartbeat hb = new OperatorHeartbeat(); hb.setNodeId(e.getKey()); hb.setGeneratedTms(currentTime); hb.setIntervalMs(heartbeatIntervalMillis); if (e.getValue().commandResponse.size() > 0) { BlockingQueue<StatsListener.OperatorResponse> commandResponse = e .getValue().commandResponse; ArrayList<StatsListener.OperatorResponse> response = new ArrayList<StatsListener.OperatorResponse>(); for (int i = 0; i < commandResponse.size(); i++) { response.add(commandResponse.poll()); } hb.requestResponse = response; } OperatorContext context = e.getValue().context; context.drainStats(hb.getOperatorStatsContainer()); if (context.getThread() == null || context.getThread().getState() != Thread.State.TERMINATED) { hb.setState(DeployState.ACTIVE); } else if (failedNodes.contains(hb.nodeId)) { hb.setState(DeployState.FAILED); } else { logger.debug("Reporting SHUTDOWN state because thread is {} and failedNodes is {}", context.getThread(), failedNodes); hb.setState(DeployState.SHUTDOWN); } stats.addNodeStats(hb); } /** * Container stats published for whoever is interested in listening. * Currently interested candidates are TupleRecorderCollection and BufferServerStatsSubscriber */ eventBus.publish(new ContainerStatsEvent(stats)); msg.setContainerStats(stats); // heartbeat call and follow-up processing //logger.debug("Sending heartbeat for {} operators.", msg.getContainerStats().size()); msg.sentTms = System.currentTimeMillis(); rsp = umbilical.processHeartbeat(msg); processHeartbeatResponse(rsp); if (rsp.hasPendingRequests) { logger.info("Waiting for pending request."); synchronized (this.heartbeatTrigger) { try { this.heartbeatTrigger.wait(500); } catch (InterruptedException ie) { logger.warn("Interrupted in heartbeat loop", ie); break; } } } } while (rsp.hasPendingRequests); } logger.debug("Exiting hearbeat loop"); umbilical.log(containerId, "[" + containerId + "] Exiting heartbeat loop.."); }
From source file:com.datatorrent.stram.security.StramWSFilterInitializer.java
License:Apache License
@Override public void initFilter(FilterContainer container, Configuration conf) { Map<String, String> params = new HashMap<String, String>(); String proxy = WebAppUtils.getProxyHostAndPort(conf); String[] parts = proxy.split(":"); params.put(StramWSFilter.PROXY_HOST, parts[0]); params.put(StramWSFilter.PROXY_URI_BASE, ConfigUtils.getSchemePrefix(new YarnConfiguration()) + proxy + System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV)); container.addFilter(FILTER_NAME, FILTER_CLASS, params); }
From source file:com.datatorrent.stram.StreamingAppMaster.java
License:Apache License
/** * @param args/*from w w w . j a v a2 s .c om*/ * Command line args * @throws Throwable */ public static void main(final String[] args) throws Throwable { StdOutErrLog.tieSystemOutAndErrToLog(); LOG.info("Master starting with classpath: {}", System.getProperty("java.class.path")); LOG.info("version: {}", VersionInfo.getBuildVersion()); StringWriter sw = new StringWriter(); for (Map.Entry<String, String> e : System.getenv().entrySet()) { sw.append("\n").append(e.getKey()).append("=").append(e.getValue()); } LOG.info("appmaster env:" + sw.toString()); Options opts = new Options(); opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); // option "help" overrides and cancels any run if (cliParser.hasOption("help")) { new HelpFormatter().printHelp("ApplicationMaster", opts); return; } Map<String, String> envs = System.getenv(); ApplicationAttemptId appAttemptID = Records.newRecord(ApplicationAttemptId.class); if (!envs.containsKey(Environment.CONTAINER_ID.name())) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); } else { throw new IllegalArgumentException("Application Attempt Id not set in the environment"); } } else { ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name())); appAttemptID = containerId.getApplicationAttemptId(); } boolean result = false; StreamingAppMasterService appMaster = null; try { appMaster = new StreamingAppMasterService(appAttemptID); LOG.info("Initializing Application Master."); Configuration conf = new YarnConfiguration(); appMaster.init(conf); appMaster.start(); result = appMaster.run(); } catch (Throwable t) { LOG.error("Exiting Application Master", t); System.exit(1); } finally { if (appMaster != null) { appMaster.stop(); } } if (result) { LOG.info("Application Master completed."); System.exit(0); } else { LOG.info("Application Master failed."); System.exit(2); } }
From source file:com.datatorrent.stram.StreamingContainerAgent.java
License:Apache License
public ContainerInfo getContainerInfo() { ContainerInfo ci = new ContainerInfo(); ci.id = container.getExternalId();/*from w w w . jav a 2 s .c o m*/ ci.host = container.host; ci.state = container.getState().name(); ci.jvmName = this.jvmName; ci.numOperators = container.getOperators().size(); ci.memoryMBAllocated = container.getAllocatedMemoryMB(); ci.lastHeartbeat = lastHeartbeatMillis; ci.memoryMBFree = this.memoryMBFree; ci.gcCollectionCount = this.gcCollectionCount; ci.gcCollectionTime = this.gcCollectionTime; ci.startedTime = container.getStartedTime(); ci.finishedTime = container.getFinishedTime(); if (this.container.nodeHttpAddress != null) { YarnConfiguration conf = new YarnConfiguration(); ci.containerLogsUrl = ConfigUtils.getSchemePrefix(conf) + this.container.nodeHttpAddress + "/node/containerlogs/" + ci.id + "/" + System.getenv(ApplicationConstants.Environment.USER.toString()); ci.rawContainerLogsUrl = ConfigUtils.getRawContainerLogsUrl(conf, container.nodeHttpAddress, container.getPlan().getLogicalPlan().getAttributes().get(LogicalPlan.APPLICATION_ID), ci.id); } return ci; }
From source file:com.datatorrent.stram.StreamingContainerManager.java
License:Apache License
public final ContainerInfo getAppMasterContainerInfo() { ContainerInfo ci = new ContainerInfo(); ci.id = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()); String nmHost = System.getenv(ApplicationConstants.Environment.NM_HOST.toString()); String nmPort = System.getenv(ApplicationConstants.Environment.NM_PORT.toString()); String nmHttpPort = System.getenv(ApplicationConstants.Environment.NM_HTTP_PORT.toString()); ci.state = "ACTIVE"; ci.jvmName = ManagementFactory.getRuntimeMXBean().getName(); ci.numOperators = 0;/*from w w w .j a va2s. co m*/ YarnConfiguration conf = new YarnConfiguration(); if (nmHost != null) { if (nmPort != null) { ci.host = nmHost + ":" + nmPort; } if (nmHttpPort != null) { String nodeHttpAddress = nmHost + ":" + nmHttpPort; if (allocatedMemoryBytes == 0) { String url = ConfigUtils.getSchemePrefix(conf) + nodeHttpAddress + "/ws/v1/node/containers/" + ci.id; WebServicesClient webServicesClient = new WebServicesClient(); try { String content = webServicesClient.process(url, String.class, new WebServicesClient.GetWebServicesHandler<String>()); JSONObject json = new JSONObject(content); int totalMemoryNeededMB = json.getJSONObject("container").getInt("totalMemoryNeededMB"); if (totalMemoryNeededMB > 0) { allocatedMemoryBytes = totalMemoryNeededMB * 1024 * 1024; } else { LOG.warn( "Could not determine the memory allocated for the streaming application master. Node manager is reporting {} MB from {}", totalMemoryNeededMB, url); } } catch (Exception ex) { LOG.warn("Could not determine the memory allocated for the streaming application master", ex); } } ci.containerLogsUrl = ConfigUtils.getSchemePrefix(conf) + nodeHttpAddress + "/node/containerlogs/" + ci.id + "/" + System.getenv(ApplicationConstants.Environment.USER.toString()); ci.rawContainerLogsUrl = ConfigUtils.getRawContainerLogsUrl(conf, nodeHttpAddress, plan.getLogicalPlan().getAttributes().get(LogicalPlan.APPLICATION_ID), ci.id); } } ci.memoryMBAllocated = (int) (allocatedMemoryBytes / (1024 * 1024)); ci.memoryMBFree = ((int) (Runtime.getRuntime().freeMemory() / (1024 * 1024))); ci.lastHeartbeat = -1; ci.startedTime = startTime; ci.finishedTime = -1; return ci; }