List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration()
From source file:ml.shifu.guagua.yarn.GuaguaYarnClient.java
License:Apache License
/** * Default constructor. Use {@link YarnConfiguration} as default conf setting. */ public GuaguaYarnClient() { this(new YarnConfiguration()); }
From source file:ml.shifu.guagua.yarn.GuaguaYarnTask.java
License:Apache License
public static void main(String[] args) { LOG.info("args:{}", Arrays.toString(args)); if (args.length != 7) { throw new IllegalStateException(String.format( "GuaguaYarnTask could not construct a TaskAttemptID for the Guagua job from args: %s", Arrays.toString(args))); }/*from ww w .j a v a 2s .co m*/ String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); try { Configuration conf = new YarnConfiguration(); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.info(token.toString()); } UserGroupInformation appTaskUGI = UserGroupInformation.createRemoteUser(jobUserName); appTaskUGI.addCredentials(credentials); @SuppressWarnings("rawtypes") final GuaguaYarnTask<?, ?> guaguaYarnTask = new GuaguaYarnTask(appAttemptId, containerId, Integer.parseInt(args[args.length - 3]), args[args.length - 2], args[args.length - 1], conf); appTaskUGI.doAs(new PrivilegedAction<Void>() { @Override public Void run() { guaguaYarnTask.run(); return null; } }); } catch (Throwable t) { LOG.error("GuaguaYarnTask threw a top-level exception, failing task", t); System.exit(2); } System.exit(0); }
From source file:nl.tudelft.graphalytics.giraph.reporting.logging.GraphalyticLogger.java
License:Apache License
public static void collectYarnLog(String applicationId, String yarnlogPath) { try {/*from ww w .ja v a 2s .c om*/ PrintStream console = System.out; File file = new File(yarnlogPath); FileOutputStream fos = new FileOutputStream(file); PrintStream ps = new PrintStream(fos); System.setOut(ps); waitInterval(20); LogsCLI logDumper = new LogsCLI(); logDumper.setConf(new YarnConfiguration()); String[] args = { "-applicationId", applicationId }; logDumper.run(args); System.setOut(console); } catch (Exception e) { e.printStackTrace(); } }
From source file:org.apache.accumulo.testing.yarn.YarnAccumuloTestRunner.java
License:Apache License
public static void main(String[] args) throws Exception { TestRunnerOpts opts = new TestRunnerOpts(); new JCommander(opts, args); verifyPath(opts.jarPath);//ww w. j a v a 2 s .c o m verifyPath(opts.testProps); verifyPath(opts.logProps); String[] mainArgs = opts.mainArgs.stream().toArray(String[]::new); BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(opts.jarPath, "/lib", opts.mainClass, mainArgs); Properties props = new Properties(); FileInputStream fis = new FileInputStream(opts.testProps); props.load(fis); fis.close(); String zookeepers = props.getProperty(TestProps.ZOOKEEPERS); final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(), zookeepers); twillRunner.start(); TwillController controller = twillRunner.prepare(new YarnTestApp(opts, props)) .addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName()) .withArguments("BundledJarRunnable", arguments.toArray()).start(); int numRunning = getNumRunning(controller); while (numRunning != opts.numContainers) { LOG.info("{} of {} containers have started in YARN.", numRunning, opts.numContainers); Thread.sleep(5000); numRunning = getNumRunning(controller); } LOG.info("{} of {} containers have started in YARN", numRunning, opts.numContainers); LOG.info("{} application was successfully started in YARN", opts.testName); }
From source file:org.apache.apex.engine.plugin.AbstractApexPluginDispatcher.java
License:Apache License
private Configuration readLaunchConfiguration() throws IOException { Path appPath = new Path(appContext.getApplicationPath()); Path configFilePath = new Path(appPath, LogicalPlan.LAUNCH_CONFIG_FILE_NAME); try {// www . jav a2 s .co m LOG.debug("Reading launch configuration file "); URI uri = appPath.toUri(); Configuration config = new YarnConfiguration(); fileContext = uri.getScheme() == null ? FileContext.getFileContext(config) : FileContext.getFileContext(uri, config); FSDataInputStream is = fileContext.open(configFilePath); config.addResource(is); LOG.debug("Read launch configuration"); return config; } catch (FileNotFoundException ex) { LOG.warn("Configuration file not found {}", configFilePath); return new Configuration(); } }
From source file:org.apache.drill.yarn.appMaster.AMYarnFacadeImpl.java
License:Apache License
@Override public void start(CallbackHandler resourceCallback, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler nodeCallback) { conf = new YarnConfiguration(); resourceMgr = AMRMClientAsync.createAMRMClientAsync(pollPeriodMs, resourceCallback); resourceMgr.init(conf);/*from w w w.jav a 2s. c o m*/ resourceMgr.start(); // Create the asynchronous node manager client nodeMgr = NMClientAsync.createNMClientAsync(nodeCallback); nodeMgr.init(conf); nodeMgr.start(); client = YarnClient.createYarnClient(); client.init(conf); client.start(); String appIdStr = System.getenv(DrillOnYarnConfig.APP_ID_ENV_VAR); if (appIdStr != null) { appId = ConverterUtils.toApplicationId(appIdStr); try { appReport = client.getApplicationReport(appId); } catch (YarnException | IOException e) { LOG.error("Failed to get YARN applicaiton report for App ID: " + appIdStr, e); } } }
From source file:org.apache.drill.yarn.client.PrintConfigCommand.java
License:Apache License
@Override public void run() { // Dump configuration if requested for diagnostic use. System.out.println("----------------------------------------------"); System.out.println("Effective Drill-on-YARN Configuration"); DrillOnYarnConfig.instance().dump(); System.out.println("----------------------------------------------"); // Dump YARN configuration. System.out.println("YARN, DFS and Hadoop Configuration"); YarnConfiguration conf = new YarnConfiguration(); try {/*w w w .ja va 2 s . c o m*/ YarnConfiguration.dumpConfiguration(conf, new OutputStreamWriter(System.out)); System.out.println(); } catch (IOException e) { // Ignore; } System.out.println("----------------------------------------------"); }
From source file:org.apache.drill.yarn.core.DfsFacade.java
License:Apache License
/** * Lazy loading of YARN configuration since it takes a long time to load. * (YARN provides no caching, sadly.)//from w w w. j a va 2 s . com */ private void loadYarnConfig() { if (yarnConf == null) { yarnConf = new YarnConfiguration(); // On some distributions, lack of proper configuration causes // DFS to default to the local file system. So, a local file // system generally means that the config is wrong, or running // the wrong build of Drill for the user's environment. URI fsUri = FileSystem.getDefaultUri(yarnConf); if (fsUri.toString().startsWith("file:/")) { System.err.println("Warning: Default DFS URI is for a local file system: " + fsUri); } } }
From source file:org.apache.drill.yarn.core.YarnRMClient.java
License:Apache License
public YarnRMClient() { this(new YarnConfiguration()); }
From source file:org.apache.flink.yarn.AbstractYarnClusterTest.java
License:Apache License
/** * Tests that the cluster retrieval of a finished YARN application fails. *///from w w w . j a v a 2 s.c o m @Test(expected = ClusterRetrieveException.class) public void testClusterClientRetrievalOfFinishedYarnApplication() throws Exception { final ApplicationId applicationId = ApplicationId.newInstance(System.currentTimeMillis(), 42); final ApplicationReport applicationReport = createApplicationReport(applicationId, YarnApplicationState.FINISHED, FinalApplicationStatus.SUCCEEDED); final YarnClient yarnClient = new TestingYarnClient( Collections.singletonMap(applicationId, applicationReport)); final YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnClient.init(yarnConfiguration); yarnClient.start(); final TestingAbstractYarnClusterDescriptor clusterDescriptor = new TestingAbstractYarnClusterDescriptor( new Configuration(), yarnConfiguration, temporaryFolder.newFolder().getAbsolutePath(), yarnClient, false); try { clusterDescriptor.retrieve(applicationId); } finally { clusterDescriptor.close(); } }