List of usage examples for org.apache.hadoop.yarn.exceptions YarnRuntimeException YarnRuntimeException
public YarnRuntimeException(String message, Throwable cause)
From source file:org.apache.solr.hadoop.hack.MiniMRYarnCluster.java
License:Apache License
@Override public void serviceInit(Configuration conf) throws Exception { conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) { conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(), "apps_staging_dir/").getAbsolutePath()); }//from w ww .j a v a 2 s. c om // By default, VMEM monitoring disabled, PMEM monitoring enabled. if (!conf.getBoolean(MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING, MRConfig.DEFAULT_MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) { conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false); conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false); } conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); try { Path stagingPath = FileContext.getFileContext(conf) .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR))); /* * Re-configure the staging path on Windows if the file system is localFs. * We need to use a absolute path that contains the drive letter. The unit * test could run on a different drive than the AM. We can run into the * issue that job files are localized to the drive where the test runs on, * while the AM starts on a different drive and fails to find the job * metafiles. Using absolute path can avoid this ambiguity. */ if (Path.WINDOWS) { if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) { conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR)).getAbsolutePath()); } } FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf); if (fc.util().exists(stagingPath)) { LOG.info(stagingPath + " exists! deleting..."); fc.delete(stagingPath, true); } LOG.info("mkdir: " + stagingPath); //mkdir the staging directory so that right permissions are set while running as proxy user fc.mkdir(stagingPath, null, true); //mkdir done directory as well String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf); Path doneDirPath = fc.makeQualified(new Path(doneDir)); fc.mkdir(doneDirPath, null, true); } catch (IOException e) { throw new YarnRuntimeException("Could not create staging directory. ", e); } conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of // which shuffle doesn't happen //configure the shuffle service in NM conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID }); conf.setClass(String.format(Locale.ENGLISH, YarnConfiguration.NM_AUX_SERVICE_FMT, ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, Service.class); // Non-standard shuffle port conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class, ContainerExecutor.class); // TestMRJobs is for testing non-uberized operation only; see TestUberAM // for corresponding uberized tests. conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); super.serviceInit(conf); }
From source file:org.apache.solr.hadoop.hack.MiniYARNCluster.java
License:Apache License
/** * @param testName name of the test// w w w . j ava2s . c om * @param noOfNodeManagers the number of node managers in the cluster * @param numLocalDirs the number of nm-local-dirs per nodemanager * @param numLogDirs the number of nm-log-dirs per nodemanager */ public MiniYARNCluster(String testName, int noOfNodeManagers, int numLocalDirs, int numLogDirs, File testWorkDir) { super(testName.replace("$", "")); this.numLocalDirs = numLocalDirs; this.numLogDirs = numLogDirs; String testSubDir = testName.replace("$", ""); File targetWorkDir = new File(testWorkDir, testSubDir); try { FileContext.getLocalFSFileContext().delete(new Path(targetWorkDir.getAbsolutePath()), true); } catch (Exception e) { LOG.warn("COULD NOT CLEANUP", e); throw new YarnRuntimeException("could not cleanup test dir: " + e, e); } if (Shell.WINDOWS) { // The test working directory can exceed the maximum path length supported // by some Windows APIs and cmd.exe (260 characters). To work around this, // create a symlink in temporary storage with a much shorter path, // targeting the full path to the test working directory. Then, use the // symlink as the test working directory. String targetPath = targetWorkDir.getAbsolutePath(); File link = new File(System.getProperty("java.io.tmpdir"), String.valueOf(System.currentTimeMillis())); String linkPath = link.getAbsolutePath(); try { FileContext.getLocalFSFileContext().delete(new Path(linkPath), true); } catch (IOException e) { throw new YarnRuntimeException("could not cleanup symlink: " + linkPath, e); } // Guarantee target exists before creating symlink. targetWorkDir.mkdirs(); ShellCommandExecutor shexec = new ShellCommandExecutor(Shell.getSymlinkCommand(targetPath, linkPath)); try { shexec.execute(); } catch (IOException e) { throw new YarnRuntimeException( String.format(Locale.ENGLISH, "failed to create symlink from %s to %s, shell output: %s", linkPath, targetPath, shexec.getOutput()), e); } this.testWorkDir = link; } else { this.testWorkDir = targetWorkDir; } resourceManagerWrapper = new ResourceManagerWrapper(); addService(resourceManagerWrapper); nodeManagers = new CustomNodeManager[noOfNodeManagers]; for (int index = 0; index < noOfNodeManagers; index++) { addService(new NodeManagerWrapper(index)); nodeManagers[index] = new CustomNodeManager(); } }
From source file:org.apache.tajo.master.container.TajoRecordFactoryPBImpl.java
License:Apache License
@SuppressWarnings("unchecked") @Override/*from w w w. j a v a 2s . com*/ public <T> T newRecordInstance(Class<T> clazz) { Constructor<?> constructor = cache.get(clazz); if (constructor == null) { Class<?> pbClazz = null; try { pbClazz = localConf.getClassByName(getPBImplClassName(clazz)); } catch (ClassNotFoundException e) { throw new YarnRuntimeException("Failed to load class: [" + getPBImplClassName(clazz) + "]", e); } try { constructor = pbClazz.getConstructor(); constructor.setAccessible(true); cache.putIfAbsent(clazz, constructor); } catch (NoSuchMethodException e) { throw new YarnRuntimeException("Could not find 0 argument constructor", e); } } try { Object retObject = constructor.newInstance(); return (T) retObject; } catch (InvocationTargetException e) { throw new YarnRuntimeException(e); } catch (IllegalAccessException e) { throw new YarnRuntimeException(e); } catch (InstantiationException e) { throw new YarnRuntimeException(e); } }
From source file:org.apache.tajo.MiniTajoYarnCluster.java
License:Apache License
@Override public void init(Configuration conf) { conf.setSocketAddr(YarnConfiguration.RM_ADDRESS, new InetSocketAddress("127.0.0.1", 0)); conf.setSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, new InetSocketAddress("127.0.0.1", 0)); conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) { conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(), "apps_staging_dir/").getAbsolutePath()); }//from w w w . j a v a 2 s .co m conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); try { Path stagingPath = FileContext.getFileContext(conf) .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR))); FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf); if (fc.util().exists(stagingPath)) { LOG.info(stagingPath + " exists! deleting..."); fc.delete(stagingPath, true); } LOG.info("mkdir: " + stagingPath); //mkdir the staging directory so that right permissions are set while running as proxy user fc.mkdir(stagingPath, null, true); //mkdir done directory as well String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf); Path doneDirPath = fc.makeQualified(new Path(doneDir)); fc.mkdir(doneDirPath, null, true); } catch (IOException e) { throw new YarnRuntimeException("Could not create staging directory. ", e); } conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of // which shuffle doesn't happen //configure the shuffle service in NM conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, PullServerAuxService.PULLSERVER_SERVICEID); conf.setClass( String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, PullServerAuxService.PULLSERVER_SERVICEID), PullServerAuxService.class, Service.class); // Non-standard shuffle port conf.setInt(TajoConf.ConfVars.PULLSERVER_PORT.name(), 0); // local directory conf.set(TajoConf.ConfVars.WORKER_TEMPORAL_DIR.name(), "/tmp/tajo-localdir"); conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class, ContainerExecutor.class); // TestMRJobs is for testing non-uberized operation only; see TestUberAM // for corresponding uberized tests. conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); conf.setInt("yarn.nodemanager.delete.debug-delay-sec", 600); super.init(conf); }
From source file:org.apache.tez.service.MiniTezTestServiceCluster.java
License:Apache License
private MiniTezTestServiceCluster(String clusterName, int numExecutorsPerService, long availableMemory, int numLocalDirs) { super(clusterName + "_TezTestServerCluster"); Preconditions.checkArgument(numExecutorsPerService > 0); Preconditions.checkArgument(availableMemory > 0); Preconditions.checkArgument(numLocalDirs > 0); String clusterNameTrimmed = clusterName.replace("$", "") + "_TezTestServerCluster"; File targetWorkDir = new File("target", clusterNameTrimmed); try {//from ww w. ja v a2 s . c o m FileContext.getLocalFSFileContext().delete(new Path(targetWorkDir.getAbsolutePath()), true); } catch (Exception e) { LOG.warn("Could not cleanup test workDir: " + targetWorkDir, e); throw new RuntimeException("Could not cleanup test workDir: " + targetWorkDir, e); } if (Shell.WINDOWS) { // The test working directory can exceed the maximum path length supported // by some Windows APIs and cmd.exe (260 characters). To work around this, // create a symlink in temporary storage with a much shorter path, // targeting the full path to the test working directory. Then, use the // symlink as the test working directory. String targetPath = targetWorkDir.getAbsolutePath(); File link = new File(System.getProperty("java.io.tmpdir"), String.valueOf(System.currentTimeMillis())); String linkPath = link.getAbsolutePath(); try { FileContext.getLocalFSFileContext().delete(new Path(linkPath), true); } catch (IOException e) { throw new YarnRuntimeException("could not cleanup symlink: " + linkPath, e); } // Guarantee target exists before creating symlink. targetWorkDir.mkdirs(); Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( Shell.getSymlinkCommand(targetPath, linkPath)); try { shexec.execute(); } catch (IOException e) { throw new YarnRuntimeException( String.format("failed to create symlink from %s to %s, shell output: %s", linkPath, targetPath, shexec.getOutput()), e); } this.testWorkDir = link; } else { this.testWorkDir = targetWorkDir; } this.numExecutorsPerService = numExecutorsPerService; this.availableMemory = availableMemory; // Setup Local Dirs localDirs = new String[numLocalDirs]; for (int i = 0; i < numLocalDirs; i++) { File f = new File(testWorkDir, "localDir"); f.mkdirs(); LOG.info("Created localDir: " + f.getAbsolutePath()); localDirs[i] = f.getAbsolutePath(); } }
From source file:org.springframework.yarn.container.MethodInvokingYarnContainerRuntimeProcessor.java
License:Apache License
@Override public T process(YarnContainerRuntime yarnContainerRuntime) { try {/* w w w . jav a 2 s . c o m*/ return delegate.process(yarnContainerRuntime); } catch (Exception e) { throw new YarnRuntimeException("Error processing container", e); } }
From source file:oz.hadoop.yarn.test.cluster.MiniYarnCluster.java
License:Apache License
/** * * @param clusterName//from ww w . j av a 2 s . c om */ private void prepareScriptExecutionEnv(String clusterName) { String testSubDir = clusterName.replace("$", ""); File targetWorkDir = new File("target", testSubDir); try { FileContext.getLocalFSFileContext().delete(new Path(targetWorkDir.getAbsolutePath()), true); } catch (Exception e) { logger.warn("COULD NOT CLEANUP", e); throw new YarnRuntimeException("could not cleanup test dir: " + e, e); } if (Shell.WINDOWS) { // The test working directory can exceed the maximum path length // supported // by some Windows APIs and cmd.exe (260 characters). To work around // this, // create a symlink in temporary storage with a much shorter path, // targeting the full path to the test working directory. Then, use // the // symlink as the test working directory. String targetPath = targetWorkDir.getAbsolutePath(); File link = new File(System.getProperty("java.io.tmpdir"), String.valueOf(System.currentTimeMillis())); String linkPath = link.getAbsolutePath(); try { FileContext.getLocalFSFileContext().delete(new Path(linkPath), true); } catch (IOException e) { throw new YarnRuntimeException("could not cleanup symlink: " + linkPath, e); } // Guarantee target exists before creating symlink. targetWorkDir.mkdirs(); ShellCommandExecutor shexec = new ShellCommandExecutor(Shell.getSymlinkCommand(targetPath, linkPath)); try { shexec.execute(); } catch (IOException e) { throw new YarnRuntimeException( String.format("failed to create symlink from %s to %s, shell output: %s", linkPath, targetPath, shexec.getOutput()), e); } this.testWorkDir = link; } else { this.testWorkDir = targetWorkDir; } }