List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceVisibility PUBLIC
LocalResourceVisibility PUBLIC
To view the source code for org.apache.hadoop.yarn.api.records LocalResourceVisibility PUBLIC.
Click Source Link
From source file:alluxio.yarn.YarnUtils.java
License:Apache License
/** * Creates a local resource for a file on HDFS. * * @param yarnConf YARN configuration// ww w .j av a 2s. c om * @param resource the path to a resource file on HDFS * @throws IOException if the file can not be found on HDFS * @return the created local resource */ public static LocalResource createLocalResourceOfFile(YarnConfiguration yarnConf, String resource) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); Path resourcePath = new Path(resource); FileStatus jarStat = FileSystem.get(resourcePath.toUri(), yarnConf).getFileStatus(resourcePath); localResource.setResource(ConverterUtils.getYarnUrlFromPath(resourcePath)); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.PUBLIC); return localResource; }
From source file:com.github.sakserv.minicluster.simpleyarnapp.Client.java
License:Apache License
private void setupAppMasterJar(Path jarPath, LocalResource appMasterJar) throws IOException { FileStatus jarStat = FileSystem.get(conf).getFileStatus(jarPath); appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); appMasterJar.setSize(jarStat.getLen()); appMasterJar.setTimestamp(jarStat.getModificationTime()); appMasterJar.setType(LocalResourceType.FILE); appMasterJar.setVisibility(LocalResourceVisibility.PUBLIC); }
From source file:com.gpiskas.yarn.Utils.java
License:Open Source License
public static void setUpLocalResource(Path resPath, LocalResource res, YarnConfiguration conf) throws IOException { Path qPath = FileContext.getFileContext().makeQualified(resPath); FileStatus status = FileSystem.get(conf).getFileStatus(qPath); res.setResource(ConverterUtils.getYarnUrlFromPath(qPath)); res.setSize(status.getLen());/*w w w. j a v a 2 s .c om*/ res.setTimestamp(status.getModificationTime()); res.setType(LocalResourceType.FILE); res.setVisibility(LocalResourceVisibility.PUBLIC); }
From source file:com.ibm.bi.dml.yarn.DMLYarnClient.java
License:Open Source License
/** * /*w ww . j a va 2 s .com*/ * @param yconf * @param path * @param lpath * @return * @throws IOException */ private Map<String, LocalResource> constructLocalResourceMap(YarnConfiguration yconf) throws IOException { Map<String, LocalResource> rMap = new HashMap<String, LocalResource>(); Path path = new Path(_hdfsJarFile); LocalResource resource = Records.newRecord(LocalResource.class); FileStatus jarStat = FileSystem.get(yconf).getFileStatus(path); resource.setResource(ConverterUtils.getYarnUrlFromPath(path)); resource.setSize(jarStat.getLen()); resource.setTimestamp(jarStat.getModificationTime()); resource.setType(LocalResourceType.FILE); resource.setVisibility(LocalResourceVisibility.PUBLIC); rMap.put(DML_JAR_NAME, resource); return rMap; }
From source file:com.yahoo.storm.yarn.StormAMRMClient.java
License:Open Source License
public void launchSupervisorOnContainer(Container container) throws IOException { // create a container launch context ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class); UserGroupInformation user = UserGroupInformation.getCurrentUser(); try {/*from w w w . ja va 2s. c om*/ Credentials credentials = user.getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); launchContext.setTokens(securityTokens); } catch (IOException e) { LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage()); } // CLC: env Map<String, String> env = new HashMap<String, String>(); env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); launchContext.setEnvironment(env); // CLC: local resources includes storm, conf Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); String storm_zip_path = (String) storm_conf.get("storm.zip.path"); Path zip = new Path(storm_zip_path); FileSystem fs = FileSystem.get(hadoopConf); String vis = (String) storm_conf.get("storm.zip.visibility"); if (vis.equals("PUBLIC")) localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC)); else if (vis.equals("PRIVATE")) localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PRIVATE)); else if (vis.equals("APPLICATION")) localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION)); String appHome = Util.getApplicationHomeForId(appAttemptId.toString()); Path confDst = Util.createConfigurationFileInFs(fs, appHome, this.storm_conf, this.hadoopConf); localResources.put("conf", Util.newYarnAppResource(fs, confDst)); launchContext.setLocalResources(localResources); // CLC: command List<String> supervisorArgs = Util.buildSupervisorCommands(this.storm_conf); launchContext.setCommands(supervisorArgs); try { LOG.info("Use NMClient to launch supervisors in container. "); nmClient.startContainer(container, launchContext); String userShortName = user.getShortUserName(); if (userShortName != null) LOG.info("Supervisor log: http://" + container.getNodeHttpAddress() + "/node/containerlogs/" + container.getId().toString() + "/" + userShortName + "/supervisor.log"); } catch (Exception e) { LOG.error("Caught an exception while trying to start a container", e); System.exit(-1); } }
From source file:com.yahoo.storm.yarn.StormOnYarn.java
License:Open Source License
private void launchApp(String appName, String queue, int amMB, String storm_zip_location) throws Exception { LOG.debug("StormOnYarn:launchApp() ..."); YarnClientApplication client_app = _yarn.createApplication(); GetNewApplicationResponse app = client_app.getNewApplicationResponse(); _appId = app.getApplicationId();//from w w w . j a v a 2 s . c o m LOG.debug("_appId:" + _appId); if (amMB > app.getMaximumResourceCapability().getMemory()) { //TODO need some sanity checks amMB = app.getMaximumResourceCapability().getMemory(); } ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); appContext.setApplicationId(app.getApplicationId()); appContext.setApplicationName(appName); appContext.setQueue(queue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the // local resources LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path String appMasterJar = findContainingJar(MasterServer.class); FileSystem fs = FileSystem.get(_hadoopConf); Path src = new Path(appMasterJar); String appHome = Util.getApplicationHomeForId(_appId.toString()); Path dst = new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + "AppMaster.jar"); fs.copyFromLocalFile(false, true, src, dst); localResources.put("AppMaster.jar", Util.newYarnAppResource(fs, dst)); String stormVersion = Util.getStormVersion(); Path zip; if (storm_zip_location != null) { zip = new Path(storm_zip_location); } else { zip = new Path("/lib/storm/" + stormVersion + "/storm.zip"); } _stormConf.put("storm.zip.path", zip.makeQualified(fs).toUri().getPath()); LocalResourceVisibility visibility = LocalResourceVisibility.PUBLIC; _stormConf.put("storm.zip.visibility", "PUBLIC"); if (!Util.isPublic(fs, zip)) { visibility = LocalResourceVisibility.APPLICATION; _stormConf.put("storm.zip.visibility", "APPLICATION"); } localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, visibility)); Path confDst = Util.createConfigurationFileInFs(fs, appHome, _stormConf, _hadoopConf); // establish a symbolic link to conf directory localResources.put("conf", Util.newYarnAppResource(fs, confDst)); // Setup security tokens Path[] paths = new Path[3]; paths[0] = dst; paths[1] = zip; paths[2] = confDst; Credentials credentials = new Credentials(); TokenCache.obtainTokensForNamenodes(credentials, paths, _hadoopConf); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); //security tokens for HDFS distributed cache amContainer.setTokens(securityTokens); // Set local resource info into app master container launch context amContainer.setLocalResources(localResources); // Set the env variables to be setup in the env where the application master // will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // add the runtime classpath needed for tests to work Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./conf"); Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./AppMaster.jar"); //Make sure that AppMaster has access to all YARN JARs List<String> yarn_classpath_cmd = java.util.Arrays.asList("yarn", "classpath"); ProcessBuilder pb = new ProcessBuilder(yarn_classpath_cmd).redirectError(Redirect.INHERIT); LOG.info("YARN CLASSPATH COMMAND = [" + yarn_classpath_cmd + "]"); pb.environment().putAll(System.getenv()); Process proc = pb.start(); BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream(), "UTF-8")); String line = ""; String yarn_class_path = (String) _stormConf.get("storm.yarn.yarn_classpath"); if (yarn_class_path == null) { StringBuilder yarn_class_path_builder = new StringBuilder(); while ((line = reader.readLine()) != null) { yarn_class_path_builder.append(line); } yarn_class_path = yarn_class_path_builder.toString(); } LOG.info("YARN CLASSPATH = [" + yarn_class_path + "]"); proc.waitFor(); reader.close(); Apps.addToEnvironment(env, Environment.CLASSPATH.name(), yarn_class_path); String stormHomeInZip = Util.getStormHomeInZip(fs, zip, stormVersion); Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./storm/" + stormHomeInZip + "/*"); Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./storm/" + stormHomeInZip + "/lib/*"); String java_home = (String) _stormConf.get("storm.yarn.java_home"); if (java_home == null) java_home = System.getenv("JAVA_HOME"); if (java_home != null && !java_home.isEmpty()) env.put("JAVA_HOME", java_home); LOG.info("Using JAVA_HOME = [" + env.get("JAVA_HOME") + "]"); env.put("appJar", appMasterJar); env.put("appName", appName); env.put("appId", new Integer(_appId.getId()).toString()); env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<String> vargs = new Vector<String>(); if (java_home != null && !java_home.isEmpty()) vargs.add(env.get("JAVA_HOME") + "/bin/java"); else vargs.add("java"); vargs.add("-Dstorm.home=./storm/" + stormHomeInZip + "/"); vargs.add("-Dlogfile.name=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/master.log"); //vargs.add("-verbose:class"); vargs.add("com.yahoo.storm.yarn.MasterServer"); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); // Set java executable command LOG.info("Setting up app master command:" + vargs); amContainer.setCommands(vargs); // Set up resource type requirements // For now, only memory is supported so we set memory requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMB); appContext.setResource(capability); appContext.setAMContainerSpec(amContainer); _yarn.submitApplication(appContext); }
From source file:eu.stratosphere.yarn.Utils.java
License:Apache License
public static void registerLocalResource(FileSystem fs, Path remoteRsrcPath, LocalResource localResource) throws IOException { FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.PUBLIC); }
From source file:hws.core.ContainerUtils.java
License:Apache License
public static void setupContainerJar(FileSystem fs, Path jarPath, LocalResource containerJar) throws IOException { FileStatus jarStat = fs.getFileStatus(jarPath); containerJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); containerJar.setSize(jarStat.getLen()); containerJar.setTimestamp(jarStat.getModificationTime()); containerJar.setType(LocalResourceType.FILE); containerJar.setVisibility(LocalResourceVisibility.PUBLIC); }
From source file:MasteringYarn.DistributedShellClient.java
private void setupJarFileForApplicationMaster(Path jarPath, LocalResource localResource) throws IOException { FileStatus jarStat = FileSystem.get(conf).getFileStatus(jarPath); localResource.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.PUBLIC); }
From source file:org.apache.samza.job.yarn.TestLocalizerResourceConfig.java
License:Apache License
@Test public void testResourceConfigIncluded() { Map<String, String> configMap = new HashMap<>(); configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme"); configMap.put("yarn.resources.myResource1.local.name", "readme"); configMap.put("yarn.resources.myResource1.local.type", "file"); configMap.put("yarn.resources.myResource1.local.visibility", "public"); Config conf = new MapConfig(configMap); LocalizerResourceConfig manager = new LocalizerResourceConfig(conf); assertEquals(1, manager.getResourceNames().size()); assertEquals("myResource1", manager.getResourceNames().get(0)); assertEquals("readme", manager.getResourceLocalName("myResource1")); assertEquals(LocalResourceType.FILE, manager.getResourceLocalType("myResource1")); assertEquals(LocalResourceVisibility.PUBLIC, manager.getResourceLocalVisibility("myResource1")); }