List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceVisibility PRIVATE
LocalResourceVisibility PRIVATE
To view the source code for org.apache.hadoop.yarn.api.records LocalResourceVisibility PRIVATE.
Click Source Link
From source file:com.yahoo.storm.yarn.StormAMRMClient.java
License:Open Source License
public void launchSupervisorOnContainer(Container container) throws IOException { // create a container launch context ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class); UserGroupInformation user = UserGroupInformation.getCurrentUser(); try {// w w w . ja v a 2 s .com Credentials credentials = user.getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); launchContext.setTokens(securityTokens); } catch (IOException e) { LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage()); } // CLC: env Map<String, String> env = new HashMap<String, String>(); env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); launchContext.setEnvironment(env); // CLC: local resources includes storm, conf Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); String storm_zip_path = (String) storm_conf.get("storm.zip.path"); Path zip = new Path(storm_zip_path); FileSystem fs = FileSystem.get(hadoopConf); String vis = (String) storm_conf.get("storm.zip.visibility"); if (vis.equals("PUBLIC")) localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC)); else if (vis.equals("PRIVATE")) localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PRIVATE)); else if (vis.equals("APPLICATION")) localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION)); String appHome = Util.getApplicationHomeForId(appAttemptId.toString()); Path confDst = Util.createConfigurationFileInFs(fs, appHome, this.storm_conf, this.hadoopConf); localResources.put("conf", Util.newYarnAppResource(fs, confDst)); launchContext.setLocalResources(localResources); // CLC: command List<String> supervisorArgs = Util.buildSupervisorCommands(this.storm_conf); launchContext.setCommands(supervisorArgs); try { LOG.info("Use NMClient to launch supervisors in container. "); nmClient.startContainer(container, launchContext); String userShortName = user.getShortUserName(); if (userShortName != null) LOG.info("Supervisor log: http://" + container.getNodeHttpAddress() + "/node/containerlogs/" + container.getId().toString() + "/" + userShortName + "/supervisor.log"); } catch (Exception e) { LOG.error("Caught an exception while trying to start a container", e); System.exit(-1); } }
From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java
License:Apache License
/** * Sets up the parameters for the Asterix config. * //from w ww .j a v a 2s .co m * @throws IOException */ private void distributeAsterixConfig() throws IOException { FileSystem fs = FileSystem.get(conf); String pathSuffix = instanceConfPath + File.separator + ASTERIX_CONF_NAME; Path dst = new Path(dfsBasePath, pathSuffix); URI paramLocation = dst.toUri(); FileStatus paramFileStatus = fs.getFileStatus(dst); Long paramLen = paramFileStatus.getLen(); Long paramTimestamp = paramFileStatus.getModificationTime(); LocalResource asterixParamLoc = Records.newRecord(LocalResource.class); asterixParamLoc.setType(LocalResourceType.FILE); asterixParamLoc.setVisibility(LocalResourceVisibility.PRIVATE); asterixParamLoc.setResource(ConverterUtils.getYarnUrlFromURI(paramLocation)); asterixParamLoc.setTimestamp(paramTimestamp); asterixParamLoc.setSize(paramLen); localResources.put(ASTERIX_CONF_NAME, asterixParamLoc); }
From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java
License:Apache License
/** * Here I am just pointing the Containers to the exisiting HDFS resources given by the Client * filesystem of the nodes.//from w ww . j av a2s. c om * * @throws IOException */ private void localizeDFSResources() throws IOException { //if performing an 'offline' task, skip a lot of resource distribution if (obliterate || backup || restore) { if (appMasterJar == null || ("").equals(appMasterJar)) { //this can happen in a jUnit testing environment. we don't need to set it there. if (!conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { throw new IllegalStateException("AM jar not provided in environment."); } else { return; } } FileSystem fs = FileSystem.get(conf); FileStatus appMasterJarStatus = fs.getFileStatus(appMasterJar); LocalResource obliteratorJar = Records.newRecord(LocalResource.class); obliteratorJar.setType(LocalResourceType.FILE); obliteratorJar.setVisibility(LocalResourceVisibility.PRIVATE); obliteratorJar.setResource(ConverterUtils.getYarnUrlFromPath(appMasterJar)); obliteratorJar.setTimestamp(appMasterJarStatus.getModificationTime()); obliteratorJar.setSize(appMasterJarStatus.getLen()); localResources.put("asterix-yarn.jar", obliteratorJar); LOG.info(localResources.values()); return; } //otherwise, distribute evertything to start up asterix LocalResource asterixZip = Records.newRecord(LocalResource.class); //this un-tar's the asterix distribution asterixZip.setType(LocalResourceType.ARCHIVE); asterixZip.setVisibility(LocalResourceVisibility.PRIVATE); try { asterixZip.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixZipPath))); } catch (URISyntaxException e) { LOG.error("Error locating Asterix zip" + " in env, path=" + asterixZipPath); throw new IOException(e); } asterixZip.setTimestamp(asterixZipTimestamp); asterixZip.setSize(asterixZipLen); localResources.put(ASTERIX_ZIP_NAME, asterixZip); //now let's do the same for the cluster description XML LocalResource asterixConf = Records.newRecord(LocalResource.class); asterixConf.setType(LocalResourceType.FILE); asterixConf.setVisibility(LocalResourceVisibility.PRIVATE); try { asterixConf.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixConfPath))); } catch (URISyntaxException e) { LOG.error("Error locating Asterix config" + " in env, path=" + asterixConfPath); throw new IOException(e); } //TODO: I could avoid localizing this everywhere by only calling this block on the metadata node. asterixConf.setTimestamp(asterixConfTimestamp); asterixConf.setSize(asterixConfLen); localResources.put("cluster-config.xml", asterixConf); //now add the libraries if there are any try { FileSystem fs = FileSystem.get(conf); Path p = new Path(dfsBasePath, instanceConfPath + File.separator + "library" + Path.SEPARATOR); if (fs.exists(p)) { FileStatus[] dataverses = fs.listStatus(p); for (FileStatus d : dataverses) { if (!d.isDirectory()) throw new IOException("Library configuration directory structure is incorrect"); FileStatus[] libraries = fs.listStatus(d.getPath()); for (FileStatus l : libraries) { if (l.isDirectory()) throw new IOException("Library configuration directory structure is incorrect"); LocalResource lr = Records.newRecord(LocalResource.class); lr.setResource(ConverterUtils.getYarnUrlFromURI(l.getPath().toUri())); lr.setSize(l.getLen()); lr.setTimestamp(l.getModificationTime()); lr.setType(LocalResourceType.ARCHIVE); lr.setVisibility(LocalResourceVisibility.PRIVATE); localResources.put("library" + Path.SEPARATOR + d.getPath().getName() + Path.SEPARATOR + l.getPath().getName().split("\\.")[0], lr); LOG.info("Found library: " + l.getPath().toString()); LOG.info(l.getPath().getName()); } } } } catch (FileNotFoundException e) { LOG.info("No external libraries present"); //do nothing, it just means there aren't libraries. that is possible and ok // it should be handled by the fs.exists(p) check though. } LOG.info(localResources.values()); }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Upload the Asterix cluster description on to the DFS. This will persist the state of the instance. * //from w w w .j a v a 2 s .c o m * @return * @throws YarnException * @throws IOException */ private List<DFSResourceCoordinate> deployConfig() throws YarnException, IOException { FileSystem fs = FileSystem.get(conf); List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); FileStatus destStatus; try { destStatus = fs.getFileStatus(dstConf); } catch (IOException e) { throw new YarnException("Asterix instance by that name does not appear to exist in DFS"); } LocalResource asterixConfLoc = Records.newRecord(LocalResource.class); asterixConfLoc.setType(LocalResourceType.FILE); asterixConfLoc.setVisibility(LocalResourceVisibility.PRIVATE); asterixConfLoc.setResource(ConverterUtils.getYarnUrlFromPath(dstConf)); asterixConfLoc.setTimestamp(destStatus.getModificationTime()); DFSResourceCoordinate conf = new DFSResourceCoordinate(); conf.envs.put(dstConf.toUri().toString(), AConstants.CONFLOCATION); conf.envs.put(Long.toString(asterixConfLoc.getSize()), AConstants.CONFLEN); conf.envs.put(Long.toString(asterixConfLoc.getTimestamp()), AConstants.CONFTIMESTAMP); conf.name = CONFIG_DEFAULT_NAME; conf.res = asterixConfLoc; resources.add(conf); return resources; }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Finds the minimal classes and JARs needed to start the AM only. * @return Resources the AM needs to start on the initial container. * @throws IllegalStateException//from w w w . j ava 2 s . c o m * @throws IOException */ private List<DFSResourceCoordinate> installAmLibs() throws IllegalStateException, IOException { List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); FileSystem fs = FileSystem.get(conf); String fullLibPath = CONF_DIR_REL + instanceFolder + "am_jars" + Path.SEPARATOR; String[] cp = System.getProperty("java.class.path").split(System.getProperty("path.separator")); String asterixJarPattern = "^(asterix).*(jar)$"; //starts with asterix,ends with jar String commonsJarPattern = "^(commons).*(jar)$"; String surefireJarPattern = "^(surefire).*(jar)$"; //for maven tests String jUnitTestPattern = "^(asterix-yarn" + File.separator + "target)$"; LOG.info(File.separator); for (String j : cp) { String[] pathComponents = j.split(Pattern.quote(File.separator)); LOG.info(j); LOG.info(pathComponents[pathComponents.length - 1]); if (pathComponents[pathComponents.length - 1].matches(asterixJarPattern) || pathComponents[pathComponents.length - 1].matches(commonsJarPattern) || pathComponents[pathComponents.length - 1].matches(surefireJarPattern) || pathComponents[pathComponents.length - 1].matches(jUnitTestPattern)) { LOG.info("Loading JAR/classpath: " + j); File f = new File(j); Path dst = new Path(fs.getHomeDirectory(), fullLibPath + f.getName()); if (!fs.exists(dst) || refresh) { fs.copyFromLocalFile(false, true, new Path(f.getAbsolutePath()), dst); } FileStatus dstSt = fs.getFileStatus(dst); LocalResource amLib = Records.newRecord(LocalResource.class); amLib.setType(LocalResourceType.FILE); amLib.setVisibility(LocalResourceVisibility.PRIVATE); amLib.setResource(ConverterUtils.getYarnUrlFromPath(dst)); amLib.setTimestamp(dstSt.getModificationTime()); amLib.setSize(dstSt.getLen()); DFSResourceCoordinate amLibCoord = new DFSResourceCoordinate(); amLibCoord.res = amLib; amLibCoord.name = f.getName(); if (f.getName().contains("asterix-yarn") || f.getName().contains("surefire")) { amLibCoord.envs.put(dst.toUri().toString(), AConstants.APPLICATIONMASTERJARLOCATION); amLibCoord.envs.put(Long.toString(dstSt.getLen()), AConstants.APPLICATIONMASTERJARLEN); amLibCoord.envs.put(Long.toString(dstSt.getModificationTime()), AConstants.APPLICATIONMASTERJARTIMESTAMP); } resources.add(amLibCoord); } } if (resources.size() == 0) { throw new IOException("Required JARs are missing. Please check your directory structure"); } return resources; }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Uploads binary resources to HDFS for use by the AM * @return//from w w w. ja va 2 s . co m * @throws IOException * @throws YarnException */ public List<DFSResourceCoordinate> distributeBinaries() throws IOException, YarnException { List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); Path src, dst; FileStatus destStatus; String pathSuffix; // adding info so we can add the jar to the App master container path // Add the asterix tarfile to HDFS for easy distribution // Keep it all archived for now so add it as a file... pathSuffix = CONF_DIR_REL + instanceFolder + "asterix-server.zip"; dst = new Path(fs.getHomeDirectory(), pathSuffix); if (refresh) { if (fs.exists(dst)) { fs.delete(dst, false); } } if (!fs.exists(dst)) { src = new Path(asterixZip); LOG.info("Copying Asterix distributable to DFS"); fs.copyFromLocalFile(false, true, src, dst); } destStatus = fs.getFileStatus(dst); LocalResource asterixTarLoc = Records.newRecord(LocalResource.class); asterixTarLoc.setType(LocalResourceType.ARCHIVE); asterixTarLoc.setVisibility(LocalResourceVisibility.PRIVATE); asterixTarLoc.setResource(ConverterUtils.getYarnUrlFromPath(dst)); asterixTarLoc.setTimestamp(destStatus.getModificationTime()); // adding info so we can add the tarball to the App master container path DFSResourceCoordinate tar = new DFSResourceCoordinate(); tar.envs.put(dst.toUri().toString(), AConstants.TARLOCATION); tar.envs.put(Long.toString(asterixTarLoc.getSize()), AConstants.TARLEN); tar.envs.put(Long.toString(asterixTarLoc.getTimestamp()), AConstants.TARTIMESTAMP); tar.res = asterixTarLoc; tar.name = "asterix-server.zip"; resources.add(tar); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); log4jRsrc.setType(LocalResourceType.FILE); log4jRsrc.setVisibility(LocalResourceVisibility.PRIVATE); log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri())); log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime()); log4jRsrc.setSize(log4jFileStatus.getLen()); DFSResourceCoordinate l4j = new DFSResourceCoordinate(); tar.res = log4jRsrc; tar.name = "log4j.properties"; resources.add(l4j); } resources.addAll(installAmLibs()); return resources; }
From source file:org.apache.asterix.aoya.AsterixApplicationMaster.java
License:Apache License
/** * Here I am just pointing the Containers to the exisiting HDFS resources given by the Client * filesystem of the nodes.//from w w w. jav a 2 s . c o m * * @throws IOException */ private void localizeDFSResources() throws IOException { //if performing an 'offline' task, skip a lot of resource distribution if (obliterate || backup || restore) { if (appMasterJar == null || ("").equals(appMasterJar)) { //this can happen in a jUnit testing environment. we don't need to set it there. if (!conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { throw new IllegalStateException("AM jar not provided in environment."); } else { return; } } FileSystem fs = FileSystem.get(conf); FileStatus appMasterJarStatus = fs.getFileStatus(appMasterJar); LocalResource obliteratorJar = Records.newRecord(LocalResource.class); obliteratorJar.setType(LocalResourceType.FILE); obliteratorJar.setVisibility(LocalResourceVisibility.PRIVATE); obliteratorJar.setResource(ConverterUtils.getYarnUrlFromPath(appMasterJar)); obliteratorJar.setTimestamp(appMasterJarStatus.getModificationTime()); obliteratorJar.setSize(appMasterJarStatus.getLen()); localResources.put("asterix-yarn.jar", obliteratorJar); LOG.info(localResources.values()); return; } //otherwise, distribute evertything to start up asterix LocalResource asterixZip = Records.newRecord(LocalResource.class); //this un-tar's the asterix distribution asterixZip.setType(LocalResourceType.ARCHIVE); asterixZip.setVisibility(LocalResourceVisibility.PRIVATE); try { asterixZip.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixZipPath))); } catch (URISyntaxException e) { LOG.error("Error locating Asterix zip" + " in env, path=" + asterixZipPath); throw new IOException(e); } asterixZip.setTimestamp(asterixZipTimestamp); asterixZip.setSize(asterixZipLen); localResources.put(ASTERIX_ZIP_NAME, asterixZip); //now let's do the same for the cluster description XML LocalResource asterixConf = Records.newRecord(LocalResource.class); asterixConf.setType(LocalResourceType.FILE); asterixConf.setVisibility(LocalResourceVisibility.PRIVATE); try { asterixConf.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixConfPath))); } catch (URISyntaxException e) { LOG.error("Error locating Asterix config" + " in env, path=" + asterixConfPath); throw new IOException(e); } //TODO: I could avoid localizing this everywhere by only calling this block on the metadata node. asterixConf.setTimestamp(asterixConfTimestamp); asterixConf.setSize(asterixConfLen); localResources.put("cluster-config.xml", asterixConf); //now add the libraries if there are any try { FileSystem fs = FileSystem.get(conf); Path p = new Path(dfsBasePath, instanceConfPath + File.separator + "library" + Path.SEPARATOR); if (fs.exists(p)) { FileStatus[] dataverses = fs.listStatus(p); for (FileStatus d : dataverses) { if (!d.isDirectory()) throw new IOException("Library configuration directory structure is incorrect"); FileStatus[] libraries = fs.listStatus(d.getPath()); for (FileStatus l : libraries) { if (l.isDirectory()) throw new IOException("Library configuration directory structure is incorrect"); LocalResource lr = Records.newRecord(LocalResource.class); lr.setResource(ConverterUtils.getYarnUrlFromURI(l.getPath().toUri())); lr.setSize(l.getLen()); lr.setTimestamp(l.getModificationTime()); lr.setType(LocalResourceType.ARCHIVE); lr.setVisibility(LocalResourceVisibility.PRIVATE); localResources.put("library" + Path.SEPARATOR + d.getPath().getName() + Path.SEPARATOR + l.getPath().getName().split("\\.")[0], lr); LOG.info("Found library: " + l.getPath().toString()); LOG.info(l.getPath().getName()); } } } } catch (FileNotFoundException e) { LOG.info("No external libraries present"); //do nothing, it just means there aren't libraries. that is possible and ok // it should be handled by the fs.exists(p) check though. } LOG.info(localResources.values()); }
From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java
License:Apache License
@Test public void testResourceMapSuccess() { Map<String, String> configMap = new HashMap<>(); configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme"); configMap.put("yarn.resources.myResource1.local.name", "readme"); configMap.put("yarn.resources.myResource1.local.type", "file"); configMap.put("yarn.resources.myResource1.local.visibility", "public"); configMap.put("yarn.resources.myResource2.path", "https://host2.com/package"); configMap.put("yarn.resources.myResource2.local.name", "__package"); configMap.put("yarn.resources.myResource2.local.type", "archive"); configMap.put("yarn.resources.myResource2.local.visibility", "private"); configMap.put("yarn.resources.myResource3.path", "https://host3.com/csr"); configMap.put("yarn.resources.myResource3.local.name", "csr"); configMap.put("yarn.resources.myResource3.local.type", "file"); configMap.put("yarn.resources.myResource3.local.visibility", "application"); configMap.put("otherconfig", "https://host4.com/not_included"); configMap.put("yarn.resources.myResource4.local.name", "notExisting"); configMap.put("yarn.resources.myResource4.local.type", "file"); configMap.put("yarn.resources.myResource4.local.visibility", "application"); Config conf = new MapConfig(configMap); YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName()); yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName()); LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);/*from w w w . j av a2s . c om*/ Map<String, LocalResource> resourceMap = mapper.getResourceMap(); assertEquals("resourceMap has 3 resources", 3, resourceMap.size()); // resource1 assertEquals("host1.com", resourceMap.get("readme").getResource().getHost()); assertEquals(LocalResourceType.FILE, resourceMap.get("readme").getType()); assertEquals(LocalResourceVisibility.PUBLIC, resourceMap.get("readme").getVisibility()); // resource 2 assertEquals("host2.com", resourceMap.get("__package").getResource().getHost()); assertEquals(LocalResourceType.ARCHIVE, resourceMap.get("__package").getType()); assertEquals(LocalResourceVisibility.PRIVATE, resourceMap.get("__package").getVisibility()); // resource 3 assertEquals("host3.com", resourceMap.get("csr").getResource().getHost()); assertEquals(LocalResourceType.FILE, resourceMap.get("csr").getType()); assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("csr").getVisibility()); // resource 4 should not exist assertNull("Resource does not exist with the name myResource4", resourceMap.get("myResource4")); assertNull("Resource does not exist with the defined config name notExisting for myResource4 either", resourceMap.get("notExisting")); }
From source file:org.apache.tez.client.TestTezClientUtils.java
License:Apache License
public static void testLocalResourceVisibility(DistributedFileSystem remoteFs, Configuration conf) throws Exception { Path topLevelDir = null;/*from w w w.j a v a2s .c o m*/ try { FsPermission publicDirPerms = new FsPermission((short) 0755); // rwxr-xr-x FsPermission privateDirPerms = new FsPermission((short) 0754); // rwxr-xr-- FsPermission publicFilePerms = new FsPermission((short) 0554); // r-xr-xr-- FsPermission privateFilePerms = new FsPermission((short) 0550); // r-xr-x--- String fsURI = remoteFs.getUri().toString(); topLevelDir = new Path(fsURI, "/testLRVisibility"); Assert.assertTrue(remoteFs.mkdirs(topLevelDir, publicDirPerms)); Path publicSubDir = new Path(topLevelDir, "public_sub_dir"); Assert.assertTrue(remoteFs.mkdirs(publicSubDir, publicDirPerms)); Path privateSubDir = new Path(topLevelDir, "private_sub_dir"); Assert.assertTrue(remoteFs.mkdirs(privateSubDir, privateDirPerms)); Path publicFile = new Path(publicSubDir, "public_file"); Assert.assertTrue(remoteFs.createNewFile(publicFile)); remoteFs.setPermission(publicFile, publicFilePerms); Path privateFile = new Path(publicSubDir, "private_file"); Assert.assertTrue(remoteFs.createNewFile(privateFile)); remoteFs.setPermission(privateFile, privateFilePerms); Path publicFileInPrivateSubdir = new Path(privateSubDir, "public_file_in_private_subdir"); Assert.assertTrue(remoteFs.createNewFile(publicFileInPrivateSubdir)); remoteFs.setPermission(publicFileInPrivateSubdir, publicFilePerms); TezConfiguration tezConf = new TezConfiguration(conf); String tmpTezLibUris = String.format("%s,%s,%s,%s", topLevelDir, publicSubDir, privateSubDir, conf.get(TezConfiguration.TEZ_LIB_URIS, "")); tezConf.set(TezConfiguration.TEZ_LIB_URIS, tmpTezLibUris); Map<String, LocalResource> lrMap = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap); Assert.assertEquals(publicFile.getName(), LocalResourceVisibility.PUBLIC, lrMap.get(publicFile.getName()).getVisibility()); Assert.assertEquals(privateFile.getName(), LocalResourceVisibility.PRIVATE, lrMap.get(privateFile.getName()).getVisibility()); Assert.assertEquals(publicFileInPrivateSubdir.getName(), LocalResourceVisibility.PRIVATE, lrMap.get(publicFileInPrivateSubdir.getName()).getVisibility()); // test tar.gz tezConf = new TezConfiguration(conf); Path tarFile = new Path(topLevelDir, "foo.tar.gz"); Assert.assertTrue(remoteFs.createNewFile(tarFile)); //public remoteFs.setPermission(tarFile, publicFilePerms); tezConf.set(TezConfiguration.TEZ_LIB_URIS, tarFile.toString()); lrMap.clear(); Assert.assertTrue(TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap)); Assert.assertEquals(LocalResourceVisibility.PUBLIC, lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility()); //private remoteFs.setPermission(tarFile, privateFilePerms); lrMap.clear(); TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap); Assert.assertEquals(LocalResourceVisibility.PRIVATE, lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility()); } finally { if (topLevelDir != null) { remoteFs.delete(topLevelDir, true); } } }
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
/** * Setup LocalResource map for Tez jars based on provided Configuration * //from ww w .j a va 2 s . c om * @param conf * Configuration to use to access Tez jars' locations * @param credentials * a credentials instance into which tokens for the Tez local * resources will be populated * @param tezJarResources Map of LocalResources to use for AM and DAGs * @return Whether the archive-based deployment of Tez was used. * @throws IOException */ static boolean setupTezJarsLocalResources(TezConfiguration conf, Credentials credentials, Map<String, LocalResource> tezJarResources) throws IOException { Preconditions.checkNotNull(credentials, "A non-null credentials object should be specified"); boolean usingTezArchive = false; if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)) { LOG.info("Ignoring '" + TezConfiguration.TEZ_LIB_URIS + "' since '" + TezConfiguration.TEZ_IGNORE_LIB_URIS + "' is set to true"); } else { // Add tez jars to local resource String[] tezJarUris = conf.getStrings(TezConfiguration.TEZ_LIB_URIS); if (tezJarUris == null || tezJarUris.length == 0) { throw new TezUncheckedException("Invalid configuration of tez jars" + ", " + TezConfiguration.TEZ_LIB_URIS + " is not defined in the configuration"); } LOG.info("Using tez.lib.uris value from configuration: " + conf.get(TezConfiguration.TEZ_LIB_URIS)); if (tezJarUris.length == 1 && (tezJarUris[0].endsWith(".tar.gz") || tezJarUris[0].endsWith(".tgz") || tezJarUris[0].endsWith(".zip") || tezJarUris[0].endsWith(".tar"))) { String fileName = tezJarUris[0]; FileStatus fStatus = getLRFileStatus(fileName, conf)[0]; LocalResourceVisibility lrVisibility; if (checkAncestorPermissionsForAllUsers(conf, fileName, FsAction.EXECUTE) && fStatus.getPermission().getOtherAction().implies(FsAction.READ)) { lrVisibility = LocalResourceVisibility.PUBLIC; } else { lrVisibility = LocalResourceVisibility.PRIVATE; } tezJarResources.put(TezConstants.TEZ_TAR_LR_NAME, LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fStatus.getPath()), LocalResourceType.ARCHIVE, lrVisibility, fStatus.getLen(), fStatus.getModificationTime())); Path[] tezJarPaths = { fStatus.getPath() }; // obtain credentials TokenCache.obtainTokensForFileSystems(credentials, tezJarPaths, conf); usingTezArchive = true; } else { // Treat as non-archives addLocalResources(conf, tezJarUris, tezJarResources, credentials); } if (tezJarResources.isEmpty()) { throw new TezUncheckedException("No files found in locations specified in " + TezConfiguration.TEZ_LIB_URIS + " . Locations: " + StringUtils.join(tezJarUris, ',')); } } // Add aux uris to local resources addLocalResources(conf, conf.getStrings(TezConfiguration.TEZ_AUX_URIS), tezJarResources, credentials); return usingTezArchive; }