List of usage examples for org.apache.hadoop.yarn.api.records LocalResource setVisibility
@Public @Stable public abstract void setVisibility(LocalResourceVisibility visibility);
LocalResourceVisibility of the resource to be localized. From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Uploads binary resources to HDFS for use by the AM * @return/*www. j a v a 2 s .co m*/ * @throws IOException * @throws YarnException */ public List<DFSResourceCoordinate> distributeBinaries() throws IOException, YarnException { List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); Path src, dst; FileStatus destStatus; String pathSuffix; // adding info so we can add the jar to the App master container path // Add the asterix tarfile to HDFS for easy distribution // Keep it all archived for now so add it as a file... pathSuffix = CONF_DIR_REL + instanceFolder + "asterix-server.zip"; dst = new Path(fs.getHomeDirectory(), pathSuffix); if (refresh) { if (fs.exists(dst)) { fs.delete(dst, false); } } if (!fs.exists(dst)) { src = new Path(asterixZip); LOG.info("Copying Asterix distributable to DFS"); fs.copyFromLocalFile(false, true, src, dst); } destStatus = fs.getFileStatus(dst); LocalResource asterixTarLoc = Records.newRecord(LocalResource.class); asterixTarLoc.setType(LocalResourceType.ARCHIVE); asterixTarLoc.setVisibility(LocalResourceVisibility.PRIVATE); asterixTarLoc.setResource(ConverterUtils.getYarnUrlFromPath(dst)); asterixTarLoc.setTimestamp(destStatus.getModificationTime()); // adding info so we can add the tarball to the App master container path DFSResourceCoordinate tar = new DFSResourceCoordinate(); tar.envs.put(dst.toUri().toString(), AConstants.TARLOCATION); tar.envs.put(Long.toString(asterixTarLoc.getSize()), AConstants.TARLEN); tar.envs.put(Long.toString(asterixTarLoc.getTimestamp()), AConstants.TARTIMESTAMP); tar.res = asterixTarLoc; tar.name = "asterix-server.zip"; resources.add(tar); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); log4jRsrc.setType(LocalResourceType.FILE); log4jRsrc.setVisibility(LocalResourceVisibility.PRIVATE); log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri())); log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime()); log4jRsrc.setSize(log4jFileStatus.getLen()); DFSResourceCoordinate l4j = new DFSResourceCoordinate(); tar.res = log4jRsrc; tar.name = "log4j.properties"; resources.add(l4j); } resources.addAll(installAmLibs()); return resources; }
From source file:edu.uci.ics.hyracks.yarn.common.resources.LocalResourceHelper.java
License:Apache License
private static LocalResource createLocalResourceFromPath(Configuration config, File path) throws IOException { LocalResource lr = Records.newRecord(LocalResource.class); URL url = ConverterUtils/*from ww w. j a v a 2 s . c o m*/ .getYarnUrlFromPath(FileContext.getFileContext().makeQualified(new Path(path.toURI()))); lr.setResource(url); lr.setVisibility(LocalResourceVisibility.APPLICATION); lr.setTimestamp(path.lastModified()); lr.setSize(path.length()); return lr; }
From source file:eu.stratosphere.yarn.Utils.java
License:Apache License
public static void registerLocalResource(FileSystem fs, Path remoteRsrcPath, LocalResource localResource) throws IOException { FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.PUBLIC); }
From source file:gobblin.yarn.YarnHelixUtils.java
License:Apache License
/** * Add a file as a Yarn {@link org.apache.hadoop.yarn.api.records.LocalResource}. * * @param fs a {@link FileSystem} instance * @param destFilePath the destination file path * @param resourceType the {@link org.apache.hadoop.yarn.api.records.LocalResourceType} of the file * @param resourceMap a {@link Map} of file names to their corresponding * {@link org.apache.hadoop.yarn.api.records.LocalResource}s * @throws IOException if there's something wrong adding the file as a * {@link org.apache.hadoop.yarn.api.records.LocalResource} */// www .j av a2 s .c o m public static void addFileAsLocalResource(FileSystem fs, Path destFilePath, LocalResourceType resourceType, Map<String, LocalResource> resourceMap) throws IOException { LocalResource fileResource = Records.newRecord(LocalResource.class); FileStatus fileStatus = fs.getFileStatus(destFilePath); fileResource.setResource(ConverterUtils.getYarnUrlFromPath(destFilePath)); fileResource.setSize(fileStatus.getLen()); fileResource.setTimestamp(fileStatus.getModificationTime()); fileResource.setType(resourceType); fileResource.setVisibility(LocalResourceVisibility.APPLICATION); resourceMap.put(destFilePath.getName(), fileResource); }
From source file:husky.client.HuskyYarnClient.java
License:Apache License
private Pair<String, LocalResource> constructLocalResource(String name, String path, LocalResourceType type) throws IOException { LOG.info("To copy " + name + "(" + path + ") from local file system"); Path resourcePath = new Path(path); if (path.startsWith("hdfs://")) { FileStatus fileStatus = mFileSystem.getFileStatus(resourcePath); if (!fileStatus.isFile()) { throw new RuntimeException("Only files can be provided as local resources."); }/*from ww w. j a v a 2s.c om*/ } else { File file = new File(path); if (!file.exists()) { throw new RuntimeException("File not exist: " + path); } if (!file.isFile()) { throw new RuntimeException("Only files can be provided as local resources."); } } // if the file is not on hdfs, upload it to hdfs first. if (!path.startsWith("hdfs://")) { Path src = resourcePath; String newPath = mLocalResourceHDFSPaths + '/' + mAppName + '/' + mAppId + '/' + name; resourcePath = new Path(newPath); mFileSystem.copyFromLocalFile(false, true, src, resourcePath); LOG.info("Upload " + path + " to " + newPath); path = newPath; } FileStatus fileStatus = mFileSystem.getFileStatus(resourcePath); LocalResource resource = Records.newRecord(LocalResource.class); resource.setType(type); resource.setVisibility(LocalResourceVisibility.APPLICATION); resource.setResource(ConverterUtils.getYarnUrlFromPath(resourcePath)); resource.setTimestamp(fileStatus.getModificationTime()); resource.setSize(fileStatus.getLen()); return new Pair<String, LocalResource>(path, resource); }
From source file:husky.server.HuskyRMCallbackHandler.java
License:Apache License
private LocalResource constructLocalResource(String name, String path, LocalResourceType type) throws IOException { LOG.info("To copy " + name + " from " + path); if (!path.startsWith("hdfs://")) { throw new RuntimeException( "Local resources provided to application master must be already located on HDFS."); }//from w w w. ja va 2 s . co m FileStatus dsfStaus = mAppMaster.getFileSystem().getFileStatus(new Path(path)); LocalResource resource = Records.newRecord(LocalResource.class); resource.setType(type); resource.setVisibility(LocalResourceVisibility.APPLICATION); resource.setResource(ConverterUtils.getYarnUrlFromPath(new Path(path))); resource.setTimestamp(dsfStaus.getModificationTime()); resource.setSize(dsfStaus.getLen()); return resource; }
From source file:hws.core.ContainerUtils.java
License:Apache License
public static void setupContainerJar(FileSystem fs, Path jarPath, LocalResource containerJar) throws IOException { FileStatus jarStat = fs.getFileStatus(jarPath); containerJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); containerJar.setSize(jarStat.getLen()); containerJar.setTimestamp(jarStat.getModificationTime()); containerJar.setType(LocalResourceType.FILE); containerJar.setVisibility(LocalResourceVisibility.PUBLIC); }
From source file:MasteringYarn.DistributedShellClient.java
private void setupJarFileForApplicationMaster(Path jarPath, LocalResource localResource) throws IOException { FileStatus jarStat = FileSystem.get(conf).getFileStatus(jarPath); localResource.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.PUBLIC); }
From source file:ml.shifu.guagua.yarn.util.YarnUtils.java
License:Apache License
/** * Boilerplate to add a file to the local resources.. * //from ww w . j av a 2 s . c o m * @param localResources * the LocalResources map to populate. * @param fs * handle to the HDFS file system. * @param target * the file to send to the remote container. */ public static void addFileToResourceMap(Map<String, LocalResource> localResources, FileSystem fs, Path target) throws IOException { LocalResource resource = Records.newRecord(LocalResource.class); FileStatus destStatus = fs.getFileStatus(target); resource.setResource(ConverterUtils.getYarnUrlFromURI(target.toUri())); resource.setSize(destStatus.getLen()); resource.setTimestamp(destStatus.getModificationTime()); resource.setType(LocalResourceType.FILE); // use FILE, even for jars! resource.setVisibility(LocalResourceVisibility.APPLICATION); localResources.put(target.getName(), resource); LOG.info("Registered file in LocalResources :{} ", target); }
From source file:org.apache.asterix.aoya.AsterixApplicationMaster.java
License:Apache License
/** * Here I am just pointing the Containers to the exisiting HDFS resources given by the Client * filesystem of the nodes.//from w w w . j a va2s . c o m * * @throws IOException */ private void localizeDFSResources() throws IOException { //if performing an 'offline' task, skip a lot of resource distribution if (obliterate || backup || restore) { if (appMasterJar == null || ("").equals(appMasterJar)) { //this can happen in a jUnit testing environment. we don't need to set it there. if (!conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { throw new IllegalStateException("AM jar not provided in environment."); } else { return; } } FileSystem fs = FileSystem.get(conf); FileStatus appMasterJarStatus = fs.getFileStatus(appMasterJar); LocalResource obliteratorJar = Records.newRecord(LocalResource.class); obliteratorJar.setType(LocalResourceType.FILE); obliteratorJar.setVisibility(LocalResourceVisibility.PRIVATE); obliteratorJar.setResource(ConverterUtils.getYarnUrlFromPath(appMasterJar)); obliteratorJar.setTimestamp(appMasterJarStatus.getModificationTime()); obliteratorJar.setSize(appMasterJarStatus.getLen()); localResources.put("asterix-yarn.jar", obliteratorJar); LOG.info(localResources.values()); return; } //otherwise, distribute evertything to start up asterix LocalResource asterixZip = Records.newRecord(LocalResource.class); //this un-tar's the asterix distribution asterixZip.setType(LocalResourceType.ARCHIVE); asterixZip.setVisibility(LocalResourceVisibility.PRIVATE); try { asterixZip.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixZipPath))); } catch (URISyntaxException e) { LOG.error("Error locating Asterix zip" + " in env, path=" + asterixZipPath); throw new IOException(e); } asterixZip.setTimestamp(asterixZipTimestamp); asterixZip.setSize(asterixZipLen); localResources.put(ASTERIX_ZIP_NAME, asterixZip); //now let's do the same for the cluster description XML LocalResource asterixConf = Records.newRecord(LocalResource.class); asterixConf.setType(LocalResourceType.FILE); asterixConf.setVisibility(LocalResourceVisibility.PRIVATE); try { asterixConf.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixConfPath))); } catch (URISyntaxException e) { LOG.error("Error locating Asterix config" + " in env, path=" + asterixConfPath); throw new IOException(e); } //TODO: I could avoid localizing this everywhere by only calling this block on the metadata node. asterixConf.setTimestamp(asterixConfTimestamp); asterixConf.setSize(asterixConfLen); localResources.put("cluster-config.xml", asterixConf); //now add the libraries if there are any try { FileSystem fs = FileSystem.get(conf); Path p = new Path(dfsBasePath, instanceConfPath + File.separator + "library" + Path.SEPARATOR); if (fs.exists(p)) { FileStatus[] dataverses = fs.listStatus(p); for (FileStatus d : dataverses) { if (!d.isDirectory()) throw new IOException("Library configuration directory structure is incorrect"); FileStatus[] libraries = fs.listStatus(d.getPath()); for (FileStatus l : libraries) { if (l.isDirectory()) throw new IOException("Library configuration directory structure is incorrect"); LocalResource lr = Records.newRecord(LocalResource.class); lr.setResource(ConverterUtils.getYarnUrlFromURI(l.getPath().toUri())); lr.setSize(l.getLen()); lr.setTimestamp(l.getModificationTime()); lr.setType(LocalResourceType.ARCHIVE); lr.setVisibility(LocalResourceVisibility.PRIVATE); localResources.put("library" + Path.SEPARATOR + d.getPath().getName() + Path.SEPARATOR + l.getPath().getName().split("\\.")[0], lr); LOG.info("Found library: " + l.getPath().toString()); LOG.info(l.getPath().getName()); } } } } catch (FileNotFoundException e) { LOG.info("No external libraries present"); //do nothing, it just means there aren't libraries. that is possible and ok // it should be handled by the fs.exists(p) check though. } LOG.info(localResources.values()); }