List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceType PATTERN
LocalResourceType PATTERN
To view the source code for org.apache.hadoop.yarn.api.records LocalResourceType PATTERN.
Click Source Link
From source file:cascading.flow.tez.util.TezUtil.java
License:Open Source License
protected static void addResource(Map<String, LocalResource> localResources, Map<String, String> environment, String fileName, FileStatus stats, Path fullPath, LocalResourceType type) throws IOException { if (localResources.containsKey(fileName)) throw new FlowException("duplicate filename added to classpath resources: " + fileName); URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(fullPath); long len = stats.getLen(); long modificationTime = stats.getModificationTime(); LocalResource resource = LocalResource.newInstance(yarnUrlFromPath, type, LocalResourceVisibility.APPLICATION, len, modificationTime); if (type == LocalResourceType.PATTERN) { // todo: parametrize this for dynamic inclusion below String pattern = "(?:classes/|lib/).*"; resource.setPattern(pattern);/* w ww .ja v a 2 s . co m*/ if (environment != null) { String current = ""; current += PWD.$$() + File.separator + fileName + File.separator + "*" + CLASS_PATH_SEPARATOR; current += PWD.$$() + File.separator + fileName + File.separator + "lib" + File.separator + "*" + CLASS_PATH_SEPARATOR; current += PWD.$$() + File.separator + fileName + File.separator + "classes" + File.separator + "*" + CLASS_PATH_SEPARATOR; String classPath = environment.get(CLASSPATH.name()); if (classPath == null) classPath = ""; else if (!classPath.startsWith(CLASS_PATH_SEPARATOR)) classPath += CLASS_PATH_SEPARATOR; classPath += current; LOG.info("adding to cluster side classpath: {} ", classPath); environment.put(CLASSPATH.name(), classPath); } } localResources.put(fileName, resource); }
From source file:com.continuuity.weave.yarn.utils.YarnUtils.java
License:Open Source License
private static LocalResource setLocalResourceType(LocalResource localResource, LocalFile localFile) { if (localFile.isArchive()) { if (localFile.getPattern() == null) { localResource.setType(LocalResourceType.ARCHIVE); } else {//from ww w .j a v a2 s.co m localResource.setType(LocalResourceType.PATTERN); localResource.setPattern(localFile.getPattern()); } } else { localResource.setType(LocalResourceType.FILE); } return localResource; }
From source file:com.scaleoutsoftware.soss.hserver.hadoop.DistributedCacheManager.java
License:Apache License
/** * Set up the distributed cache by localizing the resources, and updating * the configuration with references to the localized resources. * @param conf job configuration// w w w.j a v a 2s.c o m * @throws IOException */ public void setup(Configuration conf) throws IOException { //If we are not 0th worker, wait for 0th worker to set up the cache if (InvocationWorker.getIgWorkerIndex() > 0 && InvocationWorker.getNumberOfWorkers() > 1) { try { InvocationWorker.getSynchronizationBarrier().waitForComplete(ACTION_NAME, SYNCHRONIZATION_WAIT_MS, WAIT_GRANULARITY_MS); } catch (Exception e) { throw new RuntimeException(e); } return; } File workDir = new File(System.getProperty("user.dir")); // Generate YARN local resources objects corresponding to the distributed // cache configuration Map<String, LocalResource> localResources = new LinkedHashMap<String, LocalResource>(); MRApps.setupDistributedCache(conf, localResources); //CODE CHANGE FROM ORIGINAL FILE: //We need to clear the resources from jar files, since they are distributed through the IG. // Iterator<Map.Entry<String, LocalResource>> iterator = localResources.entrySet().iterator(); while (iterator.hasNext()) { Entry<String, LocalResource> entry = iterator.next(); if (entry.getKey().endsWith(".jar")) { iterator.remove(); } } // Generating unique numbers for FSDownload. AtomicLong uniqueNumberGenerator = new AtomicLong(System.currentTimeMillis()); // Find which resources are to be put on the local classpath Map<String, Path> classpaths = new HashMap<String, Path>(); Path[] archiveClassPaths = DistributedCache.getArchiveClassPaths(conf); if (archiveClassPaths != null) { for (Path p : archiveClassPaths) { FileSystem remoteFS = p.getFileSystem(conf); p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory())); classpaths.put(p.toUri().getPath().toString(), p); } } Path[] fileClassPaths = DistributedCache.getFileClassPaths(conf); if (fileClassPaths != null) { for (Path p : fileClassPaths) { FileSystem remoteFS = p.getFileSystem(conf); p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory())); classpaths.put(p.toUri().getPath().toString(), p); } } // Localize the resources LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LOCAL_DIR); FileContext localFSFileContext = FileContext.getLocalFSFileContext(); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ExecutorService exec = null; try { ThreadFactory tf = new ThreadFactoryBuilder() .setNameFormat("LocalDistributedCacheManager Downloader #%d").build(); exec = Executors.newCachedThreadPool(tf); Path destPath = localDirAllocator.getLocalPathForWrite(".", conf); Map<LocalResource, Future<Path>> resourcesToPaths = Maps.newHashMap(); for (LocalResource resource : localResources.values()) { Callable<Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet())), resource); Future<Path> future = exec.submit(download); resourcesToPaths.put(resource, future); } for (Entry<String, LocalResource> entry : localResources.entrySet()) { LocalResource resource = entry.getValue(); Path path; try { path = resourcesToPaths.get(resource).get(); } catch (InterruptedException e) { throw new IOException(e); } catch (ExecutionException e) { throw new IOException(e); } String pathString = path.toUri().toString(); String link = entry.getKey(); String target = new File(path.toUri()).getPath(); symlink(workDir, target, link); if (resource.getType() == LocalResourceType.ARCHIVE) { localArchives.add(pathString); } else if (resource.getType() == LocalResourceType.FILE) { localFiles.add(pathString); } else if (resource.getType() == LocalResourceType.PATTERN) { //PATTERN is not currently used in local mode throw new IllegalArgumentException( "Resource type PATTERN is not " + "implemented yet. " + resource.getResource()); } Path resourcePath; try { resourcePath = ConverterUtils.getPathFromYarnURL(resource.getResource()); } catch (URISyntaxException e) { throw new IOException(e); } LOG.info(String.format("Localized %s as %s", resourcePath, path)); String cp = resourcePath.toUri().getPath(); if (classpaths.keySet().contains(cp)) { localClasspaths.add(path.toUri().getPath().toString()); } } } finally { if (exec != null) { exec.shutdown(); } } // Update the configuration object with localized data. if (!localArchives.isEmpty()) { conf.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils.arrayToString(localArchives.toArray(new String[localArchives.size()]))); } if (!localFiles.isEmpty()) { conf.set(MRJobConfig.CACHE_LOCALFILES, StringUtils.arrayToString(localFiles.toArray(new String[localArchives.size()]))); } setupCalled = true; //If we are 0th worker, signal action complete if (InvocationWorker.getIgWorkerIndex() == 0 && InvocationWorker.getNumberOfWorkers() > 1) { try { InvocationWorker.getSynchronizationBarrier().signalComplete(ACTION_NAME); } catch (Exception e) { throw new RuntimeException(e); } } }
From source file:org.apache.tez.dag.api.DagTypeConverters.java
License:Apache License
public static List<PlanLocalResource> convertToDAGPlan(Map<String, LocalResource> lrs) { List<PlanLocalResource> planLrs = Lists.newArrayListWithCapacity(lrs.size()); for (Entry<String, LocalResource> entry : lrs.entrySet()) { PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder(); String key = entry.getKey(); LocalResource lr = entry.getValue(); localResourcesBuilder.setName(key); localResourcesBuilder.setUri(DagTypeConverters.convertToDAGPlan(lr.getResource())); localResourcesBuilder.setSize(lr.getSize()); localResourcesBuilder.setTimeStamp(lr.getTimestamp()); localResourcesBuilder.setType(DagTypeConverters.convertToDAGPlan(lr.getType())); localResourcesBuilder.setVisibility(DagTypeConverters.convertToDAGPlan(lr.getVisibility())); if (lr.getType() == LocalResourceType.PATTERN) { if (lr.getPattern() == null || lr.getPattern().isEmpty()) { throw new TezUncheckedException( "LocalResource type set to pattern" + " but pattern is null or empty"); }//w w w . ja va2 s . c o m localResourcesBuilder.setPattern(lr.getPattern()); } planLrs.add(localResourcesBuilder.build()); } return planLrs; }
From source file:org.apache.tez.dag.api.DagTypeConverters.java
License:Apache License
public static LocalResourceType convertFromDAGPlan(PlanLocalResourceType type) { switch (type) { case ARCHIVE: return LocalResourceType.ARCHIVE; case FILE://from w ww .ja v a2s . c om return LocalResourceType.FILE; case PATTERN: return LocalResourceType.PATTERN; default: throw new IllegalArgumentException("unknown 'type': " + type); } }
From source file:org.apache.tez.dag.api.DagTypeConverters.java
License:Apache License
public static PlanLocalResource convertLocalResourceToPlanLocalResource(String name, LocalResource lr) { PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder(); localResourcesBuilder.setName(name); localResourcesBuilder.setUri(DagTypeConverters.convertToDAGPlan(lr.getResource())); localResourcesBuilder.setSize(lr.getSize()); localResourcesBuilder.setTimeStamp(lr.getTimestamp()); localResourcesBuilder.setType(DagTypeConverters.convertToDAGPlan(lr.getType())); localResourcesBuilder.setVisibility(DagTypeConverters.convertToDAGPlan(lr.getVisibility())); if (lr.getType() == LocalResourceType.PATTERN) { if (lr.getPattern() == null || lr.getPattern().isEmpty()) { throw new TezUncheckedException( "LocalResource type set to pattern" + " but pattern is null or empty"); }//from www.ja va 2 s. c o m localResourcesBuilder.setPattern(lr.getPattern()); } return localResourcesBuilder.build(); }
From source file:org.apache.tez.dag.app.ContainerContext.java
License:Apache License
private static boolean localResourcesCompatible(Map<String, LocalResource> srcLRs, Map<String, LocalResource> reqLRs) { Map<String, LocalResource> reqLRsCopy = new HashMap<String, LocalResource>(reqLRs); for (Entry<String, LocalResource> srcLREntry : srcLRs.entrySet()) { LocalResource requestedLocalResource = reqLRsCopy.remove(srcLREntry.getKey()); if (requestedLocalResource != null && !srcLREntry.getValue().equals(requestedLocalResource)) { if (LOG.isDebugEnabled()) { LOG.debug("Cannot match container: Attempting to use same target resource name: " + srcLREntry.getKey() + ", but with different source resources. Already localized: " + srcLREntry.getValue() + ", requested: " + requestedLocalResource); }//from www.ja va 2s . c o m return false; } } for (Entry<String, LocalResource> additionalLREntry : reqLRsCopy.entrySet()) { LocalResource lr = additionalLREntry.getValue(); if (EnumSet.of(LocalResourceType.ARCHIVE, LocalResourceType.PATTERN).contains(lr.getType())) { return false; } } return true; }
From source file:org.apache.tez.dag.app.launcher.TezLocalCacheManager.java
License:Apache License
/** * Localize this instance's resources by downloading and symlinking them. * * @throws IOException when an error occurs in download or link *//*from www. ja v a 2 s . c o m*/ public void localize() throws IOException { String absPath = Paths.get(".").toAbsolutePath().normalize().toString(); Path cwd = fileContext.makeQualified(new Path(absPath)); ExecutorService threadPool = null; try { // construct new threads with helpful names ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("TezLocalCacheManager Downloader #%d").build(); threadPool = Executors.newCachedThreadPool(threadFactory); // start all fetches for (Map.Entry<String, LocalResource> entry : resources.entrySet()) { String resourceName = entry.getKey(); LocalResource resource = entry.getValue(); if (resource.getType() == LocalResourceType.PATTERN) { throw new IllegalArgumentException("Resource type PATTERN not supported."); } // linkPath is the path we want to symlink the file/directory into Path linkPath = new Path(cwd, entry.getKey()); if (resourceInfo.containsKey(resource)) { // We've already downloaded this resource and just need to add another link. resourceInfo.get(resource).linkPaths.add(linkPath); } else { // submit task to download the object java.nio.file.Path downloadDir = Files.createTempDirectory(tempDir, resourceName); Path dest = new Path(downloadDir.toAbsolutePath().toString()); FSDownload downloader = new FSDownload(fileContext, ugi, conf, dest, resource); Future<Path> downloadedPath = threadPool.submit(downloader); resourceInfo.put(resource, new ResourceInfo(downloadedPath, linkPath)); } } // Link each file for (Map.Entry<LocalResource, ResourceInfo> entry : resourceInfo.entrySet()) { LocalResource resource = entry.getKey(); ResourceInfo resourceMeta = entry.getValue(); for (Path linkPath : resourceMeta.linkPaths) { Path targetPath; try { // this blocks on the download completing targetPath = resourceMeta.downloadPath.get(); } catch (InterruptedException | ExecutionException e) { throw new IOException(e); } if (createSymlink(targetPath, linkPath)) { LOG.info("Localized file: {} as {}", resource, linkPath); } else { LOG.warn("Failed to create symlink: {} <- {}", targetPath, linkPath); } } } } finally { if (threadPool != null) { threadPool.shutdownNow(); } } }
From source file:org.apache.twill.internal.yarn.YarnUtils.java
License:Apache License
private static YarnLocalResource setLocalResourceType(YarnLocalResource localResource, LocalFile localFile) { if (localFile.isArchive()) { if (localFile.getPattern() == null) { localResource.setType(LocalResourceType.ARCHIVE); } else {/* ww w . j av a 2s .c om*/ localResource.setType(LocalResourceType.PATTERN); localResource.setPattern(localFile.getPattern()); } } else { localResource.setType(LocalResourceType.FILE); } return localResource; }