List of usage examples for org.apache.hadoop.yarn.util ConverterUtils getPathFromYarnURL
@Public @Deprecated public static Path getPathFromYarnURL(URL url) throws URISyntaxException
From source file:com.continuuity.weave.yarn.YarnWeavePreparer.java
License:Open Source License
private void saveLocalFiles(Map<String, LocalResource> localResources, Set<String> keys) throws IOException { Map<String, LocalFile> localFiles = Maps.transformEntries( Maps.filterKeys(localResources, Predicates.in(keys)), new Maps.EntryTransformer<String, LocalResource, LocalFile>() { @Override/*from w w w.j a v a2s. co m*/ public LocalFile transformEntry(String key, LocalResource value) { try { return new DefaultLocalFile(key, ConverterUtils.getPathFromYarnURL(value.getResource()).toUri(), value.getTimestamp(), value.getSize(), value.getType() != LocalResourceType.FILE, value.getPattern()); } catch (URISyntaxException e) { throw Throwables.propagate(e); } } }); LOG.debug("Create and copy localFiles.json"); Location location = createTempLocation("localFiles", ".json"); Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8); try { new GsonBuilder().registerTypeAdapter(LocalFile.class, new LocalFileCodec()).create() .toJson(localFiles.values(), new TypeToken<List<LocalFile>>() { }.getType(), writer); } finally { writer.close(); } LOG.debug("Done localFiles.json"); localResources.put("localFiles.json", YarnUtils.createLocalResource(location)); }
From source file:com.scaleoutsoftware.soss.hserver.hadoop.DistributedCacheManager.java
License:Apache License
/** * Set up the distributed cache by localizing the resources, and updating * the configuration with references to the localized resources. * @param conf job configuration/* www. j av a 2s .c o m*/ * @throws IOException */ public void setup(Configuration conf) throws IOException { //If we are not 0th worker, wait for 0th worker to set up the cache if (InvocationWorker.getIgWorkerIndex() > 0 && InvocationWorker.getNumberOfWorkers() > 1) { try { InvocationWorker.getSynchronizationBarrier().waitForComplete(ACTION_NAME, SYNCHRONIZATION_WAIT_MS, WAIT_GRANULARITY_MS); } catch (Exception e) { throw new RuntimeException(e); } return; } File workDir = new File(System.getProperty("user.dir")); // Generate YARN local resources objects corresponding to the distributed // cache configuration Map<String, LocalResource> localResources = new LinkedHashMap<String, LocalResource>(); MRApps.setupDistributedCache(conf, localResources); //CODE CHANGE FROM ORIGINAL FILE: //We need to clear the resources from jar files, since they are distributed through the IG. // Iterator<Map.Entry<String, LocalResource>> iterator = localResources.entrySet().iterator(); while (iterator.hasNext()) { Entry<String, LocalResource> entry = iterator.next(); if (entry.getKey().endsWith(".jar")) { iterator.remove(); } } // Generating unique numbers for FSDownload. AtomicLong uniqueNumberGenerator = new AtomicLong(System.currentTimeMillis()); // Find which resources are to be put on the local classpath Map<String, Path> classpaths = new HashMap<String, Path>(); Path[] archiveClassPaths = DistributedCache.getArchiveClassPaths(conf); if (archiveClassPaths != null) { for (Path p : archiveClassPaths) { FileSystem remoteFS = p.getFileSystem(conf); p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory())); classpaths.put(p.toUri().getPath().toString(), p); } } Path[] fileClassPaths = DistributedCache.getFileClassPaths(conf); if (fileClassPaths != null) { for (Path p : fileClassPaths) { FileSystem remoteFS = p.getFileSystem(conf); p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory())); classpaths.put(p.toUri().getPath().toString(), p); } } // Localize the resources LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LOCAL_DIR); FileContext localFSFileContext = FileContext.getLocalFSFileContext(); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ExecutorService exec = null; try { ThreadFactory tf = new ThreadFactoryBuilder() .setNameFormat("LocalDistributedCacheManager Downloader #%d").build(); exec = Executors.newCachedThreadPool(tf); Path destPath = localDirAllocator.getLocalPathForWrite(".", conf); Map<LocalResource, Future<Path>> resourcesToPaths = Maps.newHashMap(); for (LocalResource resource : localResources.values()) { Callable<Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet())), resource); Future<Path> future = exec.submit(download); resourcesToPaths.put(resource, future); } for (Entry<String, LocalResource> entry : localResources.entrySet()) { LocalResource resource = entry.getValue(); Path path; try { path = resourcesToPaths.get(resource).get(); } catch (InterruptedException e) { throw new IOException(e); } catch (ExecutionException e) { throw new IOException(e); } String pathString = path.toUri().toString(); String link = entry.getKey(); String target = new File(path.toUri()).getPath(); symlink(workDir, target, link); if (resource.getType() == LocalResourceType.ARCHIVE) { localArchives.add(pathString); } else if (resource.getType() == LocalResourceType.FILE) { localFiles.add(pathString); } else if (resource.getType() == LocalResourceType.PATTERN) { //PATTERN is not currently used in local mode throw new IllegalArgumentException( "Resource type PATTERN is not " + "implemented yet. " + resource.getResource()); } Path resourcePath; try { resourcePath = ConverterUtils.getPathFromYarnURL(resource.getResource()); } catch (URISyntaxException e) { throw new IOException(e); } LOG.info(String.format("Localized %s as %s", resourcePath, path)); String cp = resourcePath.toUri().getPath(); if (classpaths.keySet().contains(cp)) { localClasspaths.add(path.toUri().getPath().toString()); } } } finally { if (exec != null) { exec.shutdown(); } } // Update the configuration object with localized data. if (!localArchives.isEmpty()) { conf.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils.arrayToString(localArchives.toArray(new String[localArchives.size()]))); } if (!localFiles.isEmpty()) { conf.set(MRJobConfig.CACHE_LOCALFILES, StringUtils.arrayToString(localFiles.toArray(new String[localArchives.size()]))); } setupCalled = true; //If we are 0th worker, signal action complete if (InvocationWorker.getIgWorkerIndex() == 0 && InvocationWorker.getNumberOfWorkers() > 1) { try { InvocationWorker.getSynchronizationBarrier().signalComplete(ACTION_NAME); } catch (Exception e) { throw new RuntimeException(e); } } }
From source file:org.apache.flink.yarn.YarnFileStageTest.java
License:Apache License
/** * Verifies that nested directories are properly copied with the given filesystem and paths. * * @param targetFileSystem/*from w ww.j a v a 2 s . c o m*/ * file system of the target path * @param targetDir * target path (URI like <tt>hdfs://...</tt>) * @param tempFolder * JUnit temporary folder rule to create the source directory with * @param addSchemeToLocalPath * whether add the <tt>file://</tt> scheme to the local path to copy from */ static void testCopyFromLocalRecursive(FileSystem targetFileSystem, Path targetDir, TemporaryFolder tempFolder, boolean addSchemeToLocalPath) throws Exception { // directory must not yet exist assertFalse(targetFileSystem.exists(targetDir)); final File srcDir = tempFolder.newFolder(); final Path srcPath; if (addSchemeToLocalPath) { srcPath = new Path("file://" + srcDir.getAbsolutePath()); } else { srcPath = new Path(srcDir.getAbsolutePath()); } HashMap<String /* (relative) path */, /* contents */ String> srcFiles = new HashMap<>(4); // create and fill source files srcFiles.put("1", "Hello 1"); srcFiles.put("2", "Hello 2"); srcFiles.put("nested/3", "Hello nested/3"); srcFiles.put("nested/4/5", "Hello nested/4/5"); for (Map.Entry<String, String> src : srcFiles.entrySet()) { File file = new File(srcDir, src.getKey()); //noinspection ResultOfMethodCallIgnored file.getParentFile().mkdirs(); try (DataOutputStream out = new DataOutputStream(new FileOutputStream(file))) { out.writeUTF(src.getValue()); } } // copy the created directory recursively: try { List<Path> remotePaths = new ArrayList<>(); HashMap<String, LocalResource> localResources = new HashMap<>(); AbstractYarnClusterDescriptor.uploadAndRegisterFiles( Collections.singletonList(new File(srcPath.toUri().getPath())), targetFileSystem, targetDir, ApplicationId.newInstance(0, 0), remotePaths, localResources, new StringBuilder()); assertEquals(srcFiles.size(), localResources.size()); Path workDir = ConverterUtils .getPathFromYarnURL(localResources.get(srcPath.getName() + "/1").getResource()).getParent(); RemoteIterator<LocatedFileStatus> targetFilesIterator = targetFileSystem.listFiles(workDir, true); HashMap<String /* (relative) path */, /* contents */ String> targetFiles = new HashMap<>(4); final int workDirPrefixLength = workDir.toString().length() + 1; // one more for the concluding "/" while (targetFilesIterator.hasNext()) { LocatedFileStatus targetFile = targetFilesIterator.next(); int retries = 5; do { try (FSDataInputStream in = targetFileSystem.open(targetFile.getPath())) { String absolutePathString = targetFile.getPath().toString(); String relativePath = absolutePathString.substring(workDirPrefixLength); targetFiles.put(relativePath, in.readUTF()); assertEquals("extraneous data in file " + relativePath, -1, in.read()); break; } catch (FileNotFoundException e) { // For S3, read-after-write may be eventually consistent, i.e. when trying // to access the object before writing it; see // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel // -> try again a bit later Thread.sleep(50); } } while ((retries--) > 0); } assertThat(targetFiles, equalTo(srcFiles)); } finally { // clean up targetFileSystem.delete(targetDir, true); } }
From source file:org.apache.hama.bsp.BSPTaskLauncher.java
License:Apache License
private GetContainerStatusesRequest setupContainer(Container allocatedContainer, ContainerManagementProtocol cm, String user, int id) throws IOException, YarnException { LOG.info("Setting up a container for user " + user + " with id of " + id + " and containerID of " + allocatedContainer.getId() + " as " + user); // Now we setup a ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); // Set the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LocalResource packageResource = Records.newRecord(LocalResource.class); FileSystem fs = FileSystem.get(conf); Path packageFile = new Path(System.getenv(YARNBSPConstants.HAMA_YARN_LOCATION)); URL packageUrl = ConverterUtils .getYarnUrlFromPath(packageFile.makeQualified(fs.getUri(), fs.getWorkingDirectory())); LOG.info("PackageURL has been composed to " + packageUrl.toString()); try {/* w w w. j a v a2s . co m*/ LOG.info("Reverting packageURL to path: " + ConverterUtils.getPathFromYarnURL(packageUrl)); } catch (URISyntaxException e) { LOG.fatal("If you see this error the workarround does not work", e); } packageResource.setResource(packageUrl); packageResource.setSize(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_YARN_SIZE))); packageResource.setTimestamp(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_YARN_TIMESTAMP))); packageResource.setType(LocalResourceType.FILE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); localResources.put(YARNBSPConstants.APP_MASTER_JAR_PATH, packageResource); Path hamaReleaseFile = new Path(System.getenv(YARNBSPConstants.HAMA_RELEASE_LOCATION)); URL hamaReleaseUrl = ConverterUtils .getYarnUrlFromPath(hamaReleaseFile.makeQualified(fs.getUri(), fs.getWorkingDirectory())); LOG.info("Hama release URL has been composed to " + hamaReleaseUrl.toString()); LocalResource hamaReleaseRsrc = Records.newRecord(LocalResource.class); hamaReleaseRsrc.setResource(hamaReleaseUrl); hamaReleaseRsrc.setSize(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_RELEASE_SIZE))); hamaReleaseRsrc.setTimestamp(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_RELEASE_TIMESTAMP))); hamaReleaseRsrc.setType(LocalResourceType.ARCHIVE); hamaReleaseRsrc.setVisibility(LocalResourceVisibility.APPLICATION); localResources.put(YARNBSPConstants.HAMA_SYMLINK, hamaReleaseRsrc); ctx.setLocalResources(localResources); /* * TODO Package classpath seems not to work if you're in pseudo distributed * mode, because the resource must not be moved, it will never be unpacked. * So we will check if our jar file has the file:// prefix and put it into * the CP directly */ StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$()) .append(File.pathSeparatorChar).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparator); classPathEnv .append("./" + YARNBSPConstants.HAMA_SYMLINK + "/" + YARNBSPConstants.HAMA_RELEASE_VERSION + "/*"); classPathEnv.append(File.pathSeparator); classPathEnv.append( "./" + YARNBSPConstants.HAMA_SYMLINK + "/" + YARNBSPConstants.HAMA_RELEASE_VERSION + "/lib/*"); Vector<CharSequence> vargs = new Vector<CharSequence>(); vargs.add("${JAVA_HOME}/bin/java"); vargs.add("-cp " + classPathEnv + ""); vargs.add(BSPRunner.class.getCanonicalName()); vargs.add(jobId.getJtIdentifier()); vargs.add(Integer.toString(id)); vargs.add(this.jobFile.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/bsp.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/bsp.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } List<String> commands = new ArrayList<String>(); commands.add(command.toString()); ctx.setCommands(commands); LOG.info("Starting command: " + commands); StartContainerRequest startReq = Records.newRecord(StartContainerRequest.class); startReq.setContainerLaunchContext(ctx); startReq.setContainerToken(allocatedContainer.getContainerToken()); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(startReq); StartContainersRequest requestList = StartContainersRequest.newInstance(list); cm.startContainers(requestList); GetContainerStatusesRequest statusReq = Records.newRecord(GetContainerStatusesRequest.class); List<ContainerId> containerIds = new ArrayList<ContainerId>(); containerIds.add(allocatedContainer.getId()); statusReq.setContainerIds(containerIds); return statusReq; }
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
/** * Obtains tokens for the DAG based on the list of URIs setup in the DAG. The * fetched credentials are populated back into the DAG and can be retrieved * via dag.getCredentials//from w w w . j a va2s .c om * * @param dag * the dag for which credentials need to be setup * @param sessionCredentials * session credentials which have already been obtained, and will be * required for the DAG * @param conf * @throws IOException */ @Private static Credentials setupDAGCredentials(DAG dag, Credentials sessionCredentials, Configuration conf) throws IOException { Preconditions.checkNotNull(sessionCredentials); TezCommonUtils.logCredentials(LOG, sessionCredentials, "session"); Credentials dagCredentials = new Credentials(); // All session creds are required for the DAG. dagCredentials.mergeAll(sessionCredentials); // Add additional credentials based on any URIs that the user may have specified. // Obtain Credentials for any paths that the user may have configured. addFileSystemCredentialsFromURIs(dag.getURIsForCredentials(), dagCredentials, conf); // Obtain Credentials for the local resources configured on the DAG try { Set<Path> lrPaths = new HashSet<Path>(); for (Vertex v : dag.getVertices()) { for (LocalResource lr : v.getTaskLocalFiles().values()) { lrPaths.add(ConverterUtils.getPathFromYarnURL(lr.getResource())); } List<DataSourceDescriptor> dataSources = v.getDataSources(); for (DataSourceDescriptor dataSource : dataSources) { addFileSystemCredentialsFromURIs(dataSource.getURIsForCredentials(), dagCredentials, conf); } List<DataSinkDescriptor> dataSinks = v.getDataSinks(); for (DataSinkDescriptor dataSink : dataSinks) { addFileSystemCredentialsFromURIs(dataSink.getURIsForCredentials(), dagCredentials, conf); } } for (LocalResource lr : dag.getTaskLocalFiles().values()) { lrPaths.add(ConverterUtils.getPathFromYarnURL(lr.getResource())); } Path[] paths = lrPaths.toArray(new Path[lrPaths.size()]); TokenCache.obtainTokensForFileSystems(dagCredentials, paths, conf); } catch (URISyntaxException e) { throw new IOException(e); } return dagCredentials; }
From source file:yarnkit.utils.YarnUtils.java
License:Apache License
@Nonnull public static Map<String, String> convertResourceMapToProperties(@Nonnull Map<String, LocalResource> mapping) { if (mapping.isEmpty()) { return Collections.emptyMap(); }//from w w w.j a v a2 s . co m final Map<String, String> converted = new HashMap<String, String>(mapping.size()); for (Map.Entry<String, LocalResource> e : mapping.entrySet()) { String name = e.getKey(); LocalResource resource = e.getValue(); org.apache.hadoop.yarn.api.records.URL url = resource.getResource(); final Path path; try { path = ConverterUtils.getPathFromYarnURL(url); } catch (URISyntaxException ex) { throw new IllegalStateException("Failed to convert YARN URL to Path: " + url, ex); } String value = path.toString(); converted.put(name, value); } return converted; }