Example usage for org.apache.hadoop.yarn.api.records LocalResource newInstance

List of usage examples for org.apache.hadoop.yarn.api.records LocalResource newInstance

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResource
  6. newInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResource newInstance.

Prototype

@Public
    @Stable
    public static LocalResource newInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility,
            long size, long timestamp)

Source Link

Usage

From source file:io.hops.tensorflow.Client.java

License:Apache License

private Map<String, LocalResource> prepareLocalResources(FileSystem fs, ApplicationId appId,
        DistributedCacheList dcl) throws IOException {
    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the local resources
    Map<String, LocalResource> localResources = new HashMap<>();

    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    addResource(fs, appId, amJar, null, Constants.AM_JAR_PATH, null, localResources, null);

    if (!log4jPropFile.isEmpty()) {
        addResource(fs, appId, log4jPropFile, null, Constants.LOG4J_PATH, null, localResources, null);
    }/* w  w  w.j av a 2s .co  m*/

    // Write distCacheList to HDFS and add to localResources
    Path baseDir = new Path(fs.getHomeDirectory(), Constants.YARNTF_STAGING + "/" + appId.toString());
    Path dclPath = new Path(baseDir, Constants.DIST_CACHE_PATH);
    FSDataOutputStream ostream = null;
    try {
        ostream = fs.create(dclPath);
        ostream.write(SerializationUtils.serialize(dcl));
    } finally {
        IOUtils.closeQuietly(ostream);
    }
    FileStatus dclStatus = fs.getFileStatus(dclPath);
    LocalResource distCacheResource = LocalResource.newInstance(
            ConverterUtils.getYarnUrlFromURI(dclPath.toUri()), LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, dclStatus.getLen(), dclStatus.getModificationTime());
    localResources.put(Constants.DIST_CACHE_PATH, distCacheResource);

    return localResources;
}

From source file:io.hops.tensorflow.Client.java

License:Apache License

private String addResource(FileSystem fs, ApplicationId appId, String srcPath, String dstDir, String dstName,
        DistributedCacheList distCache, Map<String, LocalResource> localResources, StringBuilder pythonPath)
        throws IOException {
    Path src = new Path(srcPath);

    if (dstDir == null) {
        dstDir = ".";
    }/*w w w  . j av  a 2 s. c  om*/
    if (dstName == null) {
        dstName = src.getName();
    }

    Path baseDir = new Path(fs.getHomeDirectory(), Constants.YARNTF_STAGING + "/" + appId.toString());
    String dstPath;
    if (dstDir.startsWith(".")) {
        dstPath = dstName;
    } else {
        dstPath = dstDir + "/" + dstName;
    }
    Path dst = new Path(baseDir, dstPath);

    LOG.info("Copying from local filesystem: " + src + " -> " + dst);
    fs.copyFromLocalFile(src, dst);
    FileStatus dstStatus = fs.getFileStatus(dst);

    if (distCache != null) {
        LOG.info("Adding to distributed cache: " + srcPath + " -> " + dstPath);
        distCache.add(new DistributedCacheList.Entry(dstPath, dst.toUri(), dstStatus.getLen(),
                dstStatus.getModificationTime()));
    }

    if (localResources != null) {
        LOG.info("Adding to local environment: " + srcPath + " -> " + dstPath);
        LocalResource resource = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, dstStatus.getLen(),
                dstStatus.getModificationTime());
        localResources.put(dstPath, resource);
    }

    if (pythonPath != null) {
        pythonPath.append(File.pathSeparator).append(dstPath);
    }

    return dstName;
}

From source file:org.apache.drill.yarn.core.DfsFacade.java

License:Apache License

/**
 * Create a local resource definition for YARN. A local resource is one that
 * must be localized onto the remote node prior to running a command on that
 * node.//from   w w w . j  av a2 s.c  o m
 * <p>
 * YARN uses the size and timestamp are used to check if the file has changed
 * on HDFS to check if YARN can use an existing copy, if any.
 * <p>
 * Resources are made public.
 *
 * @param conf
 *          Configuration created from the Hadoop config files, in this case,
 *          identifies the target file system.
 * @param resourcePath
 *          the path (relative or absolute) to the file on the configured file
 *          system (usually HDFS).
 * @return a YARN local resource records that contains information about path,
 *         size, type, resource and so on that YARN requires.
 * @throws IOException
 *           if the resource does not exist on the configured file system
 */

public LocalResource makeResource(Path dfsPath, FileStatus dfsFileStatus, LocalResourceType type,
        LocalResourceVisibility visibility) throws DfsFacadeException {
    URL destUrl;
    try {
        destUrl = ConverterUtils.getYarnUrlFromPath(FileContext.getFileContext().makeQualified(dfsPath));
    } catch (UnsupportedFileSystemException e) {
        throw new DfsFacadeException("Unable to convert dfs file to a URL: " + dfsPath.toString(), e);
    }
    LocalResource resource = LocalResource.newInstance(destUrl, type, visibility, dfsFileStatus.getLen(),
            dfsFileStatus.getModificationTime());
    return resource;
}

From source file:org.apache.flink.tez.client.TezExecutor.java

License:Apache License

private static void addLocalResource(TezConfiguration tezConf, Path jarPath, DAG dag) {

    try {/* w  w  w.ja v  a  2  s.co m*/
        org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(tezConf);

        LOG.info("Jar path received is " + jarPath.toString());

        String jarFile = jarPath.getName();

        Path remoteJarPath = null;

        /*
        if (tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR) == null) {
           LOG.info("Tez staging directory is null, setting it.");
           Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
           LOG.info("Setting Tez staging directory to " + stagingDir.toString());
           tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
           LOG.info("Set Tez staging directory to " + stagingDir.toString());
        }
        Path stagingDir = new Path(tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR));
        LOG.info("Ensuring that Tez staging directory exists");
        TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
        LOG.info("Tez staging directory exists and is " + stagingDir.toString());
        */

        Path stagingDir = TezCommonUtils.getTezBaseStagingPath(tezConf);
        LOG.info("Tez staging path is " + stagingDir);
        TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
        LOG.info("Tez staging dir exists");

        remoteJarPath = fs.makeQualified(new Path(stagingDir, jarFile));
        LOG.info("Copying " + jarPath.toString() + " to " + remoteJarPath.toString());
        fs.copyFromLocalFile(jarPath, remoteJarPath);

        FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
        Credentials credentials = new Credentials();
        TokenCache.obtainTokensForNamenodes(credentials, new Path[] { remoteJarPath }, tezConf);

        Map<String, LocalResource> localResources = new TreeMap<String, LocalResource>();
        LocalResource jobJar = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(remoteJarPath),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, remoteJarStatus.getLen(),
                remoteJarStatus.getModificationTime());
        localResources.put(jarFile.toString(), jobJar);

        dag.addTaskLocalFiles(localResources);

        LOG.info("Added job jar as local resource.");
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        System.exit(-1);
    }
}

From source file:org.apache.hama.bsp.YARNBSPJobClient.java

License:Apache License

private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String fileName,
        Map<String, LocalResource> localResources) throws IOException {
    Path dstPath = new Path(fileDstPath, fileName);
    dstPath = fs.makeQualified(dstPath);
    fs.copyFromLocalFile(false, true, new Path(fileSrcPath), dstPath);
    FileStatus fileStatus = fs.getFileStatus(dstPath);
    LocalResource localRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dstPath.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, fileStatus.getLen(),
            fileStatus.getModificationTime());
    localResources.put(fileName, localRsrc);
}

From source file:org.apache.metron.maas.service.callback.LaunchContainer.java

License:Apache License

private Map.Entry<String, LocalResource> localizeResource(FileStatus status) {
    URL url = ConverterUtils.getYarnUrlFromURI(status.getPath().toUri());
    LocalResource resource = LocalResource.newInstance(url, LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, status.getLen(), status.getModificationTime());
    String name = status.getPath().getName();
    return new AbstractMap.SimpleEntry<>(name, resource);
}

From source file:org.apache.metron.maas.service.Client.java

License:Apache License

private Path addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {/* w  w w.jav  a  2s.  c o  m*/
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
    return dst;
}

From source file:org.apache.pig.backend.hadoop.executionengine.tez.TezResourceManager.java

License:Apache License

public Map<String, LocalResource> getTezResources(Set<String> resourceNames) throws Exception {
    Map<String, LocalResource> tezResources = new HashMap<String, LocalResource>();
    for (String resourceName : resourceNames) {
        // The resource name will be symlinked to the resource path in the
        // container's working directory.
        Path resourcePath = resources.get(resourceName);
        FileStatus fstat = remoteFs.getFileStatus(resourcePath);

        LocalResource tezResource = LocalResource.newInstance(
                ConverterUtils.getYarnUrlFromPath(fstat.getPath()), LocalResourceType.FILE,
                LocalResourceVisibility.APPLICATION, fstat.getLen(), fstat.getModificationTime());

        tezResources.put(resourceName, tezResource);
    }//from www  .ja  v a 2 s .  c  o  m
    return tezResources;
}

From source file:org.apache.tajo.yarn.command.LaunchCommand.java

License:Apache License

private void setupLocalResources(ContainerLaunchContext amContainer, FileSystem fs, ApplicationId appId)
        throws IOException {
    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path

    String appMasterJar = findContainingJar(ApplicationMaster.class);
    addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(), localResources,
            LocalResourceType.FILE);/*  www .j  a  v  a2s .c o  m*/

    addToLocalResources(fs, libDir, libDir, appId.getId(), localResources, LocalResourceType.FILE);

    addToLocalResources(fs, tajoArchive, "tajo", appId.getId(), localResources, LocalResourceType.ARCHIVE);

    // Set the log4j properties if needed
    if (!log4jPropFile.isEmpty()) {
        addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(), localResources,
                LocalResourceType.FILE);
    }

    //    addToLocalResources(fs, confDir, "conf", appId.getId(),
    //        localResources, LocalResourceType.FILE);

    // Tajo master conf
    Configuration tajoMasterConf = new Configuration(false);
    tajoMasterConf.addResource(new Path(confDir, "tajo-site.xml"));
    String suffix = appName + "/" + appId.getId() + "/master-conf";
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    fs.mkdirs(dst);
    Path confFile = new Path(dst, "tajo-site.xml");
    FSDataOutputStream fdos = fs.create(confFile);
    tajoMasterConf.writeXml(fdos);
    fdos.close();
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put("conf", scRsrc);

    amContainer.setLocalResources(localResources);
}

From source file:org.apache.tajo.yarn.command.LaunchCommand.java

License:Apache License

/**
 * @return Destinate n//from  w  ww  . j a va 2  s  . c  o m
 */
private Path addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, int appId,
        Map<String, LocalResource> localResources, LocalResourceType type) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileSrcPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), type,
            LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
    return dst;
}