Example usage for org.apache.hadoop.yarn.api.records LocalResourceType FILE

List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceType FILE

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResourceType
  6. FILE

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceType FILE.

Prototype

LocalResourceType FILE

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceType FILE.

Click Source Link

Document

Regular file i.e.

Usage

From source file:hws.core.ContainerUtils.java

License:Apache License

public static void setupContainerJar(FileSystem fs, Path jarPath, LocalResource containerJar)
        throws IOException {
    FileStatus jarStat = fs.getFileStatus(jarPath);
    containerJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath));
    containerJar.setSize(jarStat.getLen());
    containerJar.setTimestamp(jarStat.getModificationTime());
    containerJar.setType(LocalResourceType.FILE);
    containerJar.setVisibility(LocalResourceVisibility.PUBLIC);
}

From source file:io.amient.yarn1.YarnContainerContext.java

License:Open Source License

private void prepareLocalResourceFile(Map<String, LocalResource> localResources, String fileName,
        String remoteFileName, FileSystem distFs) throws IOException {
    final Path dst = new Path(distFs.getHomeDirectory(), remoteFileName);
    FileStatus scFileStatus = distFs.getFileStatus(dst);
    final URL yarnUrl = ConverterUtils.getYarnUrlFromURI(dst.toUri());
    LocalResource scRsrc = LocalResource.newInstance(yarnUrl, LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileName, scRsrc);
}

From source file:io.dstream.tez.utils.HadoopUtils.java

License:Apache License

/**
 * Creates a single {@link LocalResource} for the provisioned resource identified with {@link Path}
 *
 * @param fs/* w w w.  j a v  a  2s .c o  m*/
 * @param provisionedResourcePath
 * @return
 */
public static LocalResource createLocalResource(FileSystem fs, Path provisionedResourcePath) {
    try {
        FileStatus scFileStatus = fs.getFileStatus(provisionedResourcePath);
        LocalResource localResource = LocalResource.newInstance(
                ConverterUtils.getYarnUrlFromURI(provisionedResourcePath.toUri()), LocalResourceType.FILE,
                LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
        return localResource;
    } catch (Exception e) {
        throw new IllegalStateException(
                "Failed to communicate with FileSystem while creating LocalResource: " + fs, e);
    }
}

From source file:io.hops.tensorflow.ApplicationMaster.java

License:Apache License

/**
 * Parse command line options/*from ww  w  . j  ava  2 s. c om*/
 *
 * @param args
 *     Command line args
 * @return Whether init successful and run should be invoked
 * @throws ParseException
 * @throws IOException
 */
public boolean init(String[] args) throws ParseException, IOException {
    Options opts = createOptions();
    cliParser = new GnuParser().parse(opts, args);

    containerPython = cliParser.getOptionValue(PYTHON, null);

    if (args.length == 0) {
        printUsage(opts);
        throw new IllegalArgumentException("No args specified for application master to initialize");
    }

    //Check whether customer log4j.properties file exists
    if (fileExist(LOG4J_PATH)) {
        try {
            Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, LOG4J_PATH);
        } catch (Exception e) {
            LOG.warn("Can not set up custom log4j properties. " + e);
        }
    }

    if (cliParser.hasOption(HELP)) {
        printUsage(opts);
        return false;
    }

    if (cliParser.hasOption(DEBUG)) {
        dumpOutDebugInfo();
    }

    if (!cliParser.hasOption(MAIN_RELATIVE)) {
        throw new IllegalArgumentException("No main application file specified");
    }
    mainRelative = cliParser.getOptionValue(MAIN_RELATIVE);

    if (cliParser.hasOption(ARGS)) {
        arguments = cliParser.getOptionValues(ARGS);
    }

    Map<String, String> envs = System.getenv();

    if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
        if (cliParser.hasOption(APP_ATTEMPT_ID)) {
            String appIdStr = cliParser.getOptionValue(APP_ATTEMPT_ID, "");
            appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
        } else {
            throw new IllegalArgumentException("Application Attempt Id not set in the environment");
        }
    } else {
        ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name()));
        appAttemptID = containerId.getApplicationAttemptId();
    }

    if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) {
        throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_HOST.name())) {
        throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) {
        throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_PORT.name())) {
        throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment");
    }

    LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId()
            + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId="
            + appAttemptID.getAttemptId());

    if (cliParser.hasOption(ENV)) {
        String shellEnvs[] = cliParser.getOptionValues(ENV);
        for (String env : shellEnvs) {
            env = env.trim();
            int index = env.indexOf('=');
            if (index == -1) {
                environment.put(env, "");
                continue;
            }
            String key = env.substring(0, index);
            String val = "";
            if (index < (env.length() - 1)) {
                val = env.substring(index + 1);
            }
            environment.put(key, val);
        }
    }

    if (cliParser.hasOption(TENSORBOARD)) {
        environment.put("YARNTF_TENSORBOARD", "true");
    }

    if (envs.containsKey(Constants.YARNTFTIMELINEDOMAIN)) {
        domainId = envs.get(Constants.YARNTFTIMELINEDOMAIN);
    }

    containerMemory = Integer.parseInt(cliParser.getOptionValue(MEMORY, "1024"));
    containerVirtualCores = Integer.parseInt(cliParser.getOptionValue(VCORES, "1"));
    containerGPUs = Integer.parseInt(cliParser.getOptionValue(GPUS, "0"));
    tfProtocol = cliParser.getOptionValue(PROTOCOL, null);

    numWorkers = Integer.parseInt(cliParser.getOptionValue(WORKERS, "1"));
    numPses = Integer.parseInt(cliParser.getOptionValue(PSES, "1"));
    numTotalContainers = numWorkers + numPses;
    if (!(numWorkers > 0 && numPses > 0 || numWorkers == 1 && numPses == 0)) {
        throw new IllegalArgumentException("Invalid no. of workers or parameter server");
    }
    // requestPriority = Integer.parseInt(cliParser.getOptionValue(PRIORITY, "0"));

    allocationTimeout = Long.parseLong(cliParser.getOptionValue(ALLOCATION_TIMEOUT, "15")) * 1000;

    environment.put("YARNTF_MEMORY", Integer.toString(containerMemory));
    environment.put("YARNTF_VCORES", Integer.toString(containerVirtualCores));
    environment.put("YARNTF_GPUS", Integer.toString(containerGPUs));
    if (tfProtocol != null) {
        environment.put("YARNTF_PROTOCOL", tfProtocol);
    }
    environment.put("YARNTF_WORKERS", Integer.toString(numWorkers));
    environment.put("YARNTF_PSES", Integer.toString(numPses));
    environment.put("YARNTF_HOME_DIR", FileSystem.get(conf).getHomeDirectory().toString());
    environment.put("PYTHONUNBUFFERED", "true");

    DistributedCacheList distCacheList = null;
    FileInputStream fin = null;
    ObjectInputStream ois = null;
    try {
        fin = new FileInputStream(Constants.DIST_CACHE_PATH);
        ois = new ObjectInputStream(fin);
        try {
            distCacheList = (DistributedCacheList) ois.readObject();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }
    } finally {
        org.apache.commons.io.IOUtils.closeQuietly(ois);
        org.apache.commons.io.IOUtils.closeQuietly(fin);
    }
    LOG.info("Loaded distribute cache list: " + distCacheList.toString());
    for (int i = 0; i < distCacheList.size(); i++) {
        DistributedCacheList.Entry entry = distCacheList.get(i);
        LocalResource distRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(entry.uri),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, entry.size, entry.timestamp);
        localResources.put(entry.relativePath, distRsrc);
    }

    return true;
}

From source file:io.hops.tensorflow.Client.java

License:Apache License

private Map<String, LocalResource> prepareLocalResources(FileSystem fs, ApplicationId appId,
        DistributedCacheList dcl) throws IOException {
    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the local resources
    Map<String, LocalResource> localResources = new HashMap<>();

    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    addResource(fs, appId, amJar, null, Constants.AM_JAR_PATH, null, localResources, null);

    if (!log4jPropFile.isEmpty()) {
        addResource(fs, appId, log4jPropFile, null, Constants.LOG4J_PATH, null, localResources, null);
    }/*from  w  ww  .j  a  va  2  s.  c om*/

    // Write distCacheList to HDFS and add to localResources
    Path baseDir = new Path(fs.getHomeDirectory(), Constants.YARNTF_STAGING + "/" + appId.toString());
    Path dclPath = new Path(baseDir, Constants.DIST_CACHE_PATH);
    FSDataOutputStream ostream = null;
    try {
        ostream = fs.create(dclPath);
        ostream.write(SerializationUtils.serialize(dcl));
    } finally {
        IOUtils.closeQuietly(ostream);
    }
    FileStatus dclStatus = fs.getFileStatus(dclPath);
    LocalResource distCacheResource = LocalResource.newInstance(
            ConverterUtils.getYarnUrlFromURI(dclPath.toUri()), LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, dclStatus.getLen(), dclStatus.getModificationTime());
    localResources.put(Constants.DIST_CACHE_PATH, distCacheResource);

    return localResources;
}

From source file:io.hops.tensorflow.Client.java

License:Apache License

private String addResource(FileSystem fs, ApplicationId appId, String srcPath, String dstDir, String dstName,
        DistributedCacheList distCache, Map<String, LocalResource> localResources, StringBuilder pythonPath)
        throws IOException {
    Path src = new Path(srcPath);

    if (dstDir == null) {
        dstDir = ".";
    }//from  ww w  .  j ava 2 s.  com
    if (dstName == null) {
        dstName = src.getName();
    }

    Path baseDir = new Path(fs.getHomeDirectory(), Constants.YARNTF_STAGING + "/" + appId.toString());
    String dstPath;
    if (dstDir.startsWith(".")) {
        dstPath = dstName;
    } else {
        dstPath = dstDir + "/" + dstName;
    }
    Path dst = new Path(baseDir, dstPath);

    LOG.info("Copying from local filesystem: " + src + " -> " + dst);
    fs.copyFromLocalFile(src, dst);
    FileStatus dstStatus = fs.getFileStatus(dst);

    if (distCache != null) {
        LOG.info("Adding to distributed cache: " + srcPath + " -> " + dstPath);
        distCache.add(new DistributedCacheList.Entry(dstPath, dst.toUri(), dstStatus.getLen(),
                dstStatus.getModificationTime()));
    }

    if (localResources != null) {
        LOG.info("Adding to local environment: " + srcPath + " -> " + dstPath);
        LocalResource resource = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, dstStatus.getLen(),
                dstStatus.getModificationTime());
        localResources.put(dstPath, resource);
    }

    if (pythonPath != null) {
        pythonPath.append(File.pathSeparator).append(dstPath);
    }

    return dstName;
}

From source file:MasteringYarn.DistributedShellClient.java

private void setupJarFileForApplicationMaster(Path jarPath, LocalResource localResource) throws IOException {
    FileStatus jarStat = FileSystem.get(conf).getFileStatus(jarPath);
    localResource.setResource(ConverterUtils.getYarnUrlFromPath(jarPath));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.PUBLIC);
}

From source file:ml.shifu.guagua.yarn.util.YarnUtils.java

License:Apache License

/**
 * Boilerplate to add a file to the local resources..
 * /*from ww w.j  a  v a  2 s . co  m*/
 * @param localResources
 *            the LocalResources map to populate.
 * @param fs
 *            handle to the HDFS file system.
 * @param target
 *            the file to send to the remote container.
 */
public static void addFileToResourceMap(Map<String, LocalResource> localResources, FileSystem fs, Path target)
        throws IOException {
    LocalResource resource = Records.newRecord(LocalResource.class);
    FileStatus destStatus = fs.getFileStatus(target);
    resource.setResource(ConverterUtils.getYarnUrlFromURI(target.toUri()));
    resource.setSize(destStatus.getLen());
    resource.setTimestamp(destStatus.getModificationTime());
    resource.setType(LocalResourceType.FILE); // use FILE, even for jars!
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResources.put(target.getName(), resource);
    LOG.info("Registered file in LocalResources :{} ", target);
}

From source file:org.apache.asterix.aoya.AsterixApplicationMaster.java

License:Apache License

/**
 * Here I am just pointing the Containers to the exisiting HDFS resources given by the Client
 * filesystem of the nodes.//from  w w w  .ja va 2 s.  co m
 *
 * @throws IOException
 */
private void localizeDFSResources() throws IOException {
    //if performing an 'offline' task, skip a lot of resource distribution
    if (obliterate || backup || restore) {
        if (appMasterJar == null || ("").equals(appMasterJar)) {
            //this can happen in a jUnit testing environment. we don't need to set it there.
            if (!conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
                throw new IllegalStateException("AM jar not provided in environment.");
            } else {
                return;
            }
        }
        FileSystem fs = FileSystem.get(conf);
        FileStatus appMasterJarStatus = fs.getFileStatus(appMasterJar);
        LocalResource obliteratorJar = Records.newRecord(LocalResource.class);
        obliteratorJar.setType(LocalResourceType.FILE);
        obliteratorJar.setVisibility(LocalResourceVisibility.PRIVATE);
        obliteratorJar.setResource(ConverterUtils.getYarnUrlFromPath(appMasterJar));
        obliteratorJar.setTimestamp(appMasterJarStatus.getModificationTime());
        obliteratorJar.setSize(appMasterJarStatus.getLen());
        localResources.put("asterix-yarn.jar", obliteratorJar);
        LOG.info(localResources.values());
        return;
    }
    //otherwise, distribute evertything to start up asterix

    LocalResource asterixZip = Records.newRecord(LocalResource.class);

    //this un-tar's the asterix distribution
    asterixZip.setType(LocalResourceType.ARCHIVE);

    asterixZip.setVisibility(LocalResourceVisibility.PRIVATE);
    try {
        asterixZip.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixZipPath)));

    } catch (URISyntaxException e) {
        LOG.error("Error locating Asterix zip" + " in env, path=" + asterixZipPath);
        throw new IOException(e);
    }

    asterixZip.setTimestamp(asterixZipTimestamp);
    asterixZip.setSize(asterixZipLen);
    localResources.put(ASTERIX_ZIP_NAME, asterixZip);

    //now let's do the same for the cluster description XML
    LocalResource asterixConf = Records.newRecord(LocalResource.class);
    asterixConf.setType(LocalResourceType.FILE);

    asterixConf.setVisibility(LocalResourceVisibility.PRIVATE);
    try {
        asterixConf.setResource(ConverterUtils.getYarnUrlFromURI(new URI(asterixConfPath)));
    } catch (URISyntaxException e) {
        LOG.error("Error locating Asterix config" + " in env, path=" + asterixConfPath);
        throw new IOException(e);
    }
    //TODO: I could avoid localizing this everywhere by only calling this block on the metadata node.
    asterixConf.setTimestamp(asterixConfTimestamp);
    asterixConf.setSize(asterixConfLen);
    localResources.put("cluster-config.xml", asterixConf);
    //now add the libraries if there are any
    try {
        FileSystem fs = FileSystem.get(conf);
        Path p = new Path(dfsBasePath, instanceConfPath + File.separator + "library" + Path.SEPARATOR);
        if (fs.exists(p)) {
            FileStatus[] dataverses = fs.listStatus(p);
            for (FileStatus d : dataverses) {
                if (!d.isDirectory())
                    throw new IOException("Library configuration directory structure is incorrect");
                FileStatus[] libraries = fs.listStatus(d.getPath());
                for (FileStatus l : libraries) {
                    if (l.isDirectory())
                        throw new IOException("Library configuration directory structure is incorrect");
                    LocalResource lr = Records.newRecord(LocalResource.class);
                    lr.setResource(ConverterUtils.getYarnUrlFromURI(l.getPath().toUri()));
                    lr.setSize(l.getLen());
                    lr.setTimestamp(l.getModificationTime());
                    lr.setType(LocalResourceType.ARCHIVE);
                    lr.setVisibility(LocalResourceVisibility.PRIVATE);
                    localResources.put("library" + Path.SEPARATOR + d.getPath().getName() + Path.SEPARATOR
                            + l.getPath().getName().split("\\.")[0], lr);
                    LOG.info("Found library: " + l.getPath().toString());
                    LOG.info(l.getPath().getName());
                }
            }
        }
    } catch (FileNotFoundException e) {
        LOG.info("No external libraries present");
        //do nothing, it just means there aren't libraries. that is possible and ok
        // it should be handled by the fs.exists(p) check though.
    }
    LOG.info(localResources.values());

}

From source file:org.apache.flink.tez.client.TezExecutor.java

License:Apache License

private static void addLocalResource(TezConfiguration tezConf, Path jarPath, DAG dag) {

    try {//from w w w. ja  va  2  s. c o  m
        org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(tezConf);

        LOG.info("Jar path received is " + jarPath.toString());

        String jarFile = jarPath.getName();

        Path remoteJarPath = null;

        /*
        if (tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR) == null) {
           LOG.info("Tez staging directory is null, setting it.");
           Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
           LOG.info("Setting Tez staging directory to " + stagingDir.toString());
           tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
           LOG.info("Set Tez staging directory to " + stagingDir.toString());
        }
        Path stagingDir = new Path(tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR));
        LOG.info("Ensuring that Tez staging directory exists");
        TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
        LOG.info("Tez staging directory exists and is " + stagingDir.toString());
        */

        Path stagingDir = TezCommonUtils.getTezBaseStagingPath(tezConf);
        LOG.info("Tez staging path is " + stagingDir);
        TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
        LOG.info("Tez staging dir exists");

        remoteJarPath = fs.makeQualified(new Path(stagingDir, jarFile));
        LOG.info("Copying " + jarPath.toString() + " to " + remoteJarPath.toString());
        fs.copyFromLocalFile(jarPath, remoteJarPath);

        FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
        Credentials credentials = new Credentials();
        TokenCache.obtainTokensForNamenodes(credentials, new Path[] { remoteJarPath }, tezConf);

        Map<String, LocalResource> localResources = new TreeMap<String, LocalResource>();
        LocalResource jobJar = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(remoteJarPath),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, remoteJarStatus.getLen(),
                remoteJarStatus.getModificationTime());
        localResources.put(jarFile.toString(), jobJar);

        dag.addTaskLocalFiles(localResources);

        LOG.info("Added job jar as local resource.");
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        System.exit(-1);
    }
}