Example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION

List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResourceVisibility
  6. APPLICATION

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION.

Prototype

LocalResourceVisibility APPLICATION

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION.

Click Source Link

Document

Shared only among containers of the same application on the node.

Usage

From source file:edu.cmu.graphchi.toolkits.collaborative_filtering.yarn.ApplicationMaster.java

License:Apache License

public static LocalResource addLocalResource(String filePath) throws Exception {
    File file = new File(filePath);
    String fullFilePath = file.getAbsolutePath();

    FileSystem fs = FileSystem.get(IO.getConf());
    Path src = new Path(fullFilePath);
    String pathSuffix = "local-tmp/" + file.getName();
    Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
    fs.copyFromLocalFile(false, true, src, dst);
    FileStatus destStatus = fs.getFileStatus(dst);
    LocalResource resource = Records.newRecord(LocalResource.class);

    resource.setType(LocalResourceType.FILE);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    resource.setTimestamp(destStatus.getModificationTime());
    resource.setSize(destStatus.getLen());

    return resource;
}

From source file:edu.uci.ics.hyracks.yarn.common.resources.LocalResourceHelper.java

License:Apache License

private static LocalResource createLocalResourceFromPath(Configuration config, File path) throws IOException {
    LocalResource lr = Records.newRecord(LocalResource.class);
    URL url = ConverterUtils/* w ww.  j  a  v a  2s .  co m*/
            .getYarnUrlFromPath(FileContext.getFileContext().makeQualified(new Path(path.toURI())));
    lr.setResource(url);
    lr.setVisibility(LocalResourceVisibility.APPLICATION);
    lr.setTimestamp(path.lastModified());
    lr.setSize(path.length());
    return lr;
}

From source file:gobblin.yarn.YarnHelixUtils.java

License:Apache License

/**
 * Add a file as a Yarn {@link org.apache.hadoop.yarn.api.records.LocalResource}.
 *
 * @param fs a {@link FileSystem} instance
 * @param destFilePath the destination file path
 * @param resourceType the {@link org.apache.hadoop.yarn.api.records.LocalResourceType} of the file
 * @param resourceMap a {@link Map} of file names to their corresponding
 *                    {@link org.apache.hadoop.yarn.api.records.LocalResource}s
 * @throws IOException if there's something wrong adding the file as a
 *                     {@link org.apache.hadoop.yarn.api.records.LocalResource}
 *///  w  ww  . j  a  va 2  s .  c  o  m
public static void addFileAsLocalResource(FileSystem fs, Path destFilePath, LocalResourceType resourceType,
        Map<String, LocalResource> resourceMap) throws IOException {
    LocalResource fileResource = Records.newRecord(LocalResource.class);
    FileStatus fileStatus = fs.getFileStatus(destFilePath);
    fileResource.setResource(ConverterUtils.getYarnUrlFromPath(destFilePath));
    fileResource.setSize(fileStatus.getLen());
    fileResource.setTimestamp(fileStatus.getModificationTime());
    fileResource.setType(resourceType);
    fileResource.setVisibility(LocalResourceVisibility.APPLICATION);
    resourceMap.put(destFilePath.getName(), fileResource);
}

From source file:husky.client.HuskyYarnClient.java

License:Apache License

private Pair<String, LocalResource> constructLocalResource(String name, String path, LocalResourceType type)
        throws IOException {
    LOG.info("To copy " + name + "(" + path + ") from local file system");

    Path resourcePath = new Path(path);
    if (path.startsWith("hdfs://")) {
        FileStatus fileStatus = mFileSystem.getFileStatus(resourcePath);
        if (!fileStatus.isFile()) {
            throw new RuntimeException("Only files can be provided as local resources.");
        }/* w  w w  .  j a  v a 2s.c  o m*/
    } else {
        File file = new File(path);
        if (!file.exists()) {
            throw new RuntimeException("File not exist: " + path);
        }
        if (!file.isFile()) {
            throw new RuntimeException("Only files can be provided as local resources.");
        }
    }

    // if the file is not on hdfs, upload it to hdfs first.
    if (!path.startsWith("hdfs://")) {
        Path src = resourcePath;
        String newPath = mLocalResourceHDFSPaths + '/' + mAppName + '/' + mAppId + '/' + name;
        resourcePath = new Path(newPath);
        mFileSystem.copyFromLocalFile(false, true, src, resourcePath);
        LOG.info("Upload " + path + " to " + newPath);
        path = newPath;
    }

    FileStatus fileStatus = mFileSystem.getFileStatus(resourcePath);

    LocalResource resource = Records.newRecord(LocalResource.class);
    resource.setType(type);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(resourcePath));
    resource.setTimestamp(fileStatus.getModificationTime());
    resource.setSize(fileStatus.getLen());

    return new Pair<String, LocalResource>(path, resource);
}

From source file:husky.server.HuskyRMCallbackHandler.java

License:Apache License

private LocalResource constructLocalResource(String name, String path, LocalResourceType type)
        throws IOException {
    LOG.info("To copy " + name + " from " + path);
    if (!path.startsWith("hdfs://")) {
        throw new RuntimeException(
                "Local resources provided to application master must be already located on HDFS.");
    }//from  ww  w.  j a  va 2 s  . c o  m
    FileStatus dsfStaus = mAppMaster.getFileSystem().getFileStatus(new Path(path));
    LocalResource resource = Records.newRecord(LocalResource.class);
    resource.setType(type);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(new Path(path)));
    resource.setTimestamp(dsfStaus.getModificationTime());
    resource.setSize(dsfStaus.getLen());
    return resource;
}

From source file:io.amient.yarn1.YarnContainerContext.java

License:Open Source License

private void prepareLocalResourceFile(Map<String, LocalResource> localResources, String fileName,
        String remoteFileName, FileSystem distFs) throws IOException {
    final Path dst = new Path(distFs.getHomeDirectory(), remoteFileName);
    FileStatus scFileStatus = distFs.getFileStatus(dst);
    final URL yarnUrl = ConverterUtils.getYarnUrlFromURI(dst.toUri());
    LocalResource scRsrc = LocalResource.newInstance(yarnUrl, LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileName, scRsrc);
}

From source file:io.dstream.tez.utils.HadoopUtils.java

License:Apache License

/**
 * Creates a single {@link LocalResource} for the provisioned resource identified with {@link Path}
 *
 * @param fs// www . jav a 2  s.c om
 * @param provisionedResourcePath
 * @return
 */
public static LocalResource createLocalResource(FileSystem fs, Path provisionedResourcePath) {
    try {
        FileStatus scFileStatus = fs.getFileStatus(provisionedResourcePath);
        LocalResource localResource = LocalResource.newInstance(
                ConverterUtils.getYarnUrlFromURI(provisionedResourcePath.toUri()), LocalResourceType.FILE,
                LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
        return localResource;
    } catch (Exception e) {
        throw new IllegalStateException(
                "Failed to communicate with FileSystem while creating LocalResource: " + fs, e);
    }
}

From source file:io.hops.tensorflow.ApplicationMaster.java

License:Apache License

/**
 * Parse command line options//from w w w  .j  av  a  2 s . c o  m
 *
 * @param args
 *     Command line args
 * @return Whether init successful and run should be invoked
 * @throws ParseException
 * @throws IOException
 */
public boolean init(String[] args) throws ParseException, IOException {
    Options opts = createOptions();
    cliParser = new GnuParser().parse(opts, args);

    containerPython = cliParser.getOptionValue(PYTHON, null);

    if (args.length == 0) {
        printUsage(opts);
        throw new IllegalArgumentException("No args specified for application master to initialize");
    }

    //Check whether customer log4j.properties file exists
    if (fileExist(LOG4J_PATH)) {
        try {
            Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, LOG4J_PATH);
        } catch (Exception e) {
            LOG.warn("Can not set up custom log4j properties. " + e);
        }
    }

    if (cliParser.hasOption(HELP)) {
        printUsage(opts);
        return false;
    }

    if (cliParser.hasOption(DEBUG)) {
        dumpOutDebugInfo();
    }

    if (!cliParser.hasOption(MAIN_RELATIVE)) {
        throw new IllegalArgumentException("No main application file specified");
    }
    mainRelative = cliParser.getOptionValue(MAIN_RELATIVE);

    if (cliParser.hasOption(ARGS)) {
        arguments = cliParser.getOptionValues(ARGS);
    }

    Map<String, String> envs = System.getenv();

    if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
        if (cliParser.hasOption(APP_ATTEMPT_ID)) {
            String appIdStr = cliParser.getOptionValue(APP_ATTEMPT_ID, "");
            appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
        } else {
            throw new IllegalArgumentException("Application Attempt Id not set in the environment");
        }
    } else {
        ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name()));
        appAttemptID = containerId.getApplicationAttemptId();
    }

    if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) {
        throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_HOST.name())) {
        throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) {
        throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_PORT.name())) {
        throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment");
    }

    LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId()
            + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId="
            + appAttemptID.getAttemptId());

    if (cliParser.hasOption(ENV)) {
        String shellEnvs[] = cliParser.getOptionValues(ENV);
        for (String env : shellEnvs) {
            env = env.trim();
            int index = env.indexOf('=');
            if (index == -1) {
                environment.put(env, "");
                continue;
            }
            String key = env.substring(0, index);
            String val = "";
            if (index < (env.length() - 1)) {
                val = env.substring(index + 1);
            }
            environment.put(key, val);
        }
    }

    if (cliParser.hasOption(TENSORBOARD)) {
        environment.put("YARNTF_TENSORBOARD", "true");
    }

    if (envs.containsKey(Constants.YARNTFTIMELINEDOMAIN)) {
        domainId = envs.get(Constants.YARNTFTIMELINEDOMAIN);
    }

    containerMemory = Integer.parseInt(cliParser.getOptionValue(MEMORY, "1024"));
    containerVirtualCores = Integer.parseInt(cliParser.getOptionValue(VCORES, "1"));
    containerGPUs = Integer.parseInt(cliParser.getOptionValue(GPUS, "0"));
    tfProtocol = cliParser.getOptionValue(PROTOCOL, null);

    numWorkers = Integer.parseInt(cliParser.getOptionValue(WORKERS, "1"));
    numPses = Integer.parseInt(cliParser.getOptionValue(PSES, "1"));
    numTotalContainers = numWorkers + numPses;
    if (!(numWorkers > 0 && numPses > 0 || numWorkers == 1 && numPses == 0)) {
        throw new IllegalArgumentException("Invalid no. of workers or parameter server");
    }
    // requestPriority = Integer.parseInt(cliParser.getOptionValue(PRIORITY, "0"));

    allocationTimeout = Long.parseLong(cliParser.getOptionValue(ALLOCATION_TIMEOUT, "15")) * 1000;

    environment.put("YARNTF_MEMORY", Integer.toString(containerMemory));
    environment.put("YARNTF_VCORES", Integer.toString(containerVirtualCores));
    environment.put("YARNTF_GPUS", Integer.toString(containerGPUs));
    if (tfProtocol != null) {
        environment.put("YARNTF_PROTOCOL", tfProtocol);
    }
    environment.put("YARNTF_WORKERS", Integer.toString(numWorkers));
    environment.put("YARNTF_PSES", Integer.toString(numPses));
    environment.put("YARNTF_HOME_DIR", FileSystem.get(conf).getHomeDirectory().toString());
    environment.put("PYTHONUNBUFFERED", "true");

    DistributedCacheList distCacheList = null;
    FileInputStream fin = null;
    ObjectInputStream ois = null;
    try {
        fin = new FileInputStream(Constants.DIST_CACHE_PATH);
        ois = new ObjectInputStream(fin);
        try {
            distCacheList = (DistributedCacheList) ois.readObject();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }
    } finally {
        org.apache.commons.io.IOUtils.closeQuietly(ois);
        org.apache.commons.io.IOUtils.closeQuietly(fin);
    }
    LOG.info("Loaded distribute cache list: " + distCacheList.toString());
    for (int i = 0; i < distCacheList.size(); i++) {
        DistributedCacheList.Entry entry = distCacheList.get(i);
        LocalResource distRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(entry.uri),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, entry.size, entry.timestamp);
        localResources.put(entry.relativePath, distRsrc);
    }

    return true;
}

From source file:io.hops.tensorflow.Client.java

License:Apache License

private Map<String, LocalResource> prepareLocalResources(FileSystem fs, ApplicationId appId,
        DistributedCacheList dcl) throws IOException {
    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the local resources
    Map<String, LocalResource> localResources = new HashMap<>();

    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    addResource(fs, appId, amJar, null, Constants.AM_JAR_PATH, null, localResources, null);

    if (!log4jPropFile.isEmpty()) {
        addResource(fs, appId, log4jPropFile, null, Constants.LOG4J_PATH, null, localResources, null);
    }/*from w w  w.j a  v a  2 s  . co m*/

    // Write distCacheList to HDFS and add to localResources
    Path baseDir = new Path(fs.getHomeDirectory(), Constants.YARNTF_STAGING + "/" + appId.toString());
    Path dclPath = new Path(baseDir, Constants.DIST_CACHE_PATH);
    FSDataOutputStream ostream = null;
    try {
        ostream = fs.create(dclPath);
        ostream.write(SerializationUtils.serialize(dcl));
    } finally {
        IOUtils.closeQuietly(ostream);
    }
    FileStatus dclStatus = fs.getFileStatus(dclPath);
    LocalResource distCacheResource = LocalResource.newInstance(
            ConverterUtils.getYarnUrlFromURI(dclPath.toUri()), LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, dclStatus.getLen(), dclStatus.getModificationTime());
    localResources.put(Constants.DIST_CACHE_PATH, distCacheResource);

    return localResources;
}

From source file:io.hops.tensorflow.Client.java

License:Apache License

private String addResource(FileSystem fs, ApplicationId appId, String srcPath, String dstDir, String dstName,
        DistributedCacheList distCache, Map<String, LocalResource> localResources, StringBuilder pythonPath)
        throws IOException {
    Path src = new Path(srcPath);

    if (dstDir == null) {
        dstDir = ".";
    }// www  .  j ava 2 s  .c  o  m
    if (dstName == null) {
        dstName = src.getName();
    }

    Path baseDir = new Path(fs.getHomeDirectory(), Constants.YARNTF_STAGING + "/" + appId.toString());
    String dstPath;
    if (dstDir.startsWith(".")) {
        dstPath = dstName;
    } else {
        dstPath = dstDir + "/" + dstName;
    }
    Path dst = new Path(baseDir, dstPath);

    LOG.info("Copying from local filesystem: " + src + " -> " + dst);
    fs.copyFromLocalFile(src, dst);
    FileStatus dstStatus = fs.getFileStatus(dst);

    if (distCache != null) {
        LOG.info("Adding to distributed cache: " + srcPath + " -> " + dstPath);
        distCache.add(new DistributedCacheList.Entry(dstPath, dst.toUri(), dstStatus.getLen(),
                dstStatus.getModificationTime()));
    }

    if (localResources != null) {
        LOG.info("Adding to local environment: " + srcPath + " -> " + dstPath);
        LocalResource resource = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, dstStatus.getLen(),
                dstStatus.getModificationTime());
        localResources.put(dstPath, resource);
    }

    if (pythonPath != null) {
        pythonPath.append(File.pathSeparator).append(dstPath);
    }

    return dstName;
}