Example usage for org.apache.hadoop.yarn.api.records LocalResource newInstance

List of usage examples for org.apache.hadoop.yarn.api.records LocalResource newInstance

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResource
  6. newInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResource newInstance.

Prototype

@Public
    @Stable
    public static LocalResource newInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility,
            long size, long timestamp)

Source Link

Usage

From source file:org.hdl.tensorflow.yarn.util.Utils.java

License:Apache License

private static void addToLocalResources(FileSystem fs, String key, Path dst,
        Map<String, LocalResource> localResources) throws IOException {
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource resource = LocalResource.newInstance(URL.fromURI(dst.toUri()), LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(key, resource);/*  ww  w . j a  va2  s. c o  m*/
}

From source file:org.starschema.hadoop.yarn.applications.distributedshell.Client.java

License:Apache License

private void addToLocalResourcesCompressed(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {//from   ww w . j  a  v  a 2  s .  co  m
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

From source file:oz.hadoop.yarn.api.core.ApplicationContainerLauncherImpl.java

License:Apache License

/**
 *
 *///  w ww .  j ava 2  s. com
private Map<String, LocalResource> buildLocalResources() {
    try {
        Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
        String suffix = this.applicationSpecification.getString(YayaConstants.APPLICATION_NAME) + "_master/"
                + this.applicationSpecification.getInt(YayaConstants.APP_ID) + "/";
        FileSystem fs = FileSystem.get(this.yarnConfig);
        Path dst = new Path(fs.getHomeDirectory(), suffix);
        FileStatus[] deployedResources = fs.listStatus(dst);
        for (FileStatus fileStatus : deployedResources) {
            if (logger.isDebugEnabled()) {
                logger.debug("Creating local resource for: " + fileStatus.getPath());
            }
            LocalResource scRsrc = LocalResource.newInstance(
                    ConverterUtils.getYarnUrlFromURI(fileStatus.getPath().toUri()), LocalResourceType.FILE,
                    LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime());
            localResources.put(fileStatus.getPath().getName(), scRsrc);
        }
        return localResources;
    } catch (Exception e) {
        StringWriter sw = new StringWriter();
        PrintWriter writer = new PrintWriter(sw);
        e.printStackTrace(writer);
        throw new IllegalStateException("Failed to build LocalResources\n " + sw.toString(), e);
    }
}

From source file:oz.hadoop.yarn.api.core.ApplicationMasterLauncherImpl.java

License:Apache License

/**
 *
 *//* ww w. j  a  v a2  s  .  c  o  m*/
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, int appId,
        Map<String, LocalResource> localResources) {
    String suffix = this.applicationName + "_master/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);

    try {
        Path sourcePath = new Path(fileSrcPath);
        if (logger.isDebugEnabled()) {
            logger.debug("Copying '" + sourcePath + "' to " + dst);
        }
        fs.copyFromLocalFile(sourcePath, dst);
        FileStatus scFileStatus = fs.getFileStatus(dst);
        LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
                scFileStatus.getModificationTime());
        localResources.put(fileDstPath, scRsrc);
    } catch (Exception e) {
        throw new IllegalStateException("Failed to communicate with FileSystem: " + fs, e);
    }
}

From source file:oz.hadoop.yarn.api.core.YayaUtilsTests.java

License:Apache License

@Test
public void validateClassPathFromLocalSource() throws Exception {
    LocalResource l1 = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(new URI("file://foo.jar")),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1234L, 678L);
    LocalResource l2 = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(new URI("file://bar.jar")),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1234L, 678L);
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put("foo.jar", l1);
    localResources.put("bar.jar", l2);
    assertEquals("./bar.jar:./foo.jar:", YayaUtils.calculateClassPath(localResources));
}

From source file:timo.yarn_app_call_java_app.Client.java

License:Apache License

private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = Constants.APP_NAME + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {/* ww  w .java 2 s  . c  om*/
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

From source file:timo.yarn_app_call_java_daemon.Client.java

License:Apache License

/**
 * addToLocalResources/*from   ww  w .j  a v  a2s . c  o m*/
 * 
 * @param fs
 * @param fileSrcPath
 * @param fileDstPath
 * @param appId
 * @param localResources
 * @param resources
 * @throws IOException
 */
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = Constants.APP_NAME + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

From source file:x10.x10rt.yarn.Client.java

License:Open Source License

private String addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {//  www .  ja v a  2  s.  c  o m
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
    return ("/user/" + System.getProperty("user.name") + '/' + suffix);
}