Example usage for org.apache.hadoop.yarn.api.records LocalResourceType FILE

List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceType FILE

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResourceType
  6. FILE

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceType FILE.

Prototype

LocalResourceType FILE

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceType FILE.

Click Source Link

Document

Regular file i.e.

Usage

From source file:terasort.utils.Utils.java

License:Apache License

public static LocalResource createLocalResource(FileSystem fs, Path file) throws IOException {
    final LocalResourceType type = LocalResourceType.FILE;
    final LocalResourceVisibility visibility = LocalResourceVisibility.APPLICATION;
    FileStatus fstat = fs.getFileStatus(file);
    org.apache.hadoop.yarn.api.records.URL resourceURL = ConverterUtils.getYarnUrlFromPath(file);
    long resourceSize = fstat.getLen();
    long resourceModificationTime = fstat.getModificationTime();
    LocalResource lr = Records.newRecord(LocalResource.class);
    lr.setResource(resourceURL);//from w  w w .  jav  a2  s  .  c o  m
    lr.setType(type);
    lr.setSize(resourceSize);
    lr.setVisibility(visibility);
    lr.setTimestamp(resourceModificationTime);
    return lr;
}

From source file:timo.yarn_app_call_java_app.Client.java

License:Apache License

private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = Constants.APP_NAME + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {//  w  w w  .ja  v a 2s. co  m
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

From source file:timo.yarn_app_call_java_daemon.Client.java

License:Apache License

/**
 * addToLocalResources/*from  w  w  w. j ava2 s .  co  m*/
 * 
 * @param fs
 * @param fileSrcPath
 * @param fileDstPath
 * @param appId
 * @param localResources
 * @param resources
 * @throws IOException
 */
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = Constants.APP_NAME + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

From source file:x10.x10rt.yarn.Client.java

License:Open Source License

private String addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {//from w w w. ja  v a 2  s .  c  om
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
    return ("/user/" + System.getProperty("user.name") + '/' + suffix);
}

From source file:yarnkit.utils.YarnUtils.java

License:Apache License

@Nonnull
public static LocalResource createLocalResource(@Nonnull FileSystem fs, @Nonnull Path hdfsPath)
        throws IOException {
    LocalResource resource = Records.newRecord(LocalResource.class);
    FileStatus fileStat = fs.getFileStatus(hdfsPath);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(hdfsPath));
    resource.setSize(fileStat.getLen());
    resource.setTimestamp(fileStat.getModificationTime());
    resource.setType(fileStat.isFile() ? LocalResourceType.FILE : LocalResourceType.ARCHIVE);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    return resource;
}

From source file:yrun.YarnRunner.java

License:Apache License

private void setupAppMasterArgs(Path yarnRunnerArgsPath, LocalResource appMasterArgs) throws IOException {
    FileSystem fileSystem = yarnRunnerArgsPath.getFileSystem(_configuration);
    FileStatus jarStatus = fileSystem.getFileStatus(yarnRunnerArgsPath);
    appMasterArgs.setResource(ConverterUtils.getYarnUrlFromPath(yarnRunnerArgsPath));
    appMasterArgs.setSize(jarStatus.getLen());
    appMasterArgs.setTimestamp(jarStatus.getModificationTime());
    appMasterArgs.setType(LocalResourceType.FILE);
    appMasterArgs.setVisibility(LocalResourceVisibility.PUBLIC);
}

From source file:yrun.YarnRunner.java

License:Apache License

private void setupAppMasterJar(Path jarPath, LocalResource appMasterJar) throws IOException {
    FileSystem fileSystem = jarPath.getFileSystem(_configuration);
    FileStatus jarStatus = fileSystem.getFileStatus(jarPath);
    appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath));
    appMasterJar.setSize(jarStatus.getLen());
    appMasterJar.setTimestamp(jarStatus.getModificationTime());
    appMasterJar.setType(LocalResourceType.FILE);
    appMasterJar.setVisibility(LocalResourceVisibility.PUBLIC);
}