Example usage for org.apache.hadoop.yarn.api.records LocalResourceType FILE

List of usage examples for org.apache.hadoop.yarn.api.records LocalResourceType FILE

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResourceType
  6. FILE

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceType FILE.

Prototype

LocalResourceType FILE

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceType FILE.

Click Source Link

Document

Regular file i.e.

Usage

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Uploads binary resources to HDFS for use by the AM
 * @return//from   ww w. j  a va  2  s. co  m
 * @throws IOException
 * @throws YarnException
 */
public List<DFSResourceCoordinate> distributeBinaries() throws IOException, YarnException {

    List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2);
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    FileSystem fs = FileSystem.get(conf);
    Path src, dst;
    FileStatus destStatus;
    String pathSuffix;

    // adding info so we can add the jar to the App master container path

    // Add the asterix tarfile to HDFS for easy distribution
    // Keep it all archived for now so add it as a file...

    pathSuffix = CONF_DIR_REL + instanceFolder + "asterix-server.zip";
    dst = new Path(fs.getHomeDirectory(), pathSuffix);
    if (refresh) {
        if (fs.exists(dst)) {
            fs.delete(dst, false);
        }
    }
    if (!fs.exists(dst)) {
        src = new Path(asterixZip);
        LOG.info("Copying Asterix distributable to DFS");
        fs.copyFromLocalFile(false, true, src, dst);
    }
    destStatus = fs.getFileStatus(dst);
    LocalResource asterixTarLoc = Records.newRecord(LocalResource.class);
    asterixTarLoc.setType(LocalResourceType.ARCHIVE);
    asterixTarLoc.setVisibility(LocalResourceVisibility.PRIVATE);
    asterixTarLoc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    asterixTarLoc.setTimestamp(destStatus.getModificationTime());

    // adding info so we can add the tarball to the App master container path
    DFSResourceCoordinate tar = new DFSResourceCoordinate();
    tar.envs.put(dst.toUri().toString(), AConstants.TARLOCATION);
    tar.envs.put(Long.toString(asterixTarLoc.getSize()), AConstants.TARLEN);
    tar.envs.put(Long.toString(asterixTarLoc.getTimestamp()), AConstants.TARTIMESTAMP);
    tar.res = asterixTarLoc;
    tar.name = "asterix-server.zip";
    resources.add(tar);

    // Set the log4j properties if needed
    if (!log4jPropFile.isEmpty()) {
        Path log4jSrc = new Path(log4jPropFile);
        Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props");
        fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
        FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
        LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
        log4jRsrc.setType(LocalResourceType.FILE);
        log4jRsrc.setVisibility(LocalResourceVisibility.PRIVATE);
        log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
        log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
        log4jRsrc.setSize(log4jFileStatus.getLen());
        DFSResourceCoordinate l4j = new DFSResourceCoordinate();
        tar.res = log4jRsrc;
        tar.name = "log4j.properties";
        resources.add(l4j);
    }

    resources.addAll(installAmLibs());
    return resources;
}

From source file:edu.uci.ics.hyracks.yarn.common.resources.LocalResourceHelper.java

License:Apache License

public static LocalResource createFileResource(Configuration config, File path) throws IOException {
    LocalResource lr = createLocalResourceFromPath(config, path);
    lr.setType(LocalResourceType.FILE);
    return lr;/*from   w  ww.  j a  v  a 2 s  .  co m*/
}

From source file:eu.stratosphere.yarn.Utils.java

License:Apache License

public static void registerLocalResource(FileSystem fs, Path remoteRsrcPath, LocalResource localResource)
        throws IOException {
    FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
    localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.PUBLIC);
}

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

private void addLibJars(Path srcLibJarDir, Optional<Map<String, LocalResource>> resourceMap, Path destDir)
        throws IOException {
    FileSystem localFs = FileSystem.getLocal(this.yarnConfiguration);
    FileStatus[] libJarFiles = localFs.listStatus(srcLibJarDir);
    if (libJarFiles == null || libJarFiles.length == 0) {
        return;//  w ww .  jav a2  s  .  c  o m
    }

    for (FileStatus libJarFile : libJarFiles) {
        Path destFilePath = new Path(destDir, libJarFile.getPath().getName());
        this.fs.copyFromLocalFile(libJarFile.getPath(), destFilePath);
        if (resourceMap.isPresent()) {
            YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE,
                    resourceMap.get());
        }
    }
}

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

private void addAppJars(String jarFilePathList, Optional<Map<String, LocalResource>> resourceMap, Path destDir)
        throws IOException {
    for (String jarFilePath : SPLITTER.split(jarFilePathList)) {
        Path srcFilePath = new Path(jarFilePath);
        Path destFilePath = new Path(destDir, srcFilePath.getName());
        this.fs.copyFromLocalFile(srcFilePath, destFilePath);
        if (resourceMap.isPresent()) {
            YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE,
                    resourceMap.get());/*w  w w.ja v a 2s  . c o m*/
        }
    }
}

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

private void addAppLocalFiles(String localFilePathList, Optional<Map<String, LocalResource>> resourceMap,
        Path destDir) throws IOException {
    for (String localFilePath : SPLITTER.split(localFilePathList)) {
        Path srcFilePath = new Path(localFilePath);
        Path destFilePath = new Path(destDir, srcFilePath.getName());
        this.fs.copyFromLocalFile(srcFilePath, destFilePath);
        if (resourceMap.isPresent()) {
            YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE,
                    resourceMap.get());//from   w w w. j  a v  a  2  s  .  c o m
        }
    }
}

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

private void addAppRemoteFiles(String hdfsFileList, Map<String, LocalResource> resourceMap) throws IOException {
    for (String hdfsFilePath : SPLITTER.split(hdfsFileList)) {
        YarnHelixUtils.addFileAsLocalResource(this.fs, new Path(hdfsFilePath), LocalResourceType.FILE,
                resourceMap);//from  w w  w  . jav a  2s  .c  o m
    }
}

From source file:gobblin.yarn.YarnService.java

License:Apache License

private void addContainerLocalResources(Path destDir, Map<String, LocalResource> resourceMap)
        throws IOException {
    if (!this.fs.exists(destDir)) {
        LOGGER.warn(String.format("Path %s does not exist so no container LocalResource to add", destDir));
        return;/*from ww w .  j av  a 2s .  c  o m*/
    }

    FileStatus[] statuses = this.fs.listStatus(destDir);
    if (statuses != null) {
        for (FileStatus status : statuses) {
            YarnHelixUtils.addFileAsLocalResource(this.fs, status.getPath(), LocalResourceType.FILE,
                    resourceMap);
        }
    }
}

From source file:gobblin.yarn.YarnService.java

License:Apache License

private void addRemoteAppFiles(String hdfsFileList, Map<String, LocalResource> resourceMap) throws IOException {
    for (String hdfsFilePath : SPLITTER.split(hdfsFileList)) {
        Path srcFilePath = new Path(hdfsFilePath);
        YarnHelixUtils.addFileAsLocalResource(srcFilePath.getFileSystem(this.yarnConfiguration), srcFilePath,
                LocalResourceType.FILE, resourceMap);
    }/* ww w .j  a  v a  2s  . c om*/
}

From source file:husky.server.HuskyRMCallbackHandler.java

License:Apache License

private void constructLocalResources() {
    localResources = new HashMap<String, LocalResource>();

    try {/* w ww.jav  a  2 s  . c o m*/
        String[][] resources = { { "HuskyAppExec", mAppMaster.getAppExec() },
                { "HuskyMasterExec", mAppMaster.getMasterExec() },
                { "HuskyConfigFile", mAppMaster.getConfigFile() } };

        for (String[] resource : resources) {
            LocalResource lr = constructLocalResource(resource[0], resource[1], LocalResourceType.FILE);
            localResources.put(resource[0], lr);
        }

        for (String i : mAppMaster.getLocalFiles().split(",")) {
            i = i.trim();
            if (!i.isEmpty()) {
                String name = new Path(i).getName();
                LocalResource lr = constructLocalResource(name, i, LocalResourceType.FILE);
                localResources.put(name, lr);
            }
        }

        for (String i : mAppMaster.getLocalArchives().split(",")) {
            i = i.trim();
            if (!i.isEmpty()) {
                String name = new Path(i).getName();
                LocalResource lr = constructLocalResource(name, i, LocalResourceType.ARCHIVE);
                localResources.put(name, lr);
            }
        }
    } catch (IOException e) {
        LOG.log(Level.WARNING, " Failed to construct local resource map: ", e);
    }
}