Example usage for org.apache.hadoop.yarn.api.records LocalResource setType

List of usage examples for org.apache.hadoop.yarn.api.records LocalResource setType

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. LocalResource
  6. setType

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResource setType.

Prototype

@Public
@Stable
public abstract void setType(LocalResourceType type);

Source Link

Document

Set the LocalResourceType of the resource to be localized.

Usage

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Finds the minimal classes and JARs needed to start the AM only.
 * @return Resources the AM needs to start on the initial container.
 * @throws IllegalStateException//from   w w w . j a v a 2 s.co  m
 * @throws IOException
 */
private List<DFSResourceCoordinate> installAmLibs() throws IllegalStateException, IOException {
    List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2);
    FileSystem fs = FileSystem.get(conf);
    String fullLibPath = CONF_DIR_REL + instanceFolder + "am_jars" + Path.SEPARATOR;
    String[] cp = System.getProperty("java.class.path").split(System.getProperty("path.separator"));
    String asterixJarPattern = "^(asterix).*(jar)$"; //starts with asterix,ends with jar
    String commonsJarPattern = "^(commons).*(jar)$";
    String surefireJarPattern = "^(surefire).*(jar)$"; //for maven tests
    String jUnitTestPattern = "^(asterix-yarn" + File.separator + "target)$";

    LOG.info(File.separator);
    for (String j : cp) {
        String[] pathComponents = j.split(Pattern.quote(File.separator));
        LOG.info(j);
        LOG.info(pathComponents[pathComponents.length - 1]);
        if (pathComponents[pathComponents.length - 1].matches(asterixJarPattern)
                || pathComponents[pathComponents.length - 1].matches(commonsJarPattern)
                || pathComponents[pathComponents.length - 1].matches(surefireJarPattern)
                || pathComponents[pathComponents.length - 1].matches(jUnitTestPattern)) {
            LOG.info("Loading JAR/classpath: " + j);
            File f = new File(j);
            Path dst = new Path(fs.getHomeDirectory(), fullLibPath + f.getName());
            if (!fs.exists(dst) || refresh) {
                fs.copyFromLocalFile(false, true, new Path(f.getAbsolutePath()), dst);
            }
            FileStatus dstSt = fs.getFileStatus(dst);
            LocalResource amLib = Records.newRecord(LocalResource.class);
            amLib.setType(LocalResourceType.FILE);
            amLib.setVisibility(LocalResourceVisibility.PRIVATE);
            amLib.setResource(ConverterUtils.getYarnUrlFromPath(dst));
            amLib.setTimestamp(dstSt.getModificationTime());
            amLib.setSize(dstSt.getLen());
            DFSResourceCoordinate amLibCoord = new DFSResourceCoordinate();
            amLibCoord.res = amLib;
            amLibCoord.name = f.getName();
            if (f.getName().contains("asterix-yarn") || f.getName().contains("surefire")) {
                amLibCoord.envs.put(dst.toUri().toString(), AConstants.APPLICATIONMASTERJARLOCATION);
                amLibCoord.envs.put(Long.toString(dstSt.getLen()), AConstants.APPLICATIONMASTERJARLEN);
                amLibCoord.envs.put(Long.toString(dstSt.getModificationTime()),
                        AConstants.APPLICATIONMASTERJARTIMESTAMP);
            }
            resources.add(amLibCoord);
        }

    }
    if (resources.size() == 0) {
        throw new IOException("Required JARs are missing. Please check your directory structure");
    }
    return resources;
}

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Uploads binary resources to HDFS for use by the AM
 * @return//from w  ww.  j  av a 2  s.c o  m
 * @throws IOException
 * @throws YarnException
 */
public List<DFSResourceCoordinate> distributeBinaries() throws IOException, YarnException {

    List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2);
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    FileSystem fs = FileSystem.get(conf);
    Path src, dst;
    FileStatus destStatus;
    String pathSuffix;

    // adding info so we can add the jar to the App master container path

    // Add the asterix tarfile to HDFS for easy distribution
    // Keep it all archived for now so add it as a file...

    pathSuffix = CONF_DIR_REL + instanceFolder + "asterix-server.zip";
    dst = new Path(fs.getHomeDirectory(), pathSuffix);
    if (refresh) {
        if (fs.exists(dst)) {
            fs.delete(dst, false);
        }
    }
    if (!fs.exists(dst)) {
        src = new Path(asterixZip);
        LOG.info("Copying Asterix distributable to DFS");
        fs.copyFromLocalFile(false, true, src, dst);
    }
    destStatus = fs.getFileStatus(dst);
    LocalResource asterixTarLoc = Records.newRecord(LocalResource.class);
    asterixTarLoc.setType(LocalResourceType.ARCHIVE);
    asterixTarLoc.setVisibility(LocalResourceVisibility.PRIVATE);
    asterixTarLoc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    asterixTarLoc.setTimestamp(destStatus.getModificationTime());

    // adding info so we can add the tarball to the App master container path
    DFSResourceCoordinate tar = new DFSResourceCoordinate();
    tar.envs.put(dst.toUri().toString(), AConstants.TARLOCATION);
    tar.envs.put(Long.toString(asterixTarLoc.getSize()), AConstants.TARLEN);
    tar.envs.put(Long.toString(asterixTarLoc.getTimestamp()), AConstants.TARTIMESTAMP);
    tar.res = asterixTarLoc;
    tar.name = "asterix-server.zip";
    resources.add(tar);

    // Set the log4j properties if needed
    if (!log4jPropFile.isEmpty()) {
        Path log4jSrc = new Path(log4jPropFile);
        Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props");
        fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
        FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
        LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
        log4jRsrc.setType(LocalResourceType.FILE);
        log4jRsrc.setVisibility(LocalResourceVisibility.PRIVATE);
        log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
        log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
        log4jRsrc.setSize(log4jFileStatus.getLen());
        DFSResourceCoordinate l4j = new DFSResourceCoordinate();
        tar.res = log4jRsrc;
        tar.name = "log4j.properties";
        resources.add(l4j);
    }

    resources.addAll(installAmLibs());
    return resources;
}

From source file:edu.uci.ics.hyracks.yarn.common.resources.LocalResourceHelper.java

License:Apache License

public static LocalResource createFileResource(Configuration config, File path) throws IOException {
    LocalResource lr = createLocalResourceFromPath(config, path);
    lr.setType(LocalResourceType.FILE);
    return lr;/*from   w w w.  j  ava 2 s.com*/
}

From source file:edu.uci.ics.hyracks.yarn.common.resources.LocalResourceHelper.java

License:Apache License

public static LocalResource createArchiveResource(Configuration config, File path) throws IOException {
    LocalResource lr = createLocalResourceFromPath(config, path);
    lr.setType(LocalResourceType.ARCHIVE);
    return lr;/*ww  w .  j  a  v a2 s.c  o  m*/
}

From source file:eu.stratosphere.yarn.Utils.java

License:Apache License

public static void registerLocalResource(FileSystem fs, Path remoteRsrcPath, LocalResource localResource)
        throws IOException {
    FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
    localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.PUBLIC);
}

From source file:gobblin.yarn.YarnHelixUtils.java

License:Apache License

/**
 * Add a file as a Yarn {@link org.apache.hadoop.yarn.api.records.LocalResource}.
 *
 * @param fs a {@link FileSystem} instance
 * @param destFilePath the destination file path
 * @param resourceType the {@link org.apache.hadoop.yarn.api.records.LocalResourceType} of the file
 * @param resourceMap a {@link Map} of file names to their corresponding
 *                    {@link org.apache.hadoop.yarn.api.records.LocalResource}s
 * @throws IOException if there's something wrong adding the file as a
 *                     {@link org.apache.hadoop.yarn.api.records.LocalResource}
 *//* w w w . j  a v  a  2  s  .  co m*/
public static void addFileAsLocalResource(FileSystem fs, Path destFilePath, LocalResourceType resourceType,
        Map<String, LocalResource> resourceMap) throws IOException {
    LocalResource fileResource = Records.newRecord(LocalResource.class);
    FileStatus fileStatus = fs.getFileStatus(destFilePath);
    fileResource.setResource(ConverterUtils.getYarnUrlFromPath(destFilePath));
    fileResource.setSize(fileStatus.getLen());
    fileResource.setTimestamp(fileStatus.getModificationTime());
    fileResource.setType(resourceType);
    fileResource.setVisibility(LocalResourceVisibility.APPLICATION);
    resourceMap.put(destFilePath.getName(), fileResource);
}

From source file:husky.client.HuskyYarnClient.java

License:Apache License

private Pair<String, LocalResource> constructLocalResource(String name, String path, LocalResourceType type)
        throws IOException {
    LOG.info("To copy " + name + "(" + path + ") from local file system");

    Path resourcePath = new Path(path);
    if (path.startsWith("hdfs://")) {
        FileStatus fileStatus = mFileSystem.getFileStatus(resourcePath);
        if (!fileStatus.isFile()) {
            throw new RuntimeException("Only files can be provided as local resources.");
        }// w  ww .  jav  a 2 s.  c  o m
    } else {
        File file = new File(path);
        if (!file.exists()) {
            throw new RuntimeException("File not exist: " + path);
        }
        if (!file.isFile()) {
            throw new RuntimeException("Only files can be provided as local resources.");
        }
    }

    // if the file is not on hdfs, upload it to hdfs first.
    if (!path.startsWith("hdfs://")) {
        Path src = resourcePath;
        String newPath = mLocalResourceHDFSPaths + '/' + mAppName + '/' + mAppId + '/' + name;
        resourcePath = new Path(newPath);
        mFileSystem.copyFromLocalFile(false, true, src, resourcePath);
        LOG.info("Upload " + path + " to " + newPath);
        path = newPath;
    }

    FileStatus fileStatus = mFileSystem.getFileStatus(resourcePath);

    LocalResource resource = Records.newRecord(LocalResource.class);
    resource.setType(type);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(resourcePath));
    resource.setTimestamp(fileStatus.getModificationTime());
    resource.setSize(fileStatus.getLen());

    return new Pair<String, LocalResource>(path, resource);
}

From source file:husky.server.HuskyRMCallbackHandler.java

License:Apache License

private LocalResource constructLocalResource(String name, String path, LocalResourceType type)
        throws IOException {
    LOG.info("To copy " + name + " from " + path);
    if (!path.startsWith("hdfs://")) {
        throw new RuntimeException(
                "Local resources provided to application master must be already located on HDFS.");
    }// w  w  w .ja v a2s . c o m
    FileStatus dsfStaus = mAppMaster.getFileSystem().getFileStatus(new Path(path));
    LocalResource resource = Records.newRecord(LocalResource.class);
    resource.setType(type);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(new Path(path)));
    resource.setTimestamp(dsfStaus.getModificationTime());
    resource.setSize(dsfStaus.getLen());
    return resource;
}

From source file:hws.core.ContainerUtils.java

License:Apache License

public static void setupContainerJar(FileSystem fs, Path jarPath, LocalResource containerJar)
        throws IOException {
    FileStatus jarStat = fs.getFileStatus(jarPath);
    containerJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath));
    containerJar.setSize(jarStat.getLen());
    containerJar.setTimestamp(jarStat.getModificationTime());
    containerJar.setType(LocalResourceType.FILE);
    containerJar.setVisibility(LocalResourceVisibility.PUBLIC);
}

From source file:MasteringYarn.DistributedShellClient.java

private void setupJarFileForApplicationMaster(Path jarPath, LocalResource localResource) throws IOException {
    FileStatus jarStat = FileSystem.get(conf).getFileStatus(jarPath);
    localResource.setResource(ConverterUtils.getYarnUrlFromPath(jarPath));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.PUBLIC);
}