Example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION.

Prototype

LocalResourceVisibility APPLICATION

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION.

Click Source Link

Document

Shared only among containers of the same application on the node.

Usage

From source file:cascading.flow.tez.util.TezUtil.java

License:Open Source License

protected static void addResource(Map<String, LocalResource> localResources, Map<String, String> environment,
        String fileName, FileStatus stats, Path fullPath, LocalResourceType type) throws IOException {
    if (localResources.containsKey(fileName))
        throw new FlowException("duplicate filename added to classpath resources: " + fileName);

    URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(fullPath);
    long len = stats.getLen();
    long modificationTime = stats.getModificationTime();

    LocalResource resource = LocalResource.newInstance(yarnUrlFromPath, type,
            LocalResourceVisibility.APPLICATION, len, modificationTime);

    if (type == LocalResourceType.PATTERN) {
        // todo: parametrize this for dynamic inclusion below
        String pattern = "(?:classes/|lib/).*";

        resource.setPattern(pattern);// ww w.  ja v a 2s.c  om

        if (environment != null) {
            String current = "";

            current += PWD.$$() + File.separator + fileName + File.separator + "*" + CLASS_PATH_SEPARATOR;
            current += PWD.$$() + File.separator + fileName + File.separator + "lib" + File.separator + "*"
                    + CLASS_PATH_SEPARATOR;
            current += PWD.$$() + File.separator + fileName + File.separator + "classes" + File.separator + "*"
                    + CLASS_PATH_SEPARATOR;

            String classPath = environment.get(CLASSPATH.name());

            if (classPath == null)
                classPath = "";
            else if (!classPath.startsWith(CLASS_PATH_SEPARATOR))
                classPath += CLASS_PATH_SEPARATOR;

            classPath += current;

            LOG.info("adding to cluster side classpath: {} ", classPath);

            environment.put(CLASSPATH.name(), classPath);
        }
    }

    localResources.put(fileName, resource);
}

---

From source file:com.bigjob.Client.java

License:Apache License

private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, int appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    LOG.debug("HDFS Destination for Script: " + dst.toString());
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {//  ww w . ja va  2  s  .  c o  m
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

---

From source file:com.cfets.door.yarn.jboss.JBossClient.java

License:Apache License

/**
 * Main run function for the client/*from  w ww  . j  a va2s  . c  o  m*/
 * 
 * @return true if application completed successfully
 * @throws IOException
 * @throws YarnException
 */
public boolean run() throws IOException, YarnException {

    LOG.info("Running Client");
    yarnClient.start();

    YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
    LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers());

    List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
    LOG.info("Got Cluster node info from ASM");
    for (NodeReport node : clusterNodeReports) {
        LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress"
                + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers"
                + node.getNumContainers());
    }

    QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
    LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity="
            + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity()
            + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount="
            + queueInfo.getChildQueues().size());

    List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
    for (QueueUserACLInfo aclInfo : listAclInfo) {
        for (QueueACL userAcl : aclInfo.getUserAcls()) {
            LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl="
                    + userAcl.name());
        }
    }

    YarnClientApplication app = yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    int maxMem = appResponse.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

    if (amMemory > maxMem) {
        LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified="
                + amMemory + ", max=" + maxMem);
        amMemory = maxMem;
    }

    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();
    appContext.setApplicationName(appName);

    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    FileSystem fs = FileSystem.get(conf);
    Path src = new Path(appJar);
    String pathSuffix = appName + File.separator + appId.getId() + File.separator
            + JBossConstants.JBOSS_ON_YARN_APP;
    Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
    jbossAppUri = dst.toUri().toString();
    fs.copyFromLocalFile(false, true, src, dst);
    FileStatus destStatus = fs.getFileStatus(dst);
    LocalResource amJarRsrc = Records.newRecord(LocalResource.class);

    amJarRsrc.setType(LocalResourceType.FILE);
    amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    amJarRsrc.setTimestamp(destStatus.getModificationTime());
    amJarRsrc.setSize(destStatus.getLen());
    localResources.put(JBossConstants.JBOSS_ON_YARN_APP, amJarRsrc);

    if (!log4jPropFile.isEmpty()) {
        Path log4jSrc = new Path(log4jPropFile);
        Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props");
        fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
        FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
        LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
        log4jRsrc.setType(LocalResourceType.FILE);
        log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
        log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
        log4jRsrc.setSize(log4jFileStatus.getLen());
        localResources.put("log4j.properties", log4jRsrc);
    }

    amContainer.setLocalResources(localResources);

    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();

    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar)
            .append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");

    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(':');
        classPathEnv.append(System.getProperty("java.class.path"));
    }

    env.put("CLASSPATH", classPathEnv.toString());

    amContainer.setEnvironment(env);

    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    LOG.info("Setting up app master command");
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    vargs.add("-Xmx" + amMemory + "m");
    vargs.add(appMasterMainClass);
    vargs.add("--container_memory " + String.valueOf(containerMemory));
    vargs.add("--num_containers " + String.valueOf(numContainers));
    vargs.add("--priority " + String.valueOf(shellCmdPriority));
    vargs.add("--admin_user " + adminUser);
    vargs.add("--admin_password " + adminPassword);
    vargs.add("--jar " + jbossAppUri);

    if (debugFlag) {
        vargs.add("--debug");
    }

    vargs.add("1>" + JBossConstants.JBOSS_CONTAINER_LOG_DIR + "/JBossApplicationMaster.stdout");
    vargs.add("2>" + JBossConstants.JBOSS_CONTAINER_LOG_DIR + "/JBossApplicationMaster.stderr");

    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up app master command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    amContainer.setCommands(commands);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(amMemory);
    appContext.setResource(capability);

    appContext.setAMContainerSpec(amContainer);

    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    appContext.setQueue(amQueue);

    LOG.info("Submitting the application to ASM");

    yarnClient.submitApplication(appContext);

    return monitorApplication(appId);
}

---

From source file:com.cloudera.kitten.lua.AsapLuaContainerLaunchParameters.java

License:Open Source License

private void addOperatorInputs(Map<String, LocalResource> localResources) throws IOException {
    LOG.info("Inputs: " + operator.getInputFiles());
    FileSystem fs = FileSystem.get(conf);
    for (Entry<String, String> e : operator.getInputFiles().entrySet()) {
        if ((!e.getValue().startsWith("hdfs://")) && (!e.getValue().startsWith("$HDFS"))) {
            LOG.info("adding local resource: " + e);
            String inDir = dir;/* ww  w . ja  v  a  2 s .c  o  m*/
            LocalResource rsrc = Records.newRecord(LocalResource.class);
            rsrc.setType(LocalResourceType.FILE);
            rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
            LOG.info("Adding input: " + inDir + "/" + e.getValue());
            Path dst = new Path(inDir + "/" + e.getValue());
            dst = fs.makeQualified(dst);
            FileStatus stat = fs.getFileStatus(dst);
            rsrc.setSize(stat.getLen());
            rsrc.setTimestamp(stat.getModificationTime());
            rsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
            localResources.put(e.getKey(), rsrc);
        }
    }
    /*for(String in : operator.getArguments().split(" ")){
       LOG.info("Adding input: "+in);
       LocalResource nl = constructScriptResource();
       localResources.put(in, nl);
    }*/
}

---

From source file:com.cloudera.kitten.lua.AsapLuaContainerLaunchParameters.java

License:Open Source License

private LocalResource constructScriptResource() throws IOException {
    LocalResource rsrc = Records.newRecord(LocalResource.class);
    rsrc.setType(LocalResourceType.FILE);
    rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    String src = "file:///tmp/" + execScript;
    Path path = new Path(src);

    configureLocalScriptResourceForPath(rsrc, path);
    return rsrc;/*from w w w .ja  v  a2 s  . co  m*/
}

---

From source file:com.cloudera.kitten.lua.AsapLuaContainerLaunchParameters.java

License:Open Source License

private LocalResource constructExtraResource(String key) {
    LocalResource rsrc = Records.newRecord(LocalResource.class);
    rsrc.setType(LocalResourceType.FILE);
    rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    try {//w ww .  j a  v  a  2s  .  com
        Path path = new Path(localFileUris.get(key));
        configureLocalResourceForPath(rsrc, path);
    } catch (IOException e) {
        LOG.error("Error constructing extra local resource: " + key, e);
        return null;
    }
    return rsrc;
}

---

From source file:com.cloudera.kitten.lua.AsapLuaContainerLaunchParameters.java

License:Open Source License

private NamedLocalResource constructResource(LuaPair lp) throws IOException {
    LocalResource rsrc = Records.newRecord(LocalResource.class);
    LuaWrapper value = new LuaWrapper(lp.value.checktable());
    String name = lp.key.isint() ? "" : lp.key.tojstring();
    if (value.isNil(LuaFields.LOCAL_RESOURCE_TYPE)) {
        rsrc.setType(LocalResourceType.FILE);
    } else {/*w  w w  .  j a  v a2  s.c o  m*/
        rsrc.setType(LocalResourceType.valueOf(value.getString(LuaFields.LOCAL_RESOURCE_TYPE).toUpperCase()));
    }
    if (value.isNil(LuaFields.LOCAL_RESOURCE_VISIBILITY)) {
        rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    } else {
        rsrc.setVisibility(LocalResourceVisibility
                .valueOf(value.getString(LuaFields.LOCAL_RESOURCE_VISIBILITY).toUpperCase()));
    }
    if (!value.isNil(LuaFields.LOCAL_RESOURCE_URL)) {
        URI uri = URI.create(value.getString(LuaFields.LOCAL_RESOURCE_URL));
        rsrc.setResource(ConverterUtils.getYarnUrlFromURI(uri));
        if (name.isEmpty()) {
            name = (new File(uri.getPath())).getName();
        }
    } else if (!value.isNil(LuaFields.LOCAL_RESOURCE_HDFS_FILE)) {
        Path path = new Path(value.getString(LuaFields.LOCAL_RESOURCE_HDFS_FILE));
        configureLocalResourceForPath(rsrc, path);
        if (name.isEmpty()) {
            name = path.getName();
        }
    } else if (!value.isNil(LuaFields.LOCAL_RESOURCE_LOCAL_FILE)) {
        String src = value.getString(LuaFields.LOCAL_RESOURCE_LOCAL_FILE);
        Path path = new Path(localFileUris.get(src));
        configureLocalResourceForPath(rsrc, path);
        if (name.isEmpty()) {
            name = new Path(src).getName();
        }
    } else {
        throw new IllegalArgumentException("Invalid resource: no 'url', 'hdfs', or 'file' fields specified.");
    }
    return new NamedLocalResource(name, rsrc);
}

---

From source file:com.continuuity.weave.yarn.utils.YarnUtils.java

License:Open Source License

public static LocalResource createLocalResource(Location location) {
    try {/*from   w ww .ja  v  a 2  s.c om*/
        LocalResource resource = Records.newRecord(LocalResource.class);
        resource.setVisibility(LocalResourceVisibility.APPLICATION);
        resource.setType(LocalResourceType.FILE);
        resource.setResource(ConverterUtils.getYarnUrlFromURI(location.toURI()));
        resource.setTimestamp(location.lastModified());
        resource.setSize(location.length());
        return resource;
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
}

---

From source file:com.continuuity.weave.yarn.utils.YarnUtils.java

License:Open Source License

public static LocalResource createLocalResource(LocalFile localFile) {
    Preconditions.checkArgument(localFile.getLastModified() >= 0, "Last modified time should be >= 0.");
    Preconditions.checkArgument(localFile.getSize() >= 0, "File size should be >= 0.");

    LocalResource resource = Records.newRecord(LocalResource.class);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    resource.setResource(ConverterUtils.getYarnUrlFromURI(localFile.getURI()));
    resource.setTimestamp(localFile.getLastModified());
    resource.setSize(localFile.getSize());
    return setLocalResourceType(resource, localFile);
}

---

From source file:com.datatorrent.stram.LaunchContainerRunnable.java

License:Apache License

public static void addFilesToLocalResources(LocalResourceType type, String commaSeparatedFileNames,
        Map<String, LocalResource> localResources, FileSystem fs) throws IOException {
    String[] files = StringUtils.splitByWholeSeparator(commaSeparatedFileNames, StramClient.LIB_JARS_SEP);
    for (String file : files) {
        Path dst = new Path(file);
        // Create a local resource to point to the destination jar path
        FileStatus destStatus = fs.getFileStatus(dst);
        LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
        // Set the type of resource - file or archive
        amJarRsrc.setType(type);//from w w w . ja va2 s  . c om
        // Set visibility of the resource
        // Setting to most private option
        amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        // Set the resource to be copied over
        amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
        // Set timestamp and length of file so that the framework
        // can do basic sanity checks for the local resource
        // after it has been copied over to ensure it is the same
        // resource the client intended to use with the application
        amJarRsrc.setTimestamp(destStatus.getModificationTime());
        amJarRsrc.setSize(destStatus.getLen());
        localResources.put(dst.getName(), amJarRsrc);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• »