Example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION.

Prototype

LocalResourceVisibility APPLICATION

To view the source code for org.apache.hadoop.yarn.api.records LocalResourceVisibility APPLICATION.

Click Source Link

Document

Shared only among containers of the same application on the node.

Usage

From source file:ml.shifu.guagua.yarn.util.YarnUtils.java

License:Apache License

/**
 * Boilerplate to add a file to the local resources..
 * // w  w w. j  ava  2s .c  om
 * @param localResources
 *            the LocalResources map to populate.
 * @param fs
 *            handle to the HDFS file system.
 * @param target
 *            the file to send to the remote container.
 */
public static void addFileToResourceMap(Map<String, LocalResource> localResources, FileSystem fs, Path target)
        throws IOException {
    LocalResource resource = Records.newRecord(LocalResource.class);
    FileStatus destStatus = fs.getFileStatus(target);
    resource.setResource(ConverterUtils.getYarnUrlFromURI(target.toUri()));
    resource.setSize(destStatus.getLen());
    resource.setTimestamp(destStatus.getModificationTime());
    resource.setType(LocalResourceType.FILE); // use FILE, even for jars!
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResources.put(target.getName(), resource);
    LOG.info("Registered file in LocalResources :{} ", target);
}

---

From source file:org.apache.flink.tez.client.TezExecutor.java

License:Apache License

private static void addLocalResource(TezConfiguration tezConf, Path jarPath, DAG dag) {

    try {//from   ww w .j  a va 2s .c om
        org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(tezConf);

        LOG.info("Jar path received is " + jarPath.toString());

        String jarFile = jarPath.getName();

        Path remoteJarPath = null;

        /*
        if (tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR) == null) {
           LOG.info("Tez staging directory is null, setting it.");
           Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
           LOG.info("Setting Tez staging directory to " + stagingDir.toString());
           tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
           LOG.info("Set Tez staging directory to " + stagingDir.toString());
        }
        Path stagingDir = new Path(tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR));
        LOG.info("Ensuring that Tez staging directory exists");
        TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
        LOG.info("Tez staging directory exists and is " + stagingDir.toString());
        */

        Path stagingDir = TezCommonUtils.getTezBaseStagingPath(tezConf);
        LOG.info("Tez staging path is " + stagingDir);
        TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
        LOG.info("Tez staging dir exists");

        remoteJarPath = fs.makeQualified(new Path(stagingDir, jarFile));
        LOG.info("Copying " + jarPath.toString() + " to " + remoteJarPath.toString());
        fs.copyFromLocalFile(jarPath, remoteJarPath);

        FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
        Credentials credentials = new Credentials();
        TokenCache.obtainTokensForNamenodes(credentials, new Path[] { remoteJarPath }, tezConf);

        Map<String, LocalResource> localResources = new TreeMap<String, LocalResource>();
        LocalResource jobJar = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(remoteJarPath),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, remoteJarStatus.getLen(),
                remoteJarStatus.getModificationTime());
        localResources.put(jarFile.toString(), jobJar);

        dag.addTaskLocalFiles(localResources);

        LOG.info("Added job jar as local resource.");
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        System.exit(-1);
    }
}

---

From source file:org.apache.giraph.yarn.YarnUtils.java

License:Apache License

/**
 * Boilerplate to add a file to the local resources..
 * @param localResources the LocalResources map to populate.
 * @param fs handle to the HDFS file system.
 * @param target the file to send to the remote container.
 */// w w w  . j  av  a 2 s  .com
public static void addFileToResourceMap(Map<String, LocalResource> localResources, FileSystem fs, Path target)
        throws IOException {
    LocalResource resource = Records.newRecord(LocalResource.class);
    FileStatus destStatus = fs.getFileStatus(target);
    resource.setResource(ConverterUtils.getYarnUrlFromURI(target.toUri()));
    resource.setSize(destStatus.getLen());
    resource.setTimestamp(destStatus.getModificationTime());
    resource.setType(LocalResourceType.FILE); // use FILE, even for jars!
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResources.put(target.getName(), resource);
    LOG.info("Registered file in LocalResources :: " + target);
}

---

From source file:org.apache.hama.bsp.BSPTaskLauncher.java

License:Apache License

private GetContainerStatusesRequest setupContainer(Container allocatedContainer, ContainerManagementProtocol cm,
        String user, int id) throws IOException, YarnException {
    LOG.info("Setting up a container for user " + user + " with id of " + id + " and containerID of "
            + allocatedContainer.getId() + " as " + user);
    // Now we setup a ContainerLaunchContext
    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

    // Set the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    FileSystem fs = FileSystem.get(conf);
    Path packageFile = new Path(System.getenv(YARNBSPConstants.HAMA_YARN_LOCATION));
    URL packageUrl = ConverterUtils
            .getYarnUrlFromPath(packageFile.makeQualified(fs.getUri(), fs.getWorkingDirectory()));
    LOG.info("PackageURL has been composed to " + packageUrl.toString());
    try {/*from  w  w  w. ja v a 2  s.  c  o m*/
        LOG.info("Reverting packageURL to path: " + ConverterUtils.getPathFromYarnURL(packageUrl));
    } catch (URISyntaxException e) {
        LOG.fatal("If you see this error the workarround does not work", e);
    }

    packageResource.setResource(packageUrl);
    packageResource.setSize(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_YARN_SIZE)));
    packageResource.setTimestamp(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_YARN_TIMESTAMP)));
    packageResource.setType(LocalResourceType.FILE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    localResources.put(YARNBSPConstants.APP_MASTER_JAR_PATH, packageResource);

    Path hamaReleaseFile = new Path(System.getenv(YARNBSPConstants.HAMA_RELEASE_LOCATION));
    URL hamaReleaseUrl = ConverterUtils
            .getYarnUrlFromPath(hamaReleaseFile.makeQualified(fs.getUri(), fs.getWorkingDirectory()));
    LOG.info("Hama release URL has been composed to " + hamaReleaseUrl.toString());

    LocalResource hamaReleaseRsrc = Records.newRecord(LocalResource.class);
    hamaReleaseRsrc.setResource(hamaReleaseUrl);
    hamaReleaseRsrc.setSize(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_RELEASE_SIZE)));
    hamaReleaseRsrc.setTimestamp(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_RELEASE_TIMESTAMP)));
    hamaReleaseRsrc.setType(LocalResourceType.ARCHIVE);
    hamaReleaseRsrc.setVisibility(LocalResourceVisibility.APPLICATION);

    localResources.put(YARNBSPConstants.HAMA_SYMLINK, hamaReleaseRsrc);

    ctx.setLocalResources(localResources);

    /*
     * TODO Package classpath seems not to work if you're in pseudo distributed
     * mode, because the resource must not be moved, it will never be unpacked.
     * So we will check if our jar file has the file:// prefix and put it into
     * the CP directly
     */

    StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$())
            .append(File.pathSeparatorChar).append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }

    classPathEnv.append(File.pathSeparator);
    classPathEnv
            .append("./" + YARNBSPConstants.HAMA_SYMLINK + "/" + YARNBSPConstants.HAMA_RELEASE_VERSION + "/*");
    classPathEnv.append(File.pathSeparator);
    classPathEnv.append(
            "./" + YARNBSPConstants.HAMA_SYMLINK + "/" + YARNBSPConstants.HAMA_RELEASE_VERSION + "/lib/*");

    Vector<CharSequence> vargs = new Vector<CharSequence>();
    vargs.add("${JAVA_HOME}/bin/java");
    vargs.add("-cp " + classPathEnv + "");
    vargs.add(BSPRunner.class.getCanonicalName());

    vargs.add(jobId.getJtIdentifier());
    vargs.add(Integer.toString(id));
    vargs.add(this.jobFile.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/bsp.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/bsp.stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());

    ctx.setCommands(commands);
    LOG.info("Starting command: " + commands);

    StartContainerRequest startReq = Records.newRecord(StartContainerRequest.class);
    startReq.setContainerLaunchContext(ctx);
    startReq.setContainerToken(allocatedContainer.getContainerToken());

    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(startReq);
    StartContainersRequest requestList = StartContainersRequest.newInstance(list);
    cm.startContainers(requestList);

    GetContainerStatusesRequest statusReq = Records.newRecord(GetContainerStatusesRequest.class);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(allocatedContainer.getId());
    statusReq.setContainerIds(containerIds);
    return statusReq;
}

---

From source file:org.apache.hama.bsp.YARNBSPJobClient.java

License:Apache License

@Override
protected RunningJob launchJob(BSPJobID jobId, BSPJob normalJob, Path submitJobFile, FileSystem pFs)
        throws IOException {
    YARNBSPJob job = (YARNBSPJob) normalJob;

    LOG.info("Submitting job...");
    if (getConf().get("bsp.child.mem.in.mb") == null) {
        LOG.warn("BSP Child memory has not been set, YARN will guess your needs or use default values.");
    }/*  w w w  .ja va  2 s .com*/

    FileSystem fs = pFs;
    if (fs == null) {
        fs = FileSystem.get(getConf());
    }

    if (getConf().get("bsp.user.name") == null) {
        String s = getUnixUserName();
        getConf().set("bsp.user.name", s);
        LOG.debug("Retrieved username: " + s);
    }

    yarnClient.start();
    try {
        YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
        LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers="
                + clusterMetrics.getNumNodeManagers());

        List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
        LOG.info("Got Cluster node info from ASM");
        for (NodeReport node : clusterNodeReports) {
            LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress"
                    + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers"
                    + node.getNumContainers());
        }

        QueueInfo queueInfo = yarnClient.getQueueInfo("default");
        LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity="
                + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity()
                + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount="
                + queueInfo.getChildQueues().size());

        List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
        for (QueueUserACLInfo aclInfo : listAclInfo) {
            for (QueueACL userAcl : aclInfo.getUserAcls()) {
                LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl="
                        + userAcl.name());
            }
        }

        // Get a new application id
        YarnClientApplication app = yarnClient.createApplication();

        // Create a new ApplicationSubmissionContext
        //ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
        ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();

        id = appContext.getApplicationId();

        // set the application name
        appContext.setApplicationName(job.getJobName());

        // Create a new container launch context for the AM's container
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

        // Define the local resources required
        Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
        // Lets assume the jar we need for our ApplicationMaster is available in
        // HDFS at a certain known path to us and we want to make it available to
        // the ApplicationMaster in the launched container
        if (job.getJar() == null) {
            throw new IllegalArgumentException("Jar must be set in order to run the application!");
        }

        Path jarPath = new Path(job.getJar());
        jarPath = fs.makeQualified(jarPath);
        getConf().set("bsp.jar", jarPath.makeQualified(fs.getUri(), jarPath).toString());

        FileStatus jarStatus = fs.getFileStatus(jarPath);
        LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
        amJarRsrc.setType(LocalResourceType.FILE);
        amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(jarPath));
        amJarRsrc.setTimestamp(jarStatus.getModificationTime());
        amJarRsrc.setSize(jarStatus.getLen());

        // this creates a symlink in the working directory
        localResources.put(YARNBSPConstants.APP_MASTER_JAR_PATH, amJarRsrc);

        // add hama related jar files to localresources for container
        List<File> hamaJars;
        if (System.getProperty("hama.home.dir") != null)
            hamaJars = localJarfromPath(System.getProperty("hama.home.dir"));
        else
            hamaJars = localJarfromPath(getConf().get("hama.home.dir"));
        String hamaPath = getSystemDir() + "/hama";
        for (File fileEntry : hamaJars) {
            addToLocalResources(fs, fileEntry.getCanonicalPath(), hamaPath, fileEntry.getName(),
                    localResources);
        }

        // Set the local resources into the launch context
        amContainer.setLocalResources(localResources);

        // Set up the environment needed for the launch context
        Map<String, String> env = new HashMap<String, String>();
        // Assuming our classes or jars are available as local resources in the
        // working directory from which the command will be run, we need to append
        // "." to the path.
        // By default, all the hadoop specific classpaths will already be available
        // in $CLASSPATH, so we should be careful not to overwrite it.
        StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$())
                .append(File.pathSeparatorChar).append("./*");
        for (String c : yarnConf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
                YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
            classPathEnv.append(File.pathSeparatorChar);
            classPathEnv.append(c.trim());
        }

        env.put(YARNBSPConstants.HAMA_YARN_LOCATION, jarPath.toUri().toString());
        env.put(YARNBSPConstants.HAMA_YARN_SIZE, Long.toString(jarStatus.getLen()));
        env.put(YARNBSPConstants.HAMA_YARN_TIMESTAMP, Long.toString(jarStatus.getModificationTime()));

        env.put(YARNBSPConstants.HAMA_LOCATION, hamaPath);
        env.put("CLASSPATH", classPathEnv.toString());
        amContainer.setEnvironment(env);

        // Set the necessary command to execute on the allocated container
        Vector<CharSequence> vargs = new Vector<CharSequence>(5);
        vargs.add("${JAVA_HOME}/bin/java");
        vargs.add("-cp " + classPathEnv + "");
        vargs.add(ApplicationMaster.class.getCanonicalName());
        vargs.add(submitJobFile.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());

        vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/hama-appmaster.stdout");
        vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/hama-appmaster.stderr");

        // Get final commmand
        StringBuilder command = new StringBuilder();
        for (CharSequence str : vargs) {
            command.append(str).append(" ");
        }

        List<String> commands = new ArrayList<String>();
        commands.add(command.toString());
        amContainer.setCommands(commands);

        LOG.debug("Start command: " + command);

        Resource capability = Records.newRecord(Resource.class);
        // we have at least 3 threads, which comsumes 1mb each, for each bsptask and
        // a base usage of 100mb
        capability.setMemory(3 * job.getNumBspTask() + getConf().getInt("hama.appmaster.memory.mb", 100));
        LOG.info("Set memory for the application master to " + capability.getMemory() + "mb!");

        // Set the container launch content into the ApplicationSubmissionContext
        appContext.setResource(capability);

        // Setup security tokens
        if (UserGroupInformation.isSecurityEnabled()) {
            // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce
            Credentials credentials = new Credentials();
            String tokenRenewer = yarnConf.get(YarnConfiguration.RM_PRINCIPAL);
            if (tokenRenewer == null || tokenRenewer.length() == 0) {
                throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
            }

            // For now, only getting tokens for the default file-system.
            final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
            if (tokens != null) {
                for (Token<?> token : tokens) {
                    LOG.info("Got dt for " + fs.getUri() + "; " + token);
                }
            }
            DataOutputBuffer dob = new DataOutputBuffer();
            credentials.writeTokenStorageToStream(dob);
            ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
            amContainer.setTokens(fsTokens);
        }

        appContext.setAMContainerSpec(amContainer);

        // Create the request to send to the ApplicationsManager
        ApplicationId appId = appContext.getApplicationId();
        yarnClient.submitApplication(appContext);

        return monitorApplication(appId) ? new NetworkedJob() : null;
    } catch (YarnException e) {
        e.printStackTrace();
        return null;
    }
}

---

From source file:org.apache.hama.bsp.YARNBSPJobClient.java

License:Apache License

private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String fileName,
        Map<String, LocalResource> localResources) throws IOException {
    Path dstPath = new Path(fileDstPath, fileName);
    dstPath = fs.makeQualified(dstPath);
    fs.copyFromLocalFile(false, true, new Path(fileSrcPath), dstPath);
    FileStatus fileStatus = fs.getFileStatus(dstPath);
    LocalResource localRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dstPath.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, fileStatus.getLen(),
            fileStatus.getModificationTime());
    localResources.put(fileName, localRsrc);
}

---

From source file:org.apache.helix.provisioning.yarn.AppLauncher.java

License:Apache License

private LocalResource setupLocalResource(FileSystem fs, Path dst) throws Exception {
    URI uri = dst.toUri();/*from   w  w  w .  jav a 2 s.co  m*/
    String extension = uri.getPath().substring(uri.getPath().lastIndexOf(".") + 1);
    FileStatus destStatus = fs.getFileStatus(dst);
    LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
    // Set the type of resource - file or archive
    // archives are untarred at destination
    // we don't need the jar file to be untarred for now
    if (isArchive(extension)) {
        amJarRsrc.setType(LocalResourceType.ARCHIVE);
    } else {
        amJarRsrc.setType(LocalResourceType.FILE);
    }
    // Set visibility of the resource
    // Setting to most private option
    amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    // Set the resource to be copied over
    amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    // Set timestamp and length of file so that the framework
    // can do basic sanity checks for the local resource
    // after it has been copied over to ensure it is the same
    // resource the client intended to use with the application
    amJarRsrc.setTimestamp(destStatus.getModificationTime());
    amJarRsrc.setSize(destStatus.getLen());
    return amJarRsrc;
}

---

From source file:org.apache.helix.provisioning.yarn.YarnProvisioner.java

License:Apache License

private ContainerLaunchContext createLaunchContext(ContainerId containerId, Container container,
        Participant participant) throws Exception {

    ContainerLaunchContext participantContainer = Records.newRecord(ContainerLaunchContext.class);

    // Map<String, String> envs = System.getenv();
    String appName = applicationMasterConfig.getAppName();
    int appId = applicationMasterConfig.getAppId();
    String serviceName = _resourceConfig.getId().stringify();
    String serviceClasspath = applicationMasterConfig.getClassPath(serviceName);
    String mainClass = applicationMasterConfig.getMainClass(serviceName);
    String zkAddress = applicationMasterConfig.getZKAddress();

    // set the localresources needed to launch container
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LocalResource servicePackageResource = Records.newRecord(LocalResource.class);
    YarnConfiguration conf = new YarnConfiguration();
    FileSystem fs;/* w w  w .j av a  2s  .co m*/
    fs = FileSystem.get(conf);
    String pathSuffix = appName + "/" + appId + "/" + serviceName + ".tar";
    Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
    FileStatus destStatus = fs.getFileStatus(dst);

    // Set the type of resource - file or archive
    // archives are untarred at destination
    // we don't need the jar file to be untarred for now
    servicePackageResource.setType(LocalResourceType.ARCHIVE);
    // Set visibility of the resource
    // Setting to most private option
    servicePackageResource.setVisibility(LocalResourceVisibility.APPLICATION);
    // Set the resource to be copied over
    servicePackageResource.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    // Set timestamp and length of file so that the framework
    // can do basic sanity checks for the local resource
    // after it has been copied over to ensure it is the same
    // resource the client intended to use with the application
    servicePackageResource.setTimestamp(destStatus.getModificationTime());
    servicePackageResource.setSize(destStatus.getLen());
    LOG.info("Setting local resource:" + servicePackageResource + " for service" + serviceName);
    localResources.put(serviceName, servicePackageResource);

    // Set local resource info into app master container launch context
    participantContainer.setLocalResources(localResources);

    // Set the necessary security tokens as needed
    // amContainer.setContainerTokens(containerToken);

    // Set the env variables to be setup in the env where the application master will be run
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();
    env.put(serviceName, dst.getName());
    // Add AppMaster.jar location to classpath
    // At some point we should not be required to add
    // the hadoop specific classpaths to the env.
    // It should be provided out of the box.
    // For now setting all required classpaths including
    // the classpath to "." for the application jar
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar)
            .append("./*");
    classPathEnv.append(File.pathSeparatorChar);
    classPathEnv.append(serviceClasspath);
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");
    LOG.info("Setting classpath for service:\n" + classPathEnv.toString());
    env.put("CLASSPATH", classPathEnv.toString());

    participantContainer.setEnvironment(env);

    if (applicationMaster.allTokens != null) {
        LOG.info("Setting tokens: " + applicationMaster.allTokens);
        participantContainer.setTokens(applicationMaster.allTokens);
    }

    // Set the necessary command to execute the application master
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    // Set java executable command
    LOG.info("Setting up app master command");
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    // Set Xmx based on am memory size
    vargs.add("-Xmx" + 4096 + "m");
    // Set class name
    vargs.add(ParticipantLauncher.class.getCanonicalName());
    // Set params for container participant
    vargs.add("--zkAddress " + zkAddress);
    vargs.add("--cluster " + appName);
    vargs.add("--participantId " + participant.getId().stringify());
    vargs.add("--participantClass " + mainClass);

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up  container launch command " + command.toString() + " with arguments \n"
            + vargs);
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    participantContainer.setCommands(commands);
    return participantContainer;
}

---

From source file:org.apache.hoya.tools.CoreFileSystem.java

License:Apache License

/**
 * Create an AM resource from the/* w w  w  .  j a  v  a 2  s  .  c  o m*/
 *
 * @param destPath     dest path in filesystem
 * @param resourceType resource type
 * @return the resource set up wih application-level visibility and the
 * timestamp & size set from the file stats.
 */
public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
    FileStatus destStatus = fileSystem.getFileStatus(destPath);
    LocalResource amResource = Records.newRecord(LocalResource.class);
    amResource.setType(resourceType);
    // Set visibility of the resource
    // Setting to most private option
    amResource.setVisibility(LocalResourceVisibility.APPLICATION);
    // Set the resource to be copied over
    amResource.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
    // Set timestamp and length of file so that the framework
    // can do basic sanity checks for the local resource
    // after it has been copied over to ensure it is the same
    // resource the client intended to use with the application
    amResource.setTimestamp(destStatus.getModificationTime());
    amResource.setSize(destStatus.getLen());
    return amResource;
}

---

From source file:org.apache.ignite.yarn.utils.IgniteYarnUtils.java

License:Apache License

/**
 * @param file Path./*  w  w w . ja v a  2 s .co m*/
 * @param fs File system.
 * @param type Local resource type.
 * @throws Exception If failed.
 */
public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type) throws Exception {
    LocalResource resource = Records.newRecord(LocalResource.class);

    file = fs.makeQualified(file);

    FileStatus stat = fs.getFileStatus(file);

    resource.setResource(ConverterUtils.getYarnUrlFromPath(file));
    resource.setSize(stat.getLen());
    resource.setTimestamp(stat.getModificationTime());
    resource.setType(type);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);

    return resource;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• »