Example usage for org.apache.hadoop.yarn.api.records ContainerLaunchContext setCommands

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ContainerLaunchContext setCommands.

Prototype

@Public
@Stable
public abstract void setCommands(List<String> commands);

Source Link

Document

Add the list of commands for launching the container.

Usage

From source file:org.apache.hoya.providers.hbase.HBaseProviderService.java

License:Apache License

@Override
public void buildContainerLaunchContext(ContainerLaunchContext ctx, AggregateConf instanceDefinition,
        Container container, String role, HoyaFileSystem hoyaFileSystem, Path generatedConfPath,
        MapOperations resourceComponent, MapOperations appComponent, Path containerTmpDirPath)
        throws IOException, HoyaException {

    this.hoyaFileSystem = hoyaFileSystem;
    this.instanceDefinition = instanceDefinition;
    // Set the environment
    Map<String, String> env = HoyaUtils.buildEnvMap(appComponent);

    env.put(HBASE_LOG_DIR, providerUtils.getLogdir());

    env.put("PROPAGATED_CONFDIR",
            ApplicationConstants.Environment.PWD.$() + "/" + HoyaKeys.PROPAGATED_CONF_DIR_NAME);

    //local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    //add the configuration resources
    Map<String, LocalResource> confResources;
    confResources = hoyaFileSystem.submitDirectory(generatedConfPath, HoyaKeys.PROPAGATED_CONF_DIR_NAME);
    localResources.putAll(confResources);
    //Add binaries
    //now add the image if it was set
    String imageURI = instanceDefinition.getInternalOperations()
            .get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
    hoyaFileSystem.maybeAddImagePath(localResources, imageURI);
    ctx.setLocalResources(localResources);
    List<String> commands = new ArrayList<String>();

    CommandLineBuilder command = new CommandLineBuilder();

    String heap = appComponent.getOption(RoleKeys.JVM_HEAP, DEFAULT_JVM_HEAP);
    if (HoyaUtils.isSet(heap)) {
        String adjustedHeap = HoyaUtils.translateTrailingHeapUnit(heap);
        env.put("HBASE_HEAPSIZE", adjustedHeap);
    }//from ww w .j  a va2 s.  com

    String gcOpts = appComponent.getOption(RoleKeys.GC_OPTS, DEFAULT_GC_OPTS);
    if (HoyaUtils.isSet(gcOpts)) {
        env.put("SERVER_GC_OPTS", gcOpts);
    }

    //this must stay relative if it is an image
    command.add(providerUtils.buildPathToScript(instanceDefinition, "bin", HBaseKeys.HBASE_SCRIPT));
    //config dir is relative to the generated file
    command.add(ARG_CONFIG);
    command.add("$PROPAGATED_CONFDIR");

    String roleCommand;
    String logfile;
    //now look at the role
    if (ROLE_WORKER.equals(role)) {
        //role is region server
        roleCommand = REGION_SERVER;
        logfile = "/region-server.txt";
    } else if (ROLE_MASTER.equals(role)) {
        roleCommand = MASTER;

        logfile = "/master.txt";
    } else {
        throw new HoyaInternalStateException("Cannot start role %s", role);
    }

    command.add(roleCommand);
    command.add(ACTION_START);
    //log details
    command.addOutAndErrFiles(logfile, null);

    String cmdStr = command.build();

    commands.add(cmdStr);
    ctx.setCommands(commands);
    ctx.setEnvironment(env);

}

---

From source file:org.apache.ignite.yarn.ApplicationMaster.java

License:Apache License

/** {@inheritDoc} */
public synchronized void onContainersAllocated(List<Container> conts) {
    for (Container c : conts) {
        if (checkContainer(c)) {
            try {
                ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

                if (UserGroupInformation.isSecurityEnabled())
                    // Set the tokens to the newly allocated container:
                    ctx.setTokens(allTokens.duplicate());

                Map<String, String> env = new HashMap<>(System.getenv());

                env.put("IGNITE_TCP_DISCOVERY_ADDRESSES", getAddress(c.getNodeId().getHost()));

                if (props.jvmOpts() != null && !props.jvmOpts().isEmpty())
                    env.put("JVM_OPTS", props.jvmOpts());

                ctx.setEnvironment(env);

                Map<String, LocalResource> resources = new HashMap<>();

                resources.put("ignite", IgniteYarnUtils.setupFile(ignitePath, fs, LocalResourceType.ARCHIVE));
                resources.put("ignite-config.xml",
                        IgniteYarnUtils.setupFile(cfgPath, fs, LocalResourceType.FILE));

                if (props.licencePath() != null)
                    resources.put("gridgain-license.xml", IgniteYarnUtils
                            .setupFile(new Path(props.licencePath()), fs, LocalResourceType.FILE));

                if (props.userLibs() != null)
                    resources.put("libs",
                            IgniteYarnUtils.setupFile(new Path(props.userLibs()), fs, LocalResourceType.FILE));

                ctx.setLocalResources(resources);

                ctx.setCommands(Collections.singletonList(
                        (props.licencePath() != null ? "cp gridgain-license.xml ./ignite/*/ || true && " : "")
                                + "cp -r ./libs/* ./ignite/*/libs/ || true && " + "./ignite/*/bin/ignite.sh "
                                + "./ignite-config.xml" + " -J-Xmx" + ((int) props.memoryPerNode()) + "m"
                                + " -J-Xms" + ((int) props.memoryPerNode()) + "m"
                                + IgniteYarnUtils.YARN_LOG_OUT));

                log.log(Level.INFO, "Launching container: {0}.", c.getId());

                nmClient.startContainer(c, ctx);

                containers.put(c.getId(), new IgniteContainer(c.getId(), c.getNodeId(),
                        c.getResource().getVirtualCores(), c.getResource().getMemory()));
            } catch (Exception ex) {
                log.log(Level.WARNING, "Error launching container " + c.getId(), ex);
            }/* www . j  a v  a2 s  .c  om*/
        } else
            rmClient.releaseAssignedContainer(c.getId());
    }
}

---

From source file:org.apache.ignite.yarn.IgniteYarnClient.java

License:Apache License

/**
 * Main methods has one mandatory parameter and one optional parameter.
 *
 * @param args Path to jar mandatory parameter and property file is optional.
 *//*from  w  ww  .  j  av a2s  . c om*/
public static void main(String[] args) throws Exception {
    checkArguments(args);

    // Set path to app master jar.
    String pathAppMasterJar = args[0];

    ClusterProperties props = ClusterProperties.from(args.length == 2 ? args[1] : null);

    YarnConfiguration conf = new YarnConfiguration();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    FileSystem fs = FileSystem.get(conf);

    Path ignite;

    // Load ignite and jar
    if (props.ignitePath() == null)
        ignite = getIgnite(props, fs);
    else
        ignite = new Path(props.ignitePath());

    // Upload the jar file to HDFS.
    Path appJar = IgniteYarnUtils.copyLocalToHdfs(fs, pathAppMasterJar,
            props.igniteWorkDir() + File.separator + IgniteYarnUtils.JAR_NAME);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    amContainer.setCommands(Collections
            .singletonList(Environment.JAVA_HOME.$() + "/bin/java -Xmx512m " + ApplicationMaster.class.getName()
                    + IgniteYarnUtils.SPACE + ignite.toUri() + IgniteYarnUtils.YARN_LOG_OUT));

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = IgniteYarnUtils.setupFile(appJar, fs, LocalResourceType.FILE);

    amContainer.setLocalResources(Collections.singletonMap(IgniteYarnUtils.JAR_NAME, appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = props.toEnvs();

    setupAppMasterEnv(appMasterEnv, conf);

    amContainer.setEnvironment(appMasterEnv);

    // Setup security tokens
    if (UserGroupInformation.isSecurityEnabled()) {
        Credentials creds = new Credentials();

        String tokRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);

        if (tokRenewer == null || tokRenewer.length() == 0)
            throw new IOException("Master Kerberos principal for the RM is not set.");

        log.info("Found RM principal: " + tokRenewer);

        final Token<?> tokens[] = fs.addDelegationTokens(tokRenewer, creds);

        if (tokens != null)
            log.info("File system delegation tokens: " + Arrays.toString(tokens));

        amContainer.setTokens(IgniteYarnUtils.createTokenBuffer(creds));
    }

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(512);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName("ignition"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();

    yarnClient.submitApplication(appContext);

    log.log(Level.INFO, "Submitted application. Application id: {0}", appId);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();

    while (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING
            || appState == YarnApplicationState.SUBMITTED || appState == YarnApplicationState.ACCEPTED) {
        TimeUnit.SECONDS.sleep(1L);

        appReport = yarnClient.getApplicationReport(appId);

        if (appState != YarnApplicationState.ACCEPTED
                && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED)
            log.log(Level.INFO, "Application {0} is ACCEPTED.", appId);

        appState = appReport.getYarnApplicationState();
    }

    log.log(Level.INFO, "Application {0} is {1}.", new Object[] { appId, appState });
}

---

From source file:org.apache.reef.runtime.yarn.driver.unmanaged.UnmanagedAmTest.java

License:Apache License

@Test
public void testAmShutdown() throws IOException, YarnException {

    Assume.assumeTrue("This test requires a YARN Resource Manager to connect to",
            Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

    final YarnConfiguration yarnConfig = new YarnConfiguration();

    // Start YARN client and register the application

    final YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(yarnConfig);//from  ww w.jav a 2  s  . com
    yarnClient.start();

    final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
    containerContext.setCommands(Collections.<String>emptyList());
    containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
    containerContext.setEnvironment(Collections.<String, String>emptyMap());
    containerContext.setTokens(getTokens());

    final ApplicationSubmissionContext appContext = yarnClient.createApplication()
            .getApplicationSubmissionContext();
    appContext.setApplicationName("REEF_Unmanaged_AM_Test");
    appContext.setAMContainerSpec(containerContext);
    appContext.setUnmanagedAM(true);
    appContext.setQueue("default");

    final ApplicationId applicationId = appContext.getApplicationId();
    LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

    yarnClient.submitApplication(appContext);

    LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

    addToken(yarnClient.getAMRMToken(applicationId));

    // Start the AM

    final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000,
            this);
    rmClient.init(yarnConfig);
    rmClient.start();

    final NMClientAsync nmClient = new NMClientAsyncImpl(this);
    nmClient.init(yarnConfig);
    nmClient.start();

    final RegisterApplicationMasterResponse registration = rmClient
            .registerApplicationMaster(NetUtils.getHostname(), -1, null);

    LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

    LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

    // Shutdown the AM

    rmClient.stop();
    nmClient.stop();

    // Get the final application report

    final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
    final YarnApplicationState appState = appReport.getYarnApplicationState();
    final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

    LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] { applicationId,
            appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus });

    Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
    Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

    // Shutdown YARN client

    yarnClient.stop();
}

---

From source file:org.apache.reef.runtime.yarn.util.YarnTypes.java

License:Apache License

/**
 * @return a ContainerLaunchContext with the given commands and LocalResources.
 *///from w  w w . j  av a 2 s. c  o  m
public static final ContainerLaunchContext getContainerLaunchContext(final List<String> commands,
        final Map<String, LocalResource> localResources) {
    final ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setLocalResources(localResources);
    context.setCommands(commands);
    return context;
}

---

From source file:org.apache.samza.job.yarn.ContainerUtil.java

License:Apache License

protected void startContainer(Path packagePath, Container container, Map<String, String> env,
        final String cmd) {
    log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd });

    // set the local package so that the containers and app master are provisioned with it
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
    FileStatus fileStatus;/*from   www.  ja v  a2s  . c  o m*/
    try {
        fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
    } catch (IOException ioe) {
        log.error("IO Exception when accessing the package status from the filesystem", ioe);
        throw new SamzaException("IO Exception when accessing the package status from the filesystem");
    }

    packageResource.setResource(packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    ByteBuffer allTokens;
    // copy tokens (copied from dist shell example)
    try {
        Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);

        // now remove the AM->RM token so that containers cannot access it
        Iterator iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            TokenIdentifier token = ((Token) iter.next()).decodeIdentifier();
            if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    } catch (IOException ioe) {
        ioe.printStackTrace();
        throw new SamzaException("IO Exception when writing credentials to output buffer");
    }

    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(env);
    context.setTokens(allTokens.duplicate());
    context.setCommands(new ArrayList<String>() {
        {
            add(cmd);
        }
    });
    context.setLocalResources(Collections.singletonMap("__package", packageResource));

    log.debug("setting package to {}", packageResource);
    log.debug("setting context to {}", context);

    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
    startContainerRequest.setContainerLaunchContext(context);
    try {
        nmClient.startContainer(container, context);
    } catch (YarnException ye) {
        log.error("Received YarnException when starting container: " + container.getId(), ye);
        throw new SamzaException("Received YarnException when starting container: " + container.getId());
    } catch (IOException ioe) {
        log.error("Received IOException when starting container: " + container.getId(), ioe);
        throw new SamzaException("Received IOException when starting container: " + container.getId());
    }
}

---

From source file:org.apache.samza.job.yarn.refactor.YarnContainerRunner.java

License:Apache License

/**
 *    Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL
 *    specified by packagePath./*  www  . j  av a  2s.  c o  m*/
 */
private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd)
        throws SamzaContainerLaunchException {
    log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd });

    // set the local package so that the containers and app master are provisioned with it
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
    FileStatus fileStatus;
    try {
        fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
    } catch (IOException ioe) {
        log.error("IO Exception when accessing the package status from the filesystem", ioe);
        throw new SamzaContainerLaunchException(
                "IO Exception when accessing the package status from the filesystem");
    }

    packageResource.setResource(packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    ByteBuffer allTokens;
    // copy tokens (copied from dist shell example)
    try {
        Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);

        // now remove the AM->RM token so that containers cannot access it
        Iterator iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            TokenIdentifier token = ((Token) iter.next()).decodeIdentifier();
            if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    } catch (IOException ioe) {
        log.error("IOException when writing credentials.", ioe);
        throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer");
    }

    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(env);
    context.setTokens(allTokens.duplicate());
    context.setCommands(new ArrayList<String>() {
        {
            add(cmd);
        }
    });
    context.setLocalResources(Collections.singletonMap("__package", packageResource));

    log.debug("setting package to {}", packageResource);
    log.debug("setting context to {}", context);

    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
    startContainerRequest.setContainerLaunchContext(context);
    try {
        nmClient.startContainer(container, context);
    } catch (YarnException ye) {
        log.error("Received YarnException when starting container: " + container.getId(), ye);
        throw new SamzaContainerLaunchException(
                "Received YarnException when starting container: " + container.getId(), ye);
    } catch (IOException ioe) {
        log.error("Received IOException when starting container: " + container.getId(), ioe);
        throw new SamzaContainerLaunchException(
                "Received IOException when starting container: " + container.getId(), ioe);
    }
}

---

From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java

License:Apache License

/**
 * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL
 * specified by packagePath.//from ww  w  . j a  v  a2 s. c  o  m
 */
private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd)
        throws IOException {
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
    FileStatus fileStatus;
    fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
    packageResource.setResource(packageUrl);
    log.debug("Set package resource in YarnContainerRunner for {}", packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    ByteBuffer allTokens;
    // copy tokens to start the container
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);

    // now remove the AM->RM token so that containers cannot access it
    Iterator iter = credentials.getAllTokens().iterator();
    while (iter.hasNext()) {
        TokenIdentifier token = ((org.apache.hadoop.security.token.Token) iter.next()).decodeIdentifier();
        if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    Map<String, LocalResource> localResourceMap = new HashMap<>();
    localResourceMap.put("__package", packageResource);

    // include the resources from the universal resource configurations
    LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config),
            yarnConfiguration);
    localResourceMap.putAll(resourceMapper.getResourceMap());

    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(env);
    context.setTokens(allTokens.duplicate());
    context.setCommands(new ArrayList<String>() {
        {
            add(cmd);
        }
    });
    context.setLocalResources(localResourceMap);

    if (UserGroupInformation.isSecurityEnabled()) {
        Map<ApplicationAccessType, String> acls = yarnConfig.getYarnApplicationAcls();
        if (!acls.isEmpty()) {
            context.setApplicationACLs(acls);
        }
    }

    log.debug("Setting localResourceMap to {}", localResourceMap);
    log.debug("Setting context to {}", context);

    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
    startContainerRequest.setContainerLaunchContext(context);

    log.info(
            "Making an async start request for Container ID: {} on host: {} with local resource map: {} and context: {}",
            container.getId(), container.getNodeHttpAddress(), localResourceMap.toString(), context);
    nmClientAsync.startContainerAsync(container, context);
}

---

From source file:org.apache.samza.job.yarn.YarnContainerRunner.java

License:Apache License

/**
 *    Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL
 *    specified by packagePath.//from   w ww  .  j  av  a  2s. co m
 */
private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd)
        throws SamzaContainerLaunchException {
    log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd });

    // TODO: SAMZA-1144 remove the customized approach for package resource and use the common one.
    // But keep it now for backward compatibility.
    // set the local package so that the containers and app master are provisioned with it
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
    FileStatus fileStatus;
    try {
        fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
    } catch (IOException ioe) {
        log.error("IO Exception when accessing the package status from the filesystem", ioe);
        throw new SamzaContainerLaunchException(
                "IO Exception when accessing the package status from the filesystem");
    }

    packageResource.setResource(packageUrl);
    log.info("set package Resource in YarnContainerRunner for {}", packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    ByteBuffer allTokens;
    // copy tokens (copied from dist shell example)
    try {
        Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);

        // now remove the AM->RM token so that containers cannot access it
        Iterator iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            TokenIdentifier token = ((Token) iter.next()).decodeIdentifier();
            if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    } catch (IOException ioe) {
        log.error("IOException when writing credentials.", ioe);
        throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer");
    }

    Map<String, LocalResource> localResourceMap = new HashMap<>();
    localResourceMap.put("__package", packageResource);

    // include the resources from the universal resource configurations
    LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config),
            yarnConfiguration);
    localResourceMap.putAll(resourceMapper.getResourceMap());

    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(env);
    context.setTokens(allTokens.duplicate());
    context.setCommands(new ArrayList<String>() {
        {
            add(cmd);
        }
    });
    context.setLocalResources(localResourceMap);

    log.debug("setting localResourceMap to {}", localResourceMap);
    log.debug("setting context to {}", context);

    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
    startContainerRequest.setContainerLaunchContext(context);
    try {
        nmClient.startContainer(container, context);
    } catch (YarnException ye) {
        log.error("Received YarnException when starting container: " + container.getId(), ye);
        throw new SamzaContainerLaunchException(
                "Received YarnException when starting container: " + container.getId(), ye);
    } catch (IOException ioe) {
        log.error("Received IOException when starting container: " + container.getId(), ioe);
        throw new SamzaContainerLaunchException(
                "Received IOException when starting container: " + container.getId(), ioe);
    }
}

---

From source file:org.apache.tajo.yarn.command.LaunchCommand.java

License:Apache License

private void setupAMCommand(ContainerLaunchContext amContainer) {
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    // Set java executable command
    LOG.info("Setting up app master command");
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    // Set Xmx based on am memory size
    vargs.add("-Xmx32m");
    // Set class name
    vargs.add(ApplicationMaster.class.getName());

    // Set params for Application Master
    vargs.add("--qm_memory " + String.valueOf(qmMemory));
    vargs.add("--qm_vcores " + String.valueOf(qmVCores));
    vargs.add("--tr_memory " + String.valueOf(trMemory));
    vargs.add("--tr_vcores " + String.valueOf(trVCores));
    // Set params for Application Master
    vargs.add("--worker_memory " + String.valueOf(workerMemory));
    vargs.add("--worker_vcores " + String.valueOf(workerVCores));

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }/* w  w  w  .j  a  va 2 s . co m*/

    LOG.info("Completed setting up app master command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    amContainer.setCommands(commands);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• »