Example usage for org.apache.hadoop.yarn.api ApplicationConstants LOG_DIR_EXPANSION_VAR

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api ApplicationConstants LOG_DIR_EXPANSION_VAR.

Prototype

String LOG_DIR_EXPANSION_VAR

To view the source code for org.apache.hadoop.yarn.api ApplicationConstants LOG_DIR_EXPANSION_VAR.

Click Source Link

Document

The temporary environmental variable for container log directory.

Usage

From source file:org.apache.slider.providers.accumulo.AccumuloProviderService.java

License:Apache License

@Override
public void buildContainerLaunchContext(ContainerLauncher launcher, AggregateConf instanceDefinition,
        Container container, String role, SliderFileSystem fileSystem, Path generatedConfPath,
        MapOperations resourceComponent, MapOperations appComponent, Path containerTmpDirPath)
        throws IOException, SliderException {

    this.fileSystem = fileSystem;

    // Set the environment
    launcher.putEnv(SliderUtils.buildEnvMap(appComponent));

    Map<String, String> env = SliderUtils.buildEnvMap(appComponent);
    launcher.setEnv(ACCUMULO_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    ConfTreeOperations appConf = instanceDefinition.getAppConfOperations();
    String hadoop_home = ApplicationConstants.Environment.HADOOP_COMMON_HOME.$();
    MapOperations appConfGlobal = appConf.getGlobalOptions();
    hadoop_home = appConfGlobal.getOption(OPTION_HADOOP_HOME, hadoop_home);
    launcher.setEnv(HADOOP_HOME, hadoop_home);
    launcher.setEnv(HADOOP_PREFIX, hadoop_home);

    // By not setting ACCUMULO_HOME, this will cause the Accumulo script to
    // compute it on its own to an absolute path.

    launcher.setEnv(ACCUMULO_CONF_DIR,//from  w w w.j ava2 s  .  com
            ProviderUtils.convertToAppRelativePath(SliderKeys.PROPAGATED_CONF_DIR_NAME));
    launcher.setEnv(ZOOKEEPER_HOME, appConfGlobal.getMandatoryOption(OPTION_ZK_HOME));

    //local resources

    //add the configuration resources
    launcher.addLocalResources(
            fileSystem.submitDirectory(generatedConfPath, SliderKeys.PROPAGATED_CONF_DIR_NAME));

    //Add binaries
    //now add the image if it was set
    String imageURI = instanceDefinition.getInternalOperations()
            .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
    fileSystem.maybeAddImagePath(launcher.getLocalResources(), imageURI);

    CommandLineBuilder commandLine = new CommandLineBuilder();

    String heap = "-Xmx" + appComponent.getOption(RoleKeys.JVM_HEAP, DEFAULT_JVM_HEAP);
    String opt = "ACCUMULO_OTHER_OPTS";
    if (SliderUtils.isSet(heap)) {
        /* JDK7
                
              switch (role) {
                case AccumuloKeys.ROLE_MASTER:
                  opt = "ACCUMULO_MASTER_OPTS";
                  break;
                case AccumuloKeys.ROLE_TABLET:
                  opt = "ACCUMULO_TSERVER_OPTS";
                  break;
                case AccumuloKeys.ROLE_MONITOR:
                  opt = "ACCUMULO_MONITOR_OPTS";
                  break;
                case AccumuloKeys.ROLE_GARBAGE_COLLECTOR:
                  opt = "ACCUMULO_GC_OPTS";
                  break;
              }
        */
        if (AccumuloKeys.ROLE_MASTER.equals(role)) {
            opt = "ACCUMULO_MASTER_OPTS";
        } else if (AccumuloKeys.ROLE_TABLET.equals(role)) {
            opt = "ACCUMULO_TSERVER_OPTS";
        } else if (AccumuloKeys.ROLE_MONITOR.equals(role)) {
            opt = "ACCUMULO_MONITOR_OPTS";
        } else if (AccumuloKeys.ROLE_GARBAGE_COLLECTOR.equals(role)) {
            opt = "ACCUMULO_GC_OPTS";
        }
        launcher.setEnv(opt, heap);
    }

    //this must stay relative if it is an image
    commandLine.add(providerUtils.buildPathToScript(instanceDefinition, "bin", "accumulo"));

    //role is translated to the accumulo one
    commandLine.add(AccumuloRoles.serviceForRole(role));

    // Add any role specific arguments to the command line
    String additionalArgs = ProviderUtils.getAdditionalArgs(appComponent);
    if (!StringUtils.isBlank(additionalArgs)) {
        commandLine.add(additionalArgs);
    }

    commandLine.addOutAndErrFiles(role + "-out.txt", role + "-err.txt");

    launcher.addCommand(commandLine.build());
}

---

From source file:org.apache.slider.providers.accumulo.AccumuloProviderService.java

License:Apache License

public List<String> buildProcessCommandList(AggregateConf instance, File confDir, Map<String, String> env,
        String... commands) throws IOException, SliderException {
    env.put(ACCUMULO_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    String hadoop_home = System.getenv(HADOOP_HOME);
    MapOperations globalOptions = instance.getAppConfOperations().getGlobalOptions();
    hadoop_home = globalOptions.getOption(OPTION_HADOOP_HOME, hadoop_home);
    if (hadoop_home == null) {
        throw new BadConfigException("Undefined env variable/config option: " + HADOOP_HOME);
    }/*w ww.ja  va2  s  .  c o  m*/
    ProviderUtils.validatePathReferencesLocalDir("HADOOP_HOME", hadoop_home);
    env.put(HADOOP_HOME, hadoop_home);
    env.put(HADOOP_PREFIX, hadoop_home);
    //buildup accumulo home env variable to be absolute or relative
    String accumulo_home = providerUtils.buildPathToHomeDir(instance, "bin", "accumulo");
    File image = new File(accumulo_home);
    String accumuloPath = image.getAbsolutePath();
    env.put(ACCUMULO_HOME, accumuloPath);
    ProviderUtils.validatePathReferencesLocalDir("ACCUMULO_HOME", accumuloPath);
    env.put(ACCUMULO_CONF_DIR, confDir.getAbsolutePath());
    String zkHome = globalOptions.getMandatoryOption(OPTION_ZK_HOME);
    ProviderUtils.validatePathReferencesLocalDir("ZOOKEEPER_HOME", zkHome);

    env.put(ZOOKEEPER_HOME, zkHome);

    String accumuloScript = AccumuloClientProvider.buildScriptBinPath(instance);
    List<String> launchSequence = new ArrayList<String>(8);
    launchSequence.add(0, accumuloScript);
    Collections.addAll(launchSequence, commands);
    return launchSequence;
}

---

From source file:org.apache.slider.providers.agent.AgentProviderService.java

License:Apache License

@Override
public void buildContainerLaunchContext(ContainerLauncher launcher, AggregateConf instanceDefinition,
        Container container, String role, SliderFileSystem fileSystem, Path generatedConfPath,
        MapOperations resourceComponent, MapOperations appComponent, Path containerTmpDirPath)
        throws IOException, SliderException {

    String appDef = instanceDefinition.getAppConfOperations().getGlobalOptions()
            .getMandatoryOption(AgentKeys.APP_DEF);

    initializeApplicationConfiguration(instanceDefinition, fileSystem);

    log.info("Build launch context for Agent");
    log.debug(instanceDefinition.toString());

    // Set the environment
    launcher.putEnv(SliderUtils.buildEnvMap(appComponent));

    String workDir = ApplicationConstants.Environment.PWD.$();
    launcher.setEnv("AGENT_WORK_ROOT", workDir);
    log.info("AGENT_WORK_ROOT set to {}", workDir);
    String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR;
    launcher.setEnv("AGENT_LOG_ROOT", logDir);
    log.info("AGENT_LOG_ROOT set to {}", logDir);
    if (System.getenv(HADOOP_USER_NAME) != null) {
        launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
    }//w w w  . j  a v  a  2  s. c om
    // for 2-Way SSL
    launcher.setEnv(SLIDER_PASSPHRASE, instanceDefinition.getPassphrase());

    //local resources

    // TODO: Should agent need to support App Home
    String scriptPath = new File(AgentKeys.AGENT_MAIN_SCRIPT_ROOT, AgentKeys.AGENT_MAIN_SCRIPT).getPath();
    String appHome = instanceDefinition.getAppConfOperations().getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
    if (SliderUtils.isSet(appHome)) {
        scriptPath = new File(appHome, AgentKeys.AGENT_MAIN_SCRIPT).getPath();
    }

    // set PYTHONPATH
    List<String> pythonPaths = new ArrayList<String>();
    pythonPaths.add(AgentKeys.AGENT_MAIN_SCRIPT_ROOT);
    String pythonPath = StringUtils.join(File.pathSeparator, pythonPaths);
    launcher.setEnv(PYTHONPATH, pythonPath);
    log.info("PYTHONPATH set to {}", pythonPath);

    Path agentImagePath = null;
    String agentImage = instanceDefinition.getInternalOperations()
            .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
    if (SliderUtils.isUnset(agentImage)) {
        agentImagePath = new Path(new Path(
                new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR),
                        container.getId().getApplicationAttemptId().getApplicationId().toString()),
                AgentKeys.PROVIDER_AGENT), SliderKeys.AGENT_TAR);
    } else {
        agentImagePath = new Path(agentImage);
    }

    // TODO: throw exception when agent tarball is not available

    if (fileSystem.getFileSystem().exists(agentImagePath)) {
        LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE);
        launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes);
    } else {
        log.error("Required agent image slider-agent.tar.gz is unavailable.");
    }

    log.info("Using {} for agent.", scriptPath);
    LocalResource appDefRes = fileSystem.createAmResource(
            fileSystem.getFileSystem().resolvePath(new Path(appDef)), LocalResourceType.ARCHIVE);
    launcher.addLocalResource(AgentKeys.APP_DEFINITION_DIR, appDefRes);

    String agentConf = instanceDefinition.getAppConfOperations().getGlobalOptions()
            .getOption(AgentKeys.AGENT_CONF, "");
    if (SliderUtils.isSet(agentConf)) {
        LocalResource agentConfRes = fileSystem.createAmResource(
                fileSystem.getFileSystem().resolvePath(new Path(agentConf)), LocalResourceType.FILE);
        launcher.addLocalResource(AgentKeys.AGENT_CONFIG_FILE, agentConfRes);
    }

    String agentVer = instanceDefinition.getAppConfOperations().getGlobalOptions()
            .getOption(AgentKeys.AGENT_VERSION, null);
    if (agentVer != null) {
        LocalResource agentVerRes = fileSystem.createAmResource(
                fileSystem.getFileSystem().resolvePath(new Path(agentVer)), LocalResourceType.FILE);
        launcher.addLocalResource(AgentKeys.AGENT_VERSION_FILE, agentVerRes);
    }

    if (SliderUtils.isHadoopClusterSecure(getConfig())) {
        localizeServiceKeytabs(launcher, instanceDefinition, fileSystem);
    }

    MapOperations amComponent = instanceDefinition.getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM);
    boolean twoWayEnabled = amComponent != null
            ? Boolean.valueOf(amComponent.getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false))
            : false;
    if (twoWayEnabled) {
        localizeContainerSSLResources(launcher, container, fileSystem);
    }

    //add the configuration resources
    launcher.addLocalResources(
            fileSystem.submitDirectory(generatedConfPath, SliderKeys.PROPAGATED_CONF_DIR_NAME));

    String label = getContainerLabel(container, role);
    CommandLineBuilder operation = new CommandLineBuilder();

    String pythonExec = instanceDefinition.getAppConfOperations().getGlobalOptions()
            .getOption(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH, AgentKeys.PYTHON_EXE);

    operation.add(pythonExec);

    operation.add(scriptPath);
    operation.add(ARG_LABEL, label);
    operation.add(ARG_ZOOKEEPER_QUORUM);
    operation.add(getClusterOptionPropertyValue(OptionKeys.ZOOKEEPER_QUORUM));
    operation.add(ARG_ZOOKEEPER_REGISTRY_PATH);
    operation.add(getZkRegistryPath());

    String debugCmd = agentLaunchParameter.getNextLaunchParameter(role);
    if (SliderUtils.isSet(debugCmd)) {
        operation.add(ARG_DEBUG);
        operation.add(debugCmd);
    }

    operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + AgentKeys.AGENT_OUT_FILE + " 2>&1");

    launcher.addCommand(operation.build());

    // initialize the component instance state
    getComponentStatuses().put(label, new ComponentInstanceState(role, container.getId(),
            getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
}

---

From source file:org.apache.sysml.yarn.DMLYarnClient.java

License:Apache License

private String constructAMCommand(String[] args, DMLConfig conf) {
    //start command
    StringBuilder command = new StringBuilder();
    command.append(Environment.JAVA_HOME.$$() + "/bin/java");

    //add client jvm arguments (concatenation of HADOOP_CLIENT_OPTS and HADOOP_OPTS)
    if (System.getenv().containsKey(JVMOPTS_ENV_CONST)) {
        String externalArgs = System.getenv(JVMOPTS_ENV_CONST);
        //safe parsing and replacement of redundant Xmx, Xms, Xmn
        if (externalArgs != null) {
            String[] parts = externalArgs.split(" ");
            for (int i = 0; i < parts.length; i++)
                if (!(parts[i].startsWith("-Xmx") || parts[i].startsWith("-Xms")
                        || parts[i].startsWith("-Xmn"))) {
                    command.append(" ");
                    command.append(parts[i]);
                }//from w  w w  .  java2  s .com
        }
    }

    //add jvm heap configuration (specify xmn for default gcpolicy:gencon)
    int memHeap = conf.getIntValue(DMLConfig.YARN_APPMASTERMEM);
    command.append(" -Xmx" + memHeap + "m");
    command.append(" -Xms" + memHeap + "m");
    command.append(" -Xmn" + (int) (memHeap / 10) + "m");
    command.append(' ');
    command.append(DMLAppMaster.class.getName());

    //add command line args (modify script and config file path)
    for (int i = 0; i < _args.length; i++) {
        String arg = _args[i];
        command.append(' ');
        if (i > 0 && _args[i - 1].equals("-f")) {
            command.append(_hdfsDMLScript);
            command.append(" -config " + _hdfsDMLConfig);
        } else if (_args[i].startsWith("-config")) {
            //ignore because config added with -f
        } else
            command.append(arg);
    }

    //setup stdout and stderr logs 
    command.append(" 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    command.append(" 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    return command.toString();
}

---

From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java

License:Apache License

private ApplicationAttemptId allocateAndLaunchQueryMaster(QueryInProgress queryInProgress)
        throws IOException, YarnException {
    QueryId queryId = queryInProgress.getQueryId();
    ApplicationId appId = ApplicationIdUtils.queryIdToAppId(queryId);

    LOG.info("Allocate and launch ApplicationMaster for QueryMaster: queryId=" + queryId + ", appId=" + appId);

    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);//from w  ww  .j  a  va  2  s  .c o  m
    // set the application name
    appContext.setApplicationName("Tajo");

    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(5);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("default");

    ContainerLaunchContext commonContainerLaunchContext = YarnContainerProxy
            .createCommonContainerLaunchContext(masterContext.getConf(), queryId.toString(), true);

    // Setup environment by cloning from common env.
    Map<String, String> env = commonContainerLaunchContext.getEnvironment();
    Map<String, String> myEnv = new HashMap<String, String>(env.size());
    myEnv.putAll(env);

    ////////////////////////////////////////////////////////////////////////////
    // Set the local resources
    ////////////////////////////////////////////////////////////////////////////
    // Set the necessary command to execute the application master
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    // Set java executable command
    //LOG.info("Setting up app master command");
    vargs.add("${JAVA_HOME}" + "/bin/java");
    // Set Xmx based on am memory size
    String jvmOptions = masterContext.getConf().get("tajo.rm.yarn.querymaster.jvm.option", "-Xmx2000m");

    for (String eachToken : jvmOptions.split((" "))) {
        vargs.add(eachToken);
    }
    // Set Remote Debugging
    //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) {
    //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005");
    //}
    // Set class name
    vargs.add(TajoWorker.class.getCanonicalName());
    vargs.add("qm");
    vargs.add(queryId.toString()); // queryId
    vargs.add(masterContext.getTajoMasterService().getBindAddress().getHostName() + ":"
            + masterContext.getTajoMasterService().getBindAddress().getPort());

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up QueryMasterRunner command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());

    final Resource resource = Records.newRecord(Resource.class);
    // TODO - get default value from conf
    resource.setMemory(2000);
    resource.setVirtualCores(1);

    Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();

    ContainerLaunchContext masterContainerContext = BuilderUtils.newContainerLaunchContext(
            commonContainerLaunchContext.getLocalResources(), myEnv, commands, myServiceData, null,
            new HashMap<ApplicationAccessType, String>(2));

    appContext.setAMContainerSpec(masterContainerContext);

    LOG.info("Submitting QueryMaster to ResourceManager");
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED));
    ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId();

    LOG.info("Launching QueryMaster with appAttemptId: " + attemptId);

    return attemptId;
}

---

From source file:org.apache.tajo.master.YarnContainerProxy.java

License:Apache License

public ContainerLaunchContext createContainerLaunchContext(
        ContainerLaunchContext commonContainerLaunchContext) {
    // Setup environment by cloning from common env.
    Map<String, String> env = commonContainerLaunchContext.getEnvironment();
    Map<String, String> myEnv = new HashMap<String, String>(env.size());
    myEnv.putAll(env);/*  w  w  w.  ja va 2  s .  c  o  m*/

    // Duplicate the ByteBuffers for access by multiple containers.
    Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
    for (Map.Entry<String, ByteBuffer> entry : commonContainerLaunchContext.getServiceData().entrySet()) {
        myServiceData.put(entry.getKey(), entry.getValue().duplicate());
    }

    ////////////////////////////////////////////////////////////////////////////
    // Set the local resources
    ////////////////////////////////////////////////////////////////////////////
    // Set the necessary command to execute the application master
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    // Set java executable command
    //LOG.info("Setting up app master command");
    vargs.add("${JAVA_HOME}" + "/bin/java");
    // Set Xmx based on am memory size
    vargs.add("-Xmx2000m");
    // Set Remote Debugging
    //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) {
    //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005");
    //}
    // Set class name
    //vargs.add(getRunnerClass());
    vargs.add(TajoWorker.class.getCanonicalName());
    vargs.add("tr"); //workerMode
    vargs.add(getId()); // subqueryId
    vargs.add(containerMgrAddress); // nodeId
    vargs.add(containerID.toString()); // containerId
    Vector<CharSequence> taskParams = getTaskParams();
    if (taskParams != null) {
        vargs.addAll(taskParams);
    }

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up TaskRunner command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());

    return BuilderUtils.newContainerLaunchContext(commonContainerLaunchContext.getLocalResources(), myEnv,
            commands, myServiceData, null, new HashMap<ApplicationAccessType, String>());
}

---

From source file:org.apache.tajo.yarn.command.LaunchCommand.java

License:Apache License

private void setupAMCommand(ContainerLaunchContext amContainer) {
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    // Set java executable command
    LOG.info("Setting up app master command");
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    // Set Xmx based on am memory size
    vargs.add("-Xmx32m");
    // Set class name
    vargs.add(ApplicationMaster.class.getName());

    // Set params for Application Master
    vargs.add("--qm_memory " + String.valueOf(qmMemory));
    vargs.add("--qm_vcores " + String.valueOf(qmVCores));
    vargs.add("--tr_memory " + String.valueOf(trMemory));
    vargs.add("--tr_vcores " + String.valueOf(trVCores));
    // Set params for Application Master
    vargs.add("--worker_memory " + String.valueOf(workerMemory));
    vargs.add("--worker_vcores " + String.valueOf(workerVCores));

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }/*from w  w w .j a  v  a2s . c o  m*/

    LOG.info("Completed setting up app master command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    amContainer.setCommands(commands);
}

---

From source file:org.apache.tajo.yarn.command.LaunchCommand.java

License:Apache License

private void setupEnv(ContainerLaunchContext amContainer) throws IOException {
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();

    // Add AppMaster.jar location to classpath
    // At some point we should not be required to add
    // the hadoop specific classpaths to the env.
    // It should be provided out of the box.
    // For now setting all required classpaths including
    // the classpath to "." for the application jar
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar)
            .append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());//from   ww  w.j  ava  2  s .  c  om
    }

    classPathEnv.append(File.pathSeparatorChar).append("./").append(libDir).append("/*");

    classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");

    // add the runtime classpath needed for tests to work
    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(':');
        classPathEnv.append(System.getProperty("java.class.path"));
    }

    env.put("CLASSPATH", classPathEnv.toString());
    env.put(Constants.TAJO_ARCHIVE_PATH, tajoArchive);
    env.put(Constants.TAJO_ARCHIVE_ROOT, "tajo/" + getTajoRootInArchive(tajoArchive));
    env.put(Constants.TAJO_HOME, "$PWD/${" + Constants.TAJO_ARCHIVE_ROOT + "}");
    env.put(Constants.TAJO_CONF_DIR, "$PWD/conf");
    env.put(Constants.TAJO_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    env.put(Constants.TAJO_CLASSPATH, "/export/apps/hadoop/site/lib/*:$PWD/" + libDir + "/*");
    amContainer.setEnvironment(env);
}

---

From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java

License:Apache License

@Override
public ContainerLaunchContext getLaunchContext(Container container) throws IOException {
    // create a container launch context
    ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
    UserGroupInformation user = UserGroupInformation.getCurrentUser();
    try {/* w ww .j  ava2  s .  c o  m*/
        Credentials credentials = user.getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        launchContext.setTokens(securityTokens);
    } catch (IOException e) {
        LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage());
    }

    FileSystem fs = FileSystem.get(appContext.getConfiguration());

    // Set the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    String suffix = "Tajo" + "/" + appContext.getApplicationId().getId();
    Path parentPath = new Path(fs.getHomeDirectory(), suffix);

    // tar ball
    Path archivePath = new Path(parentPath, System.getenv(Constants.TAJO_ARCHIVE_PATH));
    FileStatus archiveFs = fs.getFileStatus(archivePath);
    LocalResource archiveRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(archivePath.toUri()),
            LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, archiveFs.getLen(),
            archiveFs.getModificationTime());
    localResources.put("tajo", archiveRsrc);

    Configuration tajoWorkerConf = new Configuration(false);
    tajoWorkerConf.addResource(new Path("conf", "tajo-site.xml"));
    tajoWorkerConf.set(Constants.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, appContext.getMasterHost() + ":26001");
    tajoWorkerConf.set(Constants.CATALOG_ADDRESS, appContext.getMasterHost() + ":26005");
    Path dst = new Path(parentPath, container.getId() + Path.SEPARATOR + "worker-conf");
    fs.mkdirs(dst);
    Path confFile = new Path(dst, "tajo-site.xml");
    FSDataOutputStream fdos = fs.create(confFile);
    tajoWorkerConf.writeXml(fdos);
    fdos.close();
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put("conf", scRsrc);
    launchContext.setLocalResources(localResources);

    // Set the environment
    setupEnv(launchContext);

    // Set the necessary command to execute on the allocated container
    Vector<CharSequence> vargs = new Vector<CharSequence>(5);

    // Set executable command
    // Set args for the shell command if any
    vargs.add("${" + Constants.TAJO_HOME + "}/bin/tajo");
    vargs.add("--config");
    vargs.add("${" + Constants.TAJO_CONF_DIR + "}");
    vargs.add("worker");
    // Add log redirect params
    // Add log redirect params
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    launchContext.setCommands(commands);
    return launchContext;
}

---

From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java

License:Apache License

private void setupEnv(ContainerLaunchContext amContainer) throws IOException {
    LOG.info("Set the environment for the worker");
    Map<String, String> env = new HashMap<String, String>();

    // Add AppMaster.jar location to classpath
    // At some point we should not be required to add
    // the hadoop specific classpaths to the env.
    // It should be provided out of the box.
    // For now setting all required classpaths including
    // the classpath to "." for the application jar
    StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$())
            .append(File.pathSeparatorChar).append("./*");

    env.put("CLASSPATH", classPathEnv.toString());
    env.put(Constants.TAJO_ARCHIVE_ROOT, System.getenv(Constants.TAJO_ARCHIVE_ROOT));
    env.put(Constants.TAJO_HOME, "$PWD/${" + Constants.TAJO_ARCHIVE_ROOT + "}");
    env.put(Constants.TAJO_CONF_DIR, "$PWD/conf");
    env.put(Constants.TAJO_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    env.put(Constants.TAJO_CLASSPATH, "/export/apps/hadoop/site/lib/*");
    amContainer.setEnvironment(env);/*from  www . ja  v a2 s.co  m*/
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• »