Example usage for org.apache.hadoop.yarn.client.api YarnClient init

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api YarnClient init.

Prototype

@Override
public void init(Configuration conf)

Source Link

Document

This invokes #serviceInit

Usage

From source file:yrun.YarnRunner.java

License:Apache License

public void execute() throws IOException, YarnException, InterruptedException {
    LOG.info("Using application path [" + _installPath + "]");
    Path jarPath = installThisJar(_installPath, _appJarFile);
    LOG.info("Driver installed [" + jarPath + "]");
    List<Path> installedArchivePathList = install(_installPath, _archivePathList);
    for (Path p : installedArchivePathList) {
        LOG.info("Archive installed [" + p + "]");
    }/*ww  w.  j  av  a2s  .  c  o  m*/

    YarnRunnerArgs yarnRunnerArgs = new YarnRunnerArgs();
    yarnRunnerArgs.setCommand(_command);

    Path argsPath = installThisArgs(_installPath, yarnRunnerArgs);

    final YarnClient client = YarnClient.createYarnClient();
    _configuration.setInt("yarn.nodemanager.delete.debug-delay-sec", (int) TimeUnit.HOURS.toSeconds(1));
    client.init(_configuration);
    client.start();

    YarnClientApplication app = client.createApplication();
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv, _appJarFile);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    {
        LocalResource appMasterJar = Records.newRecord(LocalResource.class);
        setupAppMasterJar(jarPath, appMasterJar);
        localResources.put(jarPath.getName(), appMasterJar);
    }
    {
        LocalResource appMasterArgs = Records.newRecord(LocalResource.class);
        setupAppMasterArgs(argsPath, appMasterArgs);
        localResources.put(MASTER_JSON, appMasterArgs);
    }

    List<String> vargs = new ArrayList<String>();
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    vargs.add("-Xmx256m");
    vargs.add("-Djava.net.preferIPv4Stack=true");
    vargs.add(YarnRunnerApplicationMaster.class.getName());

    String strCommand = "(echo ENV && set && echo CURRENT_DIR_LISTING && ls -la && echo PWD && pwd && ("
            + StringUtils.join(" ", vargs) + "))";
    strCommand += " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout";
    strCommand += " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr";
    LOG.debug("Application Master command [" + strCommand + "]");

    amContainer.setCommands(Collections.singletonList(strCommand));
    amContainer.setLocalResources(localResources);
    amContainer.setEnvironment(appMasterEnv);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName(_yarnName);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    if (_queue != null) {
        appContext.setQueue(_queue);
    }
    appContext.setApplicationType("yrun");

    ApplicationId appId = appContext.getApplicationId();
    AtomicBoolean shutdown = new AtomicBoolean();
    if (!_isDaemon) {
        addShutdownHook(client, appId, shutdown);
    }

    LOG.info("Submitting application with id [" + appId + "]");
    client.submitApplication(appContext);
    ApplicationReport report;
    YarnApplicationState state;
    do {
        report = client.getApplicationReport(appId);
        state = report.getYarnApplicationState();
        if (state == YarnApplicationState.RUNNING) {
            if (_isDaemon) {
                LOG.info("Application is running.  This is a daemon application driver program exiting.");
                return;
            }
        }
        Thread.sleep(100);
    } while (isNoLongerRunning(state));
    shutdown.set(true);
    LOG.info("Application has finished with state [" + state + "]");
}