Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.conf.*
  5. YarnConfiguration
  6. YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration()

Source Link

Usage

From source file:com.flyhz.avengers.framework.application.AnalyzeApplication.java

License:Apache License

public AnalyzeApplication() {
    // Set up the configuration
    conf = new YarnConfiguration();
    hbaseConf.set("hbase.zookeeper.quorum", "m1,s1,s2");
    hbaseConf.set("hbase.zookeeper.property.clientPort", "2181");
}

From source file:com.flyhz.avengers.framework.application.CrawlApplication.java

License:Apache License

public CrawlApplication() {
    // Set up the configuration
    conf = new YarnConfiguration();
    hbaseConf.set("hbase.zookeeper.quorum", "m1,s1,s2");
    hbaseConf.set("hbase.zookeeper.property.clientPort", "2181");

}

From source file:com.flyhz.avengers.framework.application.FetchApplication.java

License:Apache License

public FetchApplication() {
    // Set up the configuration
    conf = new YarnConfiguration();
    hbaseConf.set("hbase.zookeeper.quorum", "m1,s1,s2");
    hbaseConf.set("hbase.zookeeper.property.clientPort", "2181");
}

From source file:com.flyhz.avengers.framework.application.InitEnvApplication.java

License:Apache License

public InitEnvApplication() {
    // Set up the configuration
    conf = new YarnConfiguration();
    hbaseConf.set("hbase.zookeeper.quorum", "m1,s1,s2");
    hbaseConf.set("hbase.zookeeper.property.clientPort", "2181");
    opts = new Options();
    opts.addOption("batchId", true, "batchId");

}

From source file:com.flyhz.avengers.framework.AvengersAppMaster.java

License:Apache License

public AvengersAppMaster() {
    // Set up the configuration
    conf = new YarnConfiguration();
    hbaseConf.set("hbase.zookeeper.quorum", "m1,s1,s2");
    hbaseConf.set("hbase.zookeeper.property.clientPort", "2181");
}

From source file:com.flyhz.avengers.framework.AvengersClient.java

License:Apache License

/**
*/
public AvengersClient() throws Exception {
    this(new YarnConfiguration());
}

From source file:com.github.hdl.tensorflow.yarn.app.ApplicationMaster.java

License:Apache License

public ApplicationMaster() {
    conf = new YarnConfiguration();
}

From source file:com.github.sakserv.minicluster.simpleyarnapp.Client.java

License:Apache License

public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);/*from  w  w w.j  a  v a 2  s. c  o  m*/
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M"
            + " com.hortonworks.simpleyarnapp.ApplicationMaster" + " " + command + " " + String.valueOf(n) + " "
            + resourceManagerAddress + " " + resourceManagerHostname + " " + resourceManagerSchedulerAddress
            + " " + resourceManagerResourceTrackerAddress + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
            + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED
            && appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println("Application " + appId + " finished with" + " state " + appState + " at "
            + appReport.getFinishTime());

}

From source file:com.gpiskas.yarn.AppMaster.java

License:Open Source License

public void run() throws Exception {
    conf = new YarnConfiguration();

    // Create NM Client
    nmClient = NMClient.createNMClient();
    nmClient.init(conf);//from   w ww .  j  a  va2  s .c  om
    nmClient.start();

    // Create AM - RM Client
    AMRMClientAsync<ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
    rmClient.init(conf);
    rmClient.start();

    // Register with RM
    rmClient.registerApplicationMaster("", 0, "");
    System.out.println("AppMaster: Registered");

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    // Reqiest Containers from RM
    System.out.println("AppMaster: Requesting " + containerCount + " Containers");
    for (int i = 0; i < containerCount; ++i) {
        rmClient.addContainerRequest(new ContainerRequest(capability, null, null, priority));
    }

    while (!containersFinished()) {
        Thread.sleep(100);
    }

    System.out.println("AppMaster: Unregistered");
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
}

From source file:com.gpiskas.yarn.Client.java

License:Open Source License

private void run() throws Exception {
    conf = new YarnConfiguration();

    // Create Yarn Client
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);//from  w  ww  .  ja v  a 2s  . co m
    client.start();

    // Create Application
    YarnClientApplication app = client.createApplication();

    // Create AM Container
    ContainerLaunchContext amCLC = Records.newRecord(ContainerLaunchContext.class);
    amCLC.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M"
            + " com.gpiskas.yarn.AppMaster" + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
            + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));

    // Set AM jar
    LocalResource jar = Records.newRecord(LocalResource.class);
    Utils.setUpLocalResource(Utils.YARNAPP_JAR_PATH, jar, conf);
    amCLC.setLocalResources(Collections.singletonMap(Utils.YARNAPP_JAR_NAME, jar));

    // Set AM CLASSPATH
    Map<String, String> env = new HashMap<String, String>();
    Utils.setUpEnv(env, conf);
    amCLC.setEnvironment(env);

    // Set AM resources
    Resource res = Records.newRecord(Resource.class);
    res.setMemory(256);
    res.setVirtualCores(1);

    // Create ApplicationSubmissionContext
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName("YARNAPP");
    appContext.setQueue("default");
    appContext.setAMContainerSpec(amCLC);
    appContext.setResource(res);

    // Submit Application
    ApplicationId id = appContext.getApplicationId();
    System.out.println("Client: Submitting " + id);
    client.submitApplication(appContext);

    ApplicationReport appReport = client.getApplicationReport(id);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED
            && appState != YarnApplicationState.FAILED) {
        Thread.sleep(1000);
        appReport = client.getApplicationReport(id);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println("Client: Finished " + id + " with state " + appState);
}