Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration()

Source Link

Usage

From source file:org.apache.sqoop.submission.spark.SparkYarnSubmitter.java

License:Apache License

public static void submit(String[] args, SparkConf sparkConf) {
    YarnConfiguration yarnConfig = new YarnConfiguration();
    // convert the *- site xml to yarn conf
    ClientArguments cArgs = new ClientArguments(args, sparkConf);
    new Client(cArgs, yarnConfig, sparkConf).run();
}

From source file:org.apache.sqoop.submission.spark.YarnSqoopSparkClient.java

License:Apache License

static YarnConfiguration generateYarnSparkConf(Map<String, String> conf) {
    YarnConfiguration yarnConf = new YarnConfiguration();
    for (Map.Entry<String, String> entry : conf.entrySet()) {
        yarnConf.set(entry.getKey(), entry.getValue());
    }/*from w  w w .j a v a 2  s .  com*/
    return yarnConf;
}

From source file:org.apache.sysml.yarn.DMLAppMaster.java

License:Apache License

public void runApplicationMaster(String[] args) throws YarnException, IOException {
    _conf = new YarnConfiguration();

    //obtain application ID
    String containerIdString = System.getenv(Environment.CONTAINER_ID.name());
    ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
    _appId = containerId.getApplicationAttemptId().getApplicationId();
    LOG.info("SystemML appplication master (applicationID: " + _appId + ")");

    //initialize clients to ResourceManager
    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(_conf);/*from  w  ww  . j  a  v a2 s. com*/
    rmClient.start();

    //register with ResourceManager
    rmClient.registerApplicationMaster("", 0, ""); //host, port for rm communication
    LOG.debug("Registered the SystemML application master with resource manager");

    //start status reporter to ResourceManager
    DMLAppMasterStatusReporter reporter = new DMLAppMasterStatusReporter(rmClient, 10000);
    reporter.start();
    LOG.debug("Started status reporter (heartbeat to resource manager)");

    //set DMLscript app master context
    DMLScript.setActiveAM();

    //parse input arguments
    String[] otherArgs = new GenericOptionsParser(_conf, args).getRemainingArgs();

    //run SystemML CP
    FinalApplicationStatus status = null;
    try {
        //core dml script execution (equivalent to non-AM runtime)
        boolean success = DMLScript.executeScript(_conf, otherArgs);

        if (success)
            status = FinalApplicationStatus.SUCCEEDED;
        else
            status = FinalApplicationStatus.FAILED;
    } catch (DMLScriptException ex) {
        LOG.error(DMLYarnClient.APPMASTER_NAME + ": Failed to executed DML script due to stop call:\n\t"
                + ex.getMessage());
        status = FinalApplicationStatus.FAILED;
        writeMessageToHDFSWorkingDir(ex.getMessage());
    } catch (Exception ex) {
        LOG.error(DMLYarnClient.APPMASTER_NAME + ": Failed to executed DML script.", ex);
        status = FinalApplicationStatus.FAILED;
    } finally {
        //stop periodic status reports
        reporter.stopStatusReporter();
        LOG.debug("Stopped status reporter");

        //unregister resource manager client
        rmClient.unregisterApplicationMaster(status, "", "");
        LOG.debug("Unregistered the SystemML application master");
    }
}

From source file:org.apache.sysml.yarn.ropt.YarnClusterAnalyzer.java

License:Apache License

private static YarnClient createYarnClient() {
    YarnConfiguration conf = new YarnConfiguration();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);//from ww  w  .j av a2 s  .c om
    yarnClient.start();
    return yarnClient;
}

From source file:org.apache.tajo.master.rule.TestMasterRules.java

License:Apache License

@Test
public void testTajoConfValidationRule() throws Exception {
    TajoConf tajoConf = new TajoConf(new YarnConfiguration());

    EvaluationContext context = new EvaluationContext();
    context.addParameter(TajoConf.class.getName(), tajoConf);

    TajoConfValidationRule validationRule = new TajoConfValidationRule();
    EvaluationResult result = validationRule.evaluate(context);

    assertThat(result, is(notNullValue()));
    assertThat(result.getReturnCode(), is(EvaluationResultCode.OK));
}

From source file:org.apache.tajo.master.rule.TestMasterRules.java

License:Apache License

@Test(expected = EvaluationFailedException.class)
public void testTajoConfValidationRuleWithException() throws Exception {
    TajoConf tajoConf = new TajoConf(new YarnConfiguration());
    SelfDiagnosisRuleEngine ruleEngine = SelfDiagnosisRuleEngine.getInstance();
    SelfDiagnosisRuleSession ruleSession = ruleEngine.newRuleSession();

    tajoConf.setVar(TajoConf.ConfVars.ROOT_DIR, "invalid path.");

    EvaluationContext context = new EvaluationContext();
    context.addParameter(TajoConf.class.getName(), tajoConf);

    ruleSession.withRuleNames("TajoConfValidationRule").fireRules(context);

    fail("EvaluationFailedException exception is expected, but it does not happen.");
}

From source file:org.apache.tajo.master.rule.TestMasterRules.java

License:Apache License

@Test
public void testFileSystemRule() throws Exception {
    TajoConf tajoConf = new TajoConf(new YarnConfiguration());

    createTajoDirectories(tajoConf);/*from  w w  w.  ja  va2 s  .  co m*/

    EvaluationContext context = new EvaluationContext();
    context.addParameter(TajoConf.class.getName(), tajoConf);

    FileSystemRule fsRule = new FileSystemRule();
    EvaluationResult result = fsRule.evaluate(context);

    assertThat(result, is(notNullValue()));
    assertThat(result.getReturnCode(), is(EvaluationResultCode.OK));
}

From source file:org.apache.tajo.master.rule.TestMasterRules.java

License:Apache License

@Test
public void testFileSystemRuleWithError() throws Exception {
    TajoConf tajoConf = new TajoConf(new YarnConfiguration());

    createTajoDirectories(tajoConf);// w w w. j ava  2s  .co m
    Path systemResourceDir = TajoConf.getSystemResourceDir(tajoConf);
    FileSystem defaultFs = systemResourceDir.getFileSystem(tajoConf);
    if (defaultFs.exists(systemResourceDir)) {
        defaultFs.delete(systemResourceDir, true);
    }

    EvaluationContext context = new EvaluationContext();
    context.addParameter(TajoConf.class.getName(), tajoConf);

    FileSystemRule fsRule = new FileSystemRule();
    EvaluationResult result = fsRule.evaluate(context);

    assertThat(result, is(notNullValue()));
    assertThat(result.getReturnCode(), is(EvaluationResultCode.ERROR));
}

From source file:org.apache.tajo.master.TajoMaster.java

License:Apache License

public static void main(String[] args) throws Exception {
    StringUtils.startupShutdownMessage(TajoMaster.class, args, LOG);

    try {// w  w w. j  a v a  2  s  .c  o  m
        TajoMaster master = new TajoMaster();
        TajoConf conf = new TajoConf(new YarnConfiguration());
        master.init(conf);
        master.start();
    } catch (Throwable t) {
        LOG.fatal("Error starting TajoMaster", t);
        System.exit(-1);
    }
}

From source file:org.apache.tajo.yarn.Client.java

License:Apache License

public static void main(String[] args) throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    Client client = new Client(conf);
    client.run(args);/*w w  w  . j  a v  a2s . c om*/
}