Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration() 

Source Link

Usage

From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java

License:Apache License

public AsterixApplicationMaster() {
    // Set up the configuration and RPC
    conf = new YarnConfiguration();

}

---

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
*/
public AsterixYARNClient() throws Exception {
    this(new YarnConfiguration());
}

---

From source file:edu.uci.ics.asterix.aoya.test.AsterixYARNInstanceUtil.java

License:Apache License

public YarnConfiguration setUp() throws Exception {
    File asterixProjectDir = new File(System.getProperty("user.dir"));

    File installerTargetDir = new File(asterixProjectDir, "target");

    String[] dirsInTarget = installerTargetDir.list(new FilenameFilter() {
        @Override/*w  ww. j  a v a2s .  co m*/
        public boolean accept(File dir, String name) {
            return new File(dir, name).isDirectory() && name.startsWith("asterix-yarn")
                    && name.endsWith("binary-assembly");
        }

    });
    if (dirsInTarget.length != 1) {
        throw new IllegalStateException("Could not find binary to run YARN integration test with");
    }
    aoyaHome = installerTargetDir.getAbsolutePath() + File.separator + dirsInTarget[0];
    File asterixServerInstallerDir = new File(aoyaHome, "asterix");
    String[] zipsInFolder = asterixServerInstallerDir.list(new FilenameFilter() {
        @Override
        public boolean accept(File dir, String name) {
            return name.startsWith("asterix-server") && name.endsWith("binary-assembly.zip");
        }
    });
    if (zipsInFolder.length != 1) {
        throw new IllegalStateException("Could not find server binary to run YARN integration test with");
    }
    aoyaServerPath = asterixServerInstallerDir.getAbsolutePath() + File.separator + zipsInFolder[0];
    configPath = aoyaHome + File.separator + "configs" + File.separator + "local.xml";
    parameterPath = aoyaHome + File.separator + "conf" + File.separator + "base-asterix-configuration.xml";
    YARNCluster.getInstance().setup();
    appConf = new YarnConfiguration();
    File baseDir = new File("./target/hdfs/").getAbsoluteFile();
    FileUtil.fullyDelete(baseDir);
    appConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(appConf);
    MiniDFSCluster hdfsCluster = builder.build();
    miniCluster = YARNCluster.getInstance().getCluster();
    appConf.set("fs.defaultFS", "hdfs://localhost:" + hdfsCluster.getNameNodePort());
    miniCluster.init(appConf);
    Cluster defaultConfig = Utils.parseYarnClusterConfig(configPath);
    for (Node n : defaultConfig.getNode()) {
        n.setClusterIp(MiniYARNCluster.getHostname());
    }
    defaultConfig.getMasterNode().setClusterIp(MiniYARNCluster.getHostname());
    configPath = "target" + File.separator + "localized-aoya-config.xml";
    Utils.writeYarnClusterConfig(configPath, defaultConfig);
    miniCluster.start();
    appConf = new YarnConfiguration(miniCluster.getConfig());
    appConf.set("fs.defaultFS", "hdfs://localhost:" + hdfsCluster.getNameNodePort());
    //TODO:why must I do this!? what is not being passed properly via environment variables???
    appConf.writeXml(new FileOutputStream("target" + File.separator + "yarn-site.xml"));

    //once the cluster is created, you can get its configuration
    //with the binding details to the cluster added from the minicluster
    FileSystem fs = FileSystem.get(appConf);
    Path instanceState = new Path(fs.getHomeDirectory(), AsterixYARNClient.CONF_DIR_REL + INSTANCE_NAME + "/");
    fs.delete(instanceState, true);
    Assert.assertFalse(fs.exists(instanceState));

    File outdir = new File(PATH_ACTUAL);
    outdir.mkdirs();
    return appConf;
}

---

From source file:eu.stratosphere.yarn.Utils.java

License:Apache License

public static Configuration initializeYarnConfiguration() {
    Configuration conf = new YarnConfiguration();
    String configuredHadoopConfig = GlobalConfiguration.getString(ConfigConstants.PATH_HADOOP_CONFIG, null);
    if (configuredHadoopConfig != null) {
        LOG.info("Using hadoop configuration path from " + ConfigConstants.PATH_HADOOP_CONFIG + " setting.");
        addPathToConfig(conf, new File(configuredHadoopConfig));
        setDefaultConfValues(conf);//from   w  w  w.  j  a  v  a2  s  .  com
        return conf;
    }
    String[] envs = { "YARN_CONF_DIR", "HADOOP_CONF_DIR", "HADOOP_CONF_PATH" };
    for (int i = 0; i < envs.length; ++i) {
        String confPath = System.getenv(envs[i]);
        if (confPath != null) {
            LOG.info("Found " + envs[i] + ", adding it to configuration");
            addPathToConfig(conf, new File(confPath));
            setDefaultConfValues(conf);
            return conf;
        }
    }
    LOG.info("Could not find HADOOP_CONF_PATH, using HADOOP_HOME.");
    String hadoopHome = null;
    try {
        hadoopHome = Shell.getHadoopHome();
    } catch (IOException e) {
        LOG.fatal("Unable to get hadoop home. Please set HADOOP_HOME variable!", e);
        System.exit(1);
    }
    File tryConf = new File(hadoopHome + "/etc/hadoop");
    if (tryConf.exists()) {
        LOG.info("Found configuration using hadoop home.");
        addPathToConfig(conf, tryConf);
    } else {
        tryConf = new File(hadoopHome + "/conf");
        if (tryConf.exists()) {
            addPathToConfig(conf, tryConf);
        }
    }
    setDefaultConfValues(conf);
    return conf;
}

---

From source file:gobblin.azkaban.AzkabanGobblinYarnAppLauncher.java

License:Apache License

public AzkabanGobblinYarnAppLauncher(String jobId, Properties props) throws IOException {
    super(jobId, LOGGER);
    Config gobblinConfig = ConfigUtils.propertiesToConfig(props);
    this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(gobblinConfig, new YarnConfiguration());
}

---

From source file:gobblin.yarn.GobblinApplicationMaster.java

License:Apache License

public static void main(String[] args) throws Exception {
    Options options = buildOptions();/*from   w  w  w  .j  a  v a2 s . c o  m*/
    try {
        CommandLine cmd = new DefaultParser().parse(options, args);
        if (!cmd.hasOption(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME)) {
            printUsage(options);
            System.exit(1);
        }

        Log4jConfigurationHelper.updateLog4jConfiguration(GobblinApplicationMaster.class,
                GobblinYarnConfigurationKeys.GOBBLIN_YARN_LOG4J_CONFIGURATION_FILE,
                GobblinYarnConfigurationKeys.GOBBLIN_YARN_LOG4J_CONFIGURATION_FILE);

        LOGGER.info(JvmUtils.getJvmInputArguments());

        ContainerId containerId = ConverterUtils
                .toContainerId(System.getenv().get(ApplicationConstants.Environment.CONTAINER_ID.key()));

        try (GobblinApplicationMaster applicationMaster = new GobblinApplicationMaster(
                cmd.getOptionValue(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME), containerId,
                ConfigFactory.load(), new YarnConfiguration())) {

            applicationMaster.start();
        }
    } catch (ParseException pe) {
        printUsage(options);
        System.exit(1);
    }
}

---

From source file:gobblin.yarn.GobblinApplicationMasterTest.java

License:Open Source License

@BeforeClass
public void setUp() throws Exception {
    this.testingZKServer = new TestingServer(TEST_ZK_PORT);

    URL url = GobblinApplicationMasterTest.class.getClassLoader()
            .getResource(GobblinApplicationMasterTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    Config config = ConfigFactory.parseURL(url).resolve();

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES, "1");

    String zkConnectionString = config.getString(GobblinYarnConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    YarnHelixUtils.createGobblinYarnHelixCluster(zkConnectionString,
            config.getString(GobblinYarnConfigurationKeys.HELIX_CLUSTER_NAME_KEY));

    this.helixManager = HelixManagerFactory.getZKHelixManager(
            config.getString(GobblinYarnConfigurationKeys.HELIX_CLUSTER_NAME_KEY),
            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.PARTICIPANT, zkConnectionString);
    this.helixManager.connect();
    this.helixManager.getMessagingService().registerMessageHandlerFactory(
            Message.MessageType.SHUTDOWN.toString(), new TestShutdownMessageHandlerFactory(this));

    this.gobblinApplicationMaster = new GobblinApplicationMaster(TestHelper.TEST_APPLICATION_NAME,
            ConverterUtils.toContainerId(TestHelper.TEST_CONTROLLER_CONTAINER_ID), config, yarnConfiguration);
    this.gobblinApplicationMaster.getEventBus().register(this.gobblinApplicationMaster);
    this.gobblinApplicationMaster.connectHelixManager();
}

---

From source file:gobblin.yarn.GobblinWorkUnitRunnerTest.java

License:Open Source License

@BeforeClass
public void setUp() throws Exception {
    this.testingZKServer = new TestingServer(TEST_ZK_PORT);

    URL url = GobblinWorkUnitRunnerTest.class.getClassLoader()
            .getResource(GobblinWorkUnitRunnerTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    Config config = ConfigFactory.parseURL(url).resolve();

    String zkConnectionString = config.getString(GobblinYarnConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    YarnHelixUtils.createGobblinYarnHelixCluster(zkConnectionString,
            config.getString(GobblinYarnConfigurationKeys.HELIX_CLUSTER_NAME_KEY));

    // Participant
    this.gobblinWorkUnitRunner = new GobblinWorkUnitRunner(TestHelper.TEST_APPLICATION_NAME,
            TestHelper.TEST_HELIX_INSTANCE_NAME,
            ConverterUtils.toContainerId(TestHelper.TEST_PARTICIPANT_CONTAINER_ID), config,
            Optional.<Path>absent());
    this.gobblinWorkUnitRunner.connectHelixManager();

    // Controller
    this.gobblinApplicationMaster = new GobblinApplicationMaster(TestHelper.TEST_APPLICATION_NAME,
            ConverterUtils.toContainerId(TestHelper.TEST_CONTROLLER_CONTAINER_ID), config,
            new YarnConfiguration());
    this.gobblinApplicationMaster.connectHelixManager();

}

---

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

public static void main(String[] args) throws Exception {
    final GobblinYarnAppLauncher gobblinYarnAppLauncher = new GobblinYarnAppLauncher(ConfigFactory.load(),
            new YarnConfiguration());
    Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override//from   ww w  .j  av a 2  s.c om
        public void run() {
            try {
                gobblinYarnAppLauncher.stop();
            } catch (IOException ioe) {
                LOGGER.error("Failed to shutdown the " + GobblinYarnAppLauncher.class.getSimpleName(), ioe);
            } catch (TimeoutException te) {
                LOGGER.error("Timeout in stopping the service manager", te);
            } finally {
                if (gobblinYarnAppLauncher.emailNotificationOnShutdown) {
                    gobblinYarnAppLauncher.sendEmailOnShutdown(Optional.<ApplicationReport>absent());
                }
            }
        }
    });

    gobblinYarnAppLauncher.launch();
}

---

From source file:gobblin.yarn.GobblinYarnAppLauncherTest.java

License:Apache License

@BeforeClass
public void setUp() throws Exception {
    // Set java home in environment since it isn't set on some systems
    String javaHome = System.getProperty("java.home");
    setEnv("JAVA_HOME", javaHome);

    final YarnConfiguration clusterConf = new YarnConfiguration();
    clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");

    MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
    miniYARNCluster.init(clusterConf);/*from ww w .  j a  v  a  2s  .c  o  m*/
    miniYARNCluster.start();

    // YARN client should not be started before the Resource Manager is up
    AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() {
        @Override
        public boolean apply(Void input) {
            return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
    }, "Waiting for RM");

    this.yarnClient = this.closer.register(YarnClient.createYarnClient());
    this.yarnClient.init(clusterConf);
    this.yarnClient.start();

    // Use a random ZK port
    TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
    LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());

    // the zk port is dynamically configured
    try (PrintWriter pw = new PrintWriter("dynamic.conf")) {
        File dir = new File("target/dummydir");

        // dummy directory specified in configuration
        dir.mkdir();

        pw.println("gobblin.cluster.zk.connection.string=\"" + testingZKServer.getConnectString() + "\"");
        pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
    }

    // YARN config is dynamic and needs to be passed to other processes
    try (OutputStream os = new FileOutputStream(new File("yarn-site.xml"))) {
        clusterConf.writeXml(os);
    }

    this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer);

    URL url = GobblinYarnAppLauncherTest.class.getClassLoader()
            .getResource(GobblinYarnAppLauncherTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    this.config = ConfigFactory.parseURL(url).withValue("gobblin.cluster.zk.connection.string",
            ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())).resolve();

    String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    this.helixManager = HelixManagerFactory.getZKHelixManager(
            this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY),
            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString);

    this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• »