Example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_RESOURCE_TRACKER_ADDRESS

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_RESOURCE_TRACKER_ADDRESS.

Prototype

String RM_RESOURCE_TRACKER_ADDRESS

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RM_RESOURCE_TRACKER_ADDRESS.

Click Source Link

Usage

From source file:com.github.sakserv.minicluster.impl.MRLocalCluster.java

License:Apache License

@Override
public void configure() throws Exception {

    // Handle Windows
    WindowsLibsUtils.setHadoopHome();//from   w ww  . j  av a  2 s.  c om

    configuration.set(YarnConfiguration.RM_ADDRESS, resourceManagerAddress);
    configuration.set(YarnConfiguration.RM_HOSTNAME, resourceManagerHostname);
    configuration.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, resourceManagerSchedulerAddress);
    configuration.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, resourceManagerResourceTrackerAddress);
    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resourceManagerWebappAddress);
    configuration.set(JHAdminConfig.MR_HISTORY_ADDRESS, jobHistoryAddress);
    configuration.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    configuration.set(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, "true");
    if (getUseInJvmContainerExecutor()) {
        configuration.set(YarnConfiguration.NM_CONTAINER_EXECUTOR, inJvmContainerExecutorClass);
        configuration.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
        configuration.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
    }

    if (null != hdfsDefaultFs) {
        configuration.set("fs.defaultFS", hdfsDefaultFs);
        configuration.set("dfs.replication", "1");
    }
}

---

From source file:com.github.sakserv.minicluster.impl.YarnLocalCluster.java

License:Apache License

@Override
public void configure() throws Exception {
    // Handle Windows
    WindowsLibsUtils.setHadoopHome();//from w ww  .ja  v  a 2 s.co  m

    configuration.set(YarnConfiguration.RM_ADDRESS, resourceManagerAddress);
    configuration.set(YarnConfiguration.RM_HOSTNAME, resourceManagerHostname);
    configuration.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, resourceManagerSchedulerAddress);
    configuration.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, resourceManagerResourceTrackerAddress);
    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resourceManagerWebappAddress);
    configuration.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    if (getUseInJvmContainerExecutor()) {
        configuration.set(YarnConfiguration.NM_CONTAINER_EXECUTOR, inJvmContainerExecutorClass);
        configuration.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
        configuration.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
    }
}

---

From source file:io.hops.metadata.util.DistributedRTClientEvaluation.java

License:Apache License

public DistributedRTClientEvaluation(String rtAddress, int nbSimulatedNM, int hbPeriod, long duration,
        String output, int startingPort, int nbNMTotal)
        throws IOException, YarnException, InterruptedException {

    this.nbNM = nbSimulatedNM;
    this.hbPeriod = hbPeriod;
    this.duration = duration;
    this.output = output;
    this.nbNMTotal = nbNMTotal;
    conf = new YarnConfiguration();
    conf.setStrings(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, rtAddress);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);

    //Create NMs//from   w  w w. ja v a2s  .c  o m
    for (int i = 0; i < nbSimulatedNM; i++) {
        nmMap.put(i, NodeId.newInstance(InetAddress.getLocalHost().getHostName(), startingPort + i));
    }

    start();
}

---

From source file:io.hops.metadata.util.DistributedRTClientEvaluation.java

License:Apache License

public void start() throws YarnException, IOException, InterruptedException {

    //Assign nodes to worker threads
    //Register nodes
    ResourceTracker rt = (ResourceTracker) RpcClientFactoryPBImpl.get()
            .getClient(// w w w .ja va 2  s .c  o  m
                    ResourceTracker.class, 1, NetUtils
                            .getConnectAddress(
                                    new InetSocketAddress(
                                            conf.get(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS),
                                            conf.getInt(YarnConfiguration.RM_RESOURCE_TRACKER_PORT,
                                                    YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT))),
                    conf);
    for (NodeId nid : nmMap.values()) {
        registerClient(rt, nid);
    }
    //Wait for processing to complete
    //TODO: Get Active RMNodes from ndb instead of sleeping
    Thread.sleep(5000);
    //Send heartbeats

    for (NodeId nid : nmMap.values()) {
        //Send the Heartbeats

        executor.execute(new RTClientWorker(nid));
        Thread.sleep(hbPeriod / nbNM);

    }
    executor.shutdown();
    Thread.sleep(duration / 4);
    nbTreatedScheduler.set(0);
    nbTreatedRT.set(0);
    long start = System.currentTimeMillis();
    Thread.sleep(duration / 2);
    double nbHBTheoric = ((double) nbNM * duration / 2) / hbPeriod;
    System.out.printf("nb treatedRT %d, nb treatedScheduler %d, theorical nbhb: %f, duration: %d\n",
            nbTreatedRT.get(), nbTreatedScheduler.get(), nbHBTheoric, System.currentTimeMillis() - start);
    double treatedSchedulerRate = (double) nbTreatedScheduler.get() / nbHBTheoric;
    double treatedRTRate = (double) nbTreatedRT.get() / nbHBTheoric;
    LOG.info("treatedSchedulerRate: " + treatedSchedulerRate);

    File file = new File(output);
    if (!file.exists()) {
        file.createNewFile();
    }
    FileWriter fileWriter = new FileWriter(output, true);

    BufferedWriter bufferWritter = new BufferedWriter(fileWriter);
    bufferWritter.write(nbNMTotal + "\t" + nbTreatedRT.get() + "\t" + nbTreatedScheduler.get() + "\t"
            + nbHBTheoric + "\t" + treatedRTRate + "\t" + treatedSchedulerRate + "\n");
    bufferWritter.close();

    Thread.sleep(1000);
    System.exit(0);
}

---