Example usage for org.apache.hadoop.yarn.client.api YarnClient createYarnClient

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api YarnClient createYarnClient.

Prototype

@Public
public static YarnClient createYarnClient() 

Source Link

Document

Create a new instance of YarnClient.

Usage

From source file:org.apache.metron.maas.service.MaasIntegrationTest.java

License:Apache License

public void testDSShell(boolean haveDomain) throws Exception {
    MaaSConfig config = new MaaSConfig() {
        {// w w  w .  jav  a2s.c o m
            setServiceRoot("/maas/service");
            setQueueConfig(new HashMap<String, Object>() {
                {
                    put(ZKQueue.ZK_PATH, "/maas/queue");
                }
            });
        }
    };
    String configRoot = "/maas/config";
    byte[] configData = ConfigUtil.INSTANCE.toBytes(config);
    try {
        client.setData().forPath(configRoot, configData);
    } catch (KeeperException.NoNodeException e) {
        client.create().creatingParentsIfNeeded().forPath(configRoot, configData);
    }
    String[] args = { "--jar", yarnComponent.getAppMasterJar(), "--zk_quorum",
            zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--master_memory", "512",
            "--master_vcores", "2", };
    if (haveDomain) {
        String[] domainArgs = { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group",
                "--modify_acls", "writer_user writer_group", "--create" };
        List<String> argsList = new ArrayList<String>(Arrays.asList(args));
        argsList.addAll(Arrays.asList(domainArgs));
        args = argsList.toArray(new String[argsList.size()]);
    }

    YarnConfiguration conf = yarnComponent.getConfig();
    LOG.info("Initializing DS Client");
    final Client client = new Client(new Configuration(conf));
    boolean initSuccess = client.init(args);
    Assert.assertTrue(initSuccess);
    LOG.info("Running DS Client");
    final AtomicBoolean result = new AtomicBoolean(false);
    Thread t = new Thread() {
        @Override
        public void run() {
            try {
                result.set(client.run());
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    };
    t.start();

    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(new Configuration(conf));
    yarnClient.start();
    String hostName = NetUtils.getHostname();

    boolean verified = false;
    String errorMessage = "";
    while (!verified) {
        List<ApplicationReport> apps = yarnClient.getApplications();
        if (apps.size() == 0) {
            Thread.sleep(10);
            continue;
        }
        ApplicationReport appReport = apps.get(0);
        if (appReport.getHost().equals("N/A")) {
            Thread.sleep(10);
            continue;
        }
        errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport.getHost()
                + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'.";
        if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) {
            verified = true;
        }
        if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) {
            break;
        }
    }
    Assert.assertTrue(errorMessage, verified);
    FileSystem fs = FileSystem.get(conf);
    try {
        new ModelSubmission().execute(FileSystem.get(conf),
                new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum",
                        zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--local_model_path",
                        "src/test/resources/maas", "--hdfs_model_path",
                        new Path(fs.getHomeDirectory(), "maas/dummy").toString(), "--num_instances", "1",
                        "--memory", "100", "--mode", "ADD", "--log4j", "src/test/resources/log4j.properties"

                });
        ServiceDiscoverer discoverer = new ServiceDiscoverer(this.client, config.getServiceRoot());
        discoverer.start();
        {
            boolean passed = false;
            for (int i = 0; i < 100; ++i) {
                try {
                    List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
                    if (endpoints != null && endpoints.size() == 1) {
                        LOG.trace("Found endpoints: " + endpoints.get(0));
                        String output = makeRESTcall(
                                new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey"));
                        if (output.contains("casey")) {
                            passed = true;
                            break;
                        }
                    }
                } catch (Exception e) {
                }
                Thread.sleep(2000);
            }
            Assert.assertTrue(passed);
        }

        {
            List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
            Assert.assertNotNull(endpoints);
            Assert.assertEquals(1, endpoints.size());
        }
        new ModelSubmission().execute(FileSystem.get(conf),
                new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum",
                        zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--num_instances",
                        "1", "--mode", "REMOVE",

                });
        {
            boolean passed = false;
            for (int i = 0; i < 100; ++i) {
                try {
                    List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0"));
                    //ensure that the endpoint is dead.
                    if (endpoints == null || endpoints.size() == 0) {
                        passed = true;
                        break;
                    }
                } catch (Exception e) {
                }
                Thread.sleep(2000);
            }
            Assert.assertTrue(passed);
        }
    } finally {
        cleanup();
    }
}

---

From source file:org.apache.oozie.action.hadoop.LauncherMainHadoopUtils.java

License:Apache License

public static void killChildYarnJobs(Configuration actionConf) {
    try {/*from  w  ww  .j av  a 2s . c o m*/
        Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf);
        if (!childYarnJobs.isEmpty()) {
            System.out.println();
            System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher");
            System.out.println("Killing existing jobs and starting over:");
            YarnClient yarnClient = YarnClient.createYarnClient();
            yarnClient.init(actionConf);
            yarnClient.start();
            for (ApplicationId app : childYarnJobs) {
                System.out.print("Killing job [" + app + "] ... ");
                yarnClient.killApplication(app);
                System.out.println("Done");
            }
            System.out.println();
        }
    } catch (YarnException ye) {
        throw new RuntimeException("Exception occurred while killing child job(s)", ye);
    } catch (IOException ioe) {
        throw new RuntimeException("Exception occurred while killing child job(s)", ioe);
    }
}

---

From source file:org.apache.reef.runtime.yarn.client.unmanaged.UnmanagedAmYarnSubmissionHelper.java

License:Apache License

UnmanagedAmYarnSubmissionHelper(final YarnConfiguration yarnConfiguration, final YarnProxyUser yarnProxyUser,
        final SecurityTokenProvider tokenProvider) throws IOException, YarnException {

    this.tokenProvider = tokenProvider;
    this.yarnProxyUser = yarnProxyUser;

    LOG.log(Level.FINE, "Initializing YARN Client");
    this.yarnClient = YarnClient.createYarnClient();
    this.yarnClient.init(yarnConfiguration);
    this.yarnClient.start();
    LOG.log(Level.FINE, "Initialized YARN Client");

    LOG.log(Level.FINE, "Requesting UNMANAGED Application ID from YARN.");

    final ContainerLaunchContext launchContext = YarnTypes.getContainerLaunchContext(
            Collections.<String>emptyList(), Collections.<String, LocalResource>emptyMap(),
            tokenProvider.getTokens());// w  w w.  j  a  v a2s. c om

    final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication();

    this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext();
    this.applicationSubmissionContext.setAMContainerSpec(launchContext);
    this.applicationSubmissionContext.setUnmanagedAM(true);

    this.applicationId = this.applicationSubmissionContext.getApplicationId();

    LOG.log(Level.INFO, "YARN UNMANAGED Application ID: {0}", this.applicationId);
}

---

From source file:org.apache.reef.runtime.yarn.client.YarnJobSubmissionHandler.java

License:Apache License

@Inject
YarnJobSubmissionHandler(final YarnConfiguration yarnConfiguration, final JobJarMaker jobJarMaker,
        final REEFFileNames filenames, final ClasspathProvider classpath,
        final ConfigurationSerializer configurationSerializer,
        final @Parameter(JVMHeapSlack.class) double jvmSlack) throws IOException {

    this.yarnConfiguration = yarnConfiguration;
    this.jobJarMaker = jobJarMaker;
    this.filenames = filenames;
    this.classpath = classpath;
    this.configurationSerializer = configurationSerializer;
    this.jvmSlack = jvmSlack;

    this.fileSystem = FileSystem.get(yarnConfiguration);

    this.yarnClient = YarnClient.createYarnClient();
    this.yarnClient.init(this.yarnConfiguration);
    this.yarnClient.start();
}

---

From source file:org.apache.reef.runtime.yarn.client.YarnSubmissionHelper.java

License:Apache License

public YarnSubmissionHelper(final YarnConfiguration yarnConfiguration, final REEFFileNames fileNames,
        final ClasspathProvider classpath, final SecurityTokenProvider tokenProvider,
        final List<String> commandPrefixList) throws IOException, YarnException {
    this.fileNames = fileNames;
    this.classpath = classpath;

    LOG.log(Level.FINE, "Initializing YARN Client");
    this.yarnClient = YarnClient.createYarnClient();
    this.yarnClient.init(yarnConfiguration);
    this.yarnClient.start();
    LOG.log(Level.FINE, "Initialized YARN Client");

    LOG.log(Level.FINE, "Requesting Application ID from YARN.");
    this.yarnClientApplication = this.yarnClient.createApplication();
    this.applicationResponse = yarnClientApplication.getNewApplicationResponse();
    this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext();
    this.applicationId = applicationSubmissionContext.getApplicationId();
    this.tokenProvider = tokenProvider;
    this.commandPrefixList = commandPrefixList;
    this.launcherClazz = REEFLauncher.class;
    this.confFileName = this.fileNames.getDriverConfigurationPath();
    LOG.log(Level.FINEST, "YARN Application ID: {0}", applicationId);
}

---

From source file:org.apache.reef.runtime.yarn.driver.unmanaged.UnmanagedAmTest.java

License:Apache License

@Test
public void testAmShutdown() throws IOException, YarnException {

    Assume.assumeTrue("This test requires a YARN Resource Manager to connect to",
            Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

    final YarnConfiguration yarnConfig = new YarnConfiguration();

    // Start YARN client and register the application

    final YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(yarnConfig);//from  w  w w  .j a va  2s  .com
    yarnClient.start();

    final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
    containerContext.setCommands(Collections.<String>emptyList());
    containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
    containerContext.setEnvironment(Collections.<String, String>emptyMap());
    containerContext.setTokens(getTokens());

    final ApplicationSubmissionContext appContext = yarnClient.createApplication()
            .getApplicationSubmissionContext();
    appContext.setApplicationName("REEF_Unmanaged_AM_Test");
    appContext.setAMContainerSpec(containerContext);
    appContext.setUnmanagedAM(true);
    appContext.setQueue("default");

    final ApplicationId applicationId = appContext.getApplicationId();
    LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

    yarnClient.submitApplication(appContext);

    LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

    addToken(yarnClient.getAMRMToken(applicationId));

    // Start the AM

    final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000,
            this);
    rmClient.init(yarnConfig);
    rmClient.start();

    final NMClientAsync nmClient = new NMClientAsyncImpl(this);
    nmClient.init(yarnConfig);
    nmClient.start();

    final RegisterApplicationMasterResponse registration = rmClient
            .registerApplicationMaster(NetUtils.getHostname(), -1, null);

    LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

    LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

    // Shutdown the AM

    rmClient.stop();
    nmClient.stop();

    // Get the final application report

    final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
    final YarnApplicationState appState = appReport.getYarnApplicationState();
    final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

    LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] { applicationId,
            appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus });

    Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
    Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

    // Shutdown YARN client

    yarnClient.stop();
}

---

From source file:org.apache.samza.autoscaling.utils.YarnUtil.java

License:Apache License

public YarnUtil(String rmAddress, int rmPort) {
    this.httpclient = HttpClientBuilder.create().build();
    this.rmServer = new HttpHost(rmAddress, rmPort, "http");
    log.info("setting rm server to : " + rmServer);
    YarnConfiguration hConfig = new YarnConfiguration();
    hConfig.set(YarnConfiguration.RM_ADDRESS, rmAddress + ":" + YarnConfiguration.DEFAULT_RM_PORT);
    yarnClient = YarnClient.createYarnClient();
    yarnClient.init(hConfig);//  ww  w  .  jav  a2  s  .c o  m
    yarnClient.start();
}

---

From source file:org.apache.sysml.yarn.ropt.YarnClusterAnalyzer.java

License:Apache License

private static YarnClient createYarnClient() {
    YarnConfiguration conf = new YarnConfiguration();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);//from w w w  . ja  v  a2  s  . c om
    yarnClient.start();
    return yarnClient;
}

---

From source file:org.apache.tajo.yarn.command.TajoCommand.java

License:Apache License

public TajoCommand(Configuration conf) {
    this.conf = conf;
    yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
}

---

From source file:org.apache.tez.client.FrameworkClient.java

License:Apache License

public static FrameworkClient createFrameworkClient(TezConfiguration tezConf) {

    boolean isLocal = tezConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
            TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
    if (isLocal) {
        return ReflectionUtils.createClazzInstance("org.apache.tez.client.LocalClient");
    }//from  w  w w. j  a  v a 2s .co m
    return new TezYarnClient(YarnClient.createYarnClient());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• »