Example usage for org.apache.hadoop.yarn.api.records YarnApplicationState RUNNING

List of usage examples for org.apache.hadoop.yarn.api.records YarnApplicationState RUNNING

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. YarnApplicationState
  6. RUNNING

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records YarnApplicationState RUNNING.

Prototype

YarnApplicationState RUNNING

To view the source code for org.apache.hadoop.yarn.api.records YarnApplicationState RUNNING.

Click Source Link

Document

Application which is currently running.

Usage

From source file:com.cloudera.branchreduce.impl.thrift.Client.java

License:Open Source License

@Override
public int handle(YarnClientService clientService) throws Exception {
    clientService.startAndWait();//from  www  .j a v  a2 s.  c  o m
    if (!clientService.isRunning()) {
        LOG.error("BranchReduce job did not start, exiting...");
        return 1;
    }

    Lord.Client client = null;
    while (clientService.isRunning()) {
        ApplicationReport report = clientService.getApplicationReport();
        if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
            String originalTrackingUrl = report.getOriginalTrackingUrl();
            if (originalTrackingUrl != null && originalTrackingUrl.contains(":")) {
                System.out.println("Original Tracking URL = " + originalTrackingUrl);
                String[] pieces = originalTrackingUrl.split(":");
                TSocket socket = new TSocket(pieces[0], Integer.valueOf(pieces[1]));
                TProtocol protocol = new TBinaryProtocol(socket);
                client = new Lord.Client(protocol);
                socket.open();
                break;
            }
        }
    }

    if (client == null) {
        LOG.error("Could not connect to thrift service to get status");
        return 1;
    }

    Configuration conf = clientService.getParameters().getConfiguration();
    Class<GlobalState> globalStatusClass = (Class<GlobalState>) conf
            .getClass(BranchReduceConfig.GLOBAL_STATE_CLASS, GlobalState.class);

    boolean finished = false;
    while (!clientService.isApplicationFinished()) {
        if (!finished) {
            GlobalStatusResponse resp = client.getGlobalStatus(new GlobalStatusRequest());
            this.value = Writables.fromByteBuffer(resp.bufferForGlobalState(), globalStatusClass);
            if (resp.isFinished()) {
                LOG.info("Job finished running.");
                finished = true;
            }
            LOG.info(value);
        }
        Thread.sleep(1000);
    }

    clientService.stopAndWait();
    ApplicationReport report = clientService.getFinalReport();
    if (report.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED) {
        System.out.println("Job complete.");
        System.out.println(value);
        return 0;
    } else {
        System.out.println("Final app state: " + report.getFinalApplicationStatus());
        System.out.println("Last global state:");
        System.out.println(value);
        return 1;
    }
}

From source file:com.cloudera.kitten.client.KittenClient.java

License:Open Source License

public int handle(YarnClientService service) throws Exception {
    service.startAndWait();/* w w  w.  j av  a2 s.co  m*/
    if (!service.isRunning()) {
        LOG.error("Service failed to startup, exiting...");
        return 1;
    }

    String trackingUrl = null;
    while (!service.isApplicationFinished()) {
        Thread.sleep(1000);

        if (trackingUrl == null) {
            ApplicationReport report = service.getApplicationReport();
            if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
                trackingUrl = report.getTrackingUrl();
                if (trackingUrl == null || trackingUrl.isEmpty()) {
                    LOG.info("Application is running, but did not specify a tracking URL");
                    trackingUrl = "";
                } else {
                    LOG.info("Master Tracking URL = " + trackingUrl);
                }
            }
        }
    }

    service.stopAndWait();
    System.exit(0);
    return 0;
}

From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java

License:Apache License

/**
 * Test to verify Llama deletes old reservations on startup.
 *///  w ww  .j av  a  2s .co  m
@Test(timeout = 60000)
public void testLlamaDeletesOldReservationsOnStartup() throws Exception {
    YarnClient client = null;
    LlamaAM llamaAM1 = null, llamaAM2 = null, llamaAM3 = null;
    EnumSet<YarnApplicationState> running = EnumSet.of(YarnApplicationState.RUNNING);
    try {
        startYarn(createMiniYarnConfig(false));

        client = YarnClient.createYarnClient();
        client.init(miniYarn.getConfig());
        client.start();
        Assert.assertEquals("Non-zero YARN apps even before any reservations", 0,
                client.getApplications().size());

        llamaAM1 = LlamaAM.create(getLlamaConfiguration());
        llamaAM1.start();
        Assert.assertEquals("Mismatch between #YARN apps and #Queues", 2,
                client.getApplications(running).size());

        // Start another Llama of the same cluster-id to see if old YARN apps
        // are deleted.
        llamaAM2 = LlamaAM.create(getLlamaConfiguration());
        llamaAM2.start();
        Assert.assertEquals(
                "Mismatch between #YARN apps and #Queues. Only apps"
                        + " from the latest started Llama should be running.",
                2, client.getApplications(running).size());

        // Start Llama of different cluster-id to see old YARN apps are not
        // deleted.
        Configuration confWithDifferentCluserId = getLlamaConfiguration();
        confWithDifferentCluserId.set(LlamaAM.CLUSTER_ID, "new-cluster");
        llamaAM3 = LlamaAM.create(confWithDifferentCluserId);
        llamaAM3.start();
        Assert.assertEquals("Mismatch between #YARN apps and #Queues for " + "multiple clusters", 4,
                client.getApplications(running).size());

    } finally {
        client.stop();
        llamaAM1.stop();
        llamaAM2.stop();
        llamaAM3.stop();
        stopYarn();
    }
}

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

@Override
public void deleteAllReservations() throws LlamaException {
    try {/*from   ww w.j  av  a2s. com*/
        ugi.doAs(new PrivilegedExceptionAction<Void>() {
            @Override
            public Void run() throws Exception {
                List<ApplicationReport> apps = yarnClient.getApplications(Collections.singleton(appType),
                        EnumSet.of(YarnApplicationState.RUNNING));
                for (ApplicationReport app : apps) {
                    yarnClient.killApplication(app.getApplicationId());
                }
                return null;
            }
        });
    } catch (Throwable ex) {
        throw new LlamaException(ex, ErrorCode.AM_CANNOT_START);
    }
}

From source file:com.datatorrent.stram.cli.ApexCli.java

License:Apache License

private ApplicationReport assertRunningApp(ApplicationReport app) {
    ApplicationReport r;//w ww.j  a  v a  2s  .  c  o  m
    try {
        r = yarnClient.getApplicationReport(app.getApplicationId());
        if (r.getYarnApplicationState() != YarnApplicationState.RUNNING) {
            String msg = String.format("Application %s not running (status %s)", r.getApplicationId().getId(),
                    r.getYarnApplicationState());
            throw new CliException(msg);
        }
    } catch (YarnException rmExc) {
        throw new CliException("Unable to determine application status", rmExc);
    } catch (IOException rmExc) {
        throw new CliException("Unable to determine application status", rmExc);
    }
    return r;
}

From source file:com.datatorrent.stram.cli.ApexCliShutdownCommandTest.java

License:Apache License

private ApplicationReport mockRunningApplicationReport(String appId, String appName) {
    ApplicationReport app = mock(ApplicationReport.class);
    ApplicationId applicationId = mock(ApplicationId.class);

    when(applicationId.toString()).thenReturn(appId);
    when(app.getApplicationId()).thenReturn(applicationId);

    when(app.getName()).thenReturn(appName);

    when(app.getYarnApplicationState()).thenReturn(YarnApplicationState.RUNNING);
    when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);

    when(app.getTrackingUrl()).thenReturn("http://example.com");

    return app;/*from w  w  w. j av a 2  s. c  o m*/
}

From source file:com.datatorrent.stram.client.StramClientUtils.java

License:Apache License

public static ApplicationReport getStartedAppInstanceByName(YarnClient clientRMService, String appName,
        String user, String excludeAppId) throws YarnException, IOException {
    List<ApplicationReport> applications = clientRMService
            .getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE),
                    EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.ACCEPTED,
                            YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING,
                            YarnApplicationState.SUBMITTED));
    // see whether there is an app with the app name and user name running
    for (ApplicationReport app : applications) {
        if (!app.getApplicationId().toString().equals(excludeAppId) && app.getName().equals(appName)
                && app.getUser().equals(user)) {
            return app;
        }// www . j a va  2s .  c  om
    }
    return null;
}

From source file:com.github.hdl.tensorflow.yarn.app.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException//from ww w.jav a2 s . c om
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        ApplicationReport report = yarnClient.getApplicationReport(appId);

        //      LOG.info("Got application report from ASM for"
        //          + ", appId=" + appId.getId()
        //          + ", clientToAMToken=" + report.getClientToAMToken()
        //          + ", appDiagnostics=" + report.getDiagnostics()
        //          + ", appMasterHost=" + report.getHost()
        //          + ", appQueue=" + report.getQueue()
        //          + ", appMasterRpcPort=" + report.getRpcPort()
        //          + ", appStartTime=" + report.getStartTime()
        //          + ", yarnAppState=" + report.getYarnApplicationState().toString()
        //          + ", tfAppFinalState=" + report.getFinalApplicationStatus().toString()
        //          + ", appTrackingUrl=" + report.getTrackingUrl()
        //          + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus tfStatus = report.getFinalApplicationStatus();

        if (YarnApplicationState.RUNNING == state) {
            if (appRpc == null) {
                String hostname = report.getHost();
                int port = report.getRpcPort();
                LOG.info("application master rpc host: " + hostname + "; port: " + port);
                appRpc = new TFApplicationRpcClient(hostname, port).getRpc();
            }

            if (appRpc != null && isEmptyString(clusterSpecJsonString)) {
                clusterSpecJsonString = appRpc.getClusterSpec();
                LOG.info("cluster spec is " + clusterSpecJsonString);
                if (!isEmptyString(clusterSpecJsonString)) {
                    TFClient tfClient = new TFClient(tfClientPy);
                    if (isEnableTensorBoard) {
                        Thread tensorBoardThread = new Thread() {
                            @Override
                            public void run() {
                                tfClient.startTensorBoardClient(tensorboardEventDir);
                            }
                        };
                        tensorBoardThread.start();
                        LOG.info("Launching tensorboard ...");
                        try {
                            Thread.sleep(3000);
                        } catch (InterruptedException e) {
                            LOG.debug("Thread sleep in monitoring loop interrupted");
                        }
                        if (tensorBoardThread.isAlive()) {
                            LOG.info("the tensorboard launched successfully on the localhost:6006");
                        } else {
                            LOG.info("the tensorboard launched failed");
                        }
                    }
                    tfClient.startTensorflowClient(clusterSpecJsonString);
                }
            }
        }

        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == tfStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", tfAppFinalState=" + tfStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", tfAppFinalState="
                    + tfStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

    }

}

From source file:com.twitter.hraven.hadoopJobMonitor.ClusterStatusChecker.java

License:Apache License

/**
 * Get the app list from RM and check status of each
 *//*  www .ja v a2s .c  o  m*/
@Override
public void run() {
    // 1. get the list of running apps
    LOG.info("Running " + ClusterStatusChecker.class.getName());
    try {
        YarnConfiguration yConf = new YarnConfiguration();
        LOG.info(yConf.get(YarnConfiguration.RM_ADDRESS));
        LOG.info("Getting appList ...");
        // TODO: in future hadoop API we will be able to filter the app list
        EnumSet<YarnApplicationState> states = EnumSet.of(YarnApplicationState.RUNNING);
        List<ApplicationReport> appList = rmDelegate.getApplications(states);
        LOG.info("appList received. size is: " + appList.size());
        for (ApplicationReport appReport : appList)
            checkAppStatus(appReport);
    } catch (YarnRuntimeException e) {
        LOG.error("Error in getting application list from RM", e);
    } catch (YarnException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

From source file:com.twitter.hraven.hadoopJobMonitor.ClusterStatusChecker.java

License:Apache License

/**
 * The pre-check performed on the app status. The idea is to reduce the avoid
 * paying the cost of the actual check, if it app does not need it.
 * /*  ww w  .jav  a 2s  .c  o  m*/
 * @param appReport
 * @return true if app passes this pre-check
 */
private boolean preAppStatusCheck(ApplicationReport appReport) {
    YarnApplicationState appState = appReport.getYarnApplicationState();
    LOG.debug("checking app " + appReport.getApplicationId() + " state is " + appState);
    return appState == YarnApplicationState.RUNNING;
}