Example usage for org.apache.hadoop.yarn.api.records YarnApplicationState FINISHED

List of usage examples for org.apache.hadoop.yarn.api.records YarnApplicationState FINISHED

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. YarnApplicationState
  6. FINISHED

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records YarnApplicationState FINISHED.

Prototype

YarnApplicationState FINISHED

To view the source code for org.apache.hadoop.yarn.api.records YarnApplicationState FINISHED.

Click Source Link

Document

Application which finished successfully.

Usage

From source file:org.springframework.yarn.examples.BatchTests.java

License:Apache License

@Test
public void testAppSubmission() throws Exception {
    String[] args = new String[] {
            "--spring.yarn.client.files[0]=file:build/libs/test-batch-restart-db-appmaster-2.0.0.BUILD-SNAPSHOT.jar",
            "--spring.yarn.client.files[1]=file:build/libs/test-batch-restart-db-container-2.0.0.BUILD-SNAPSHOT.jar" };

    @SuppressWarnings("resource")
    FsShell shell = new FsShell(getConfiguration());
    shell.touchz("/tmp/remoteStep1partition0");
    shell.touchz("/tmp/remoteStep1partition1");

    ApplicationInfo info1 = submitApplicationAndWait(BatchClientApplication.class, args, 4, TimeUnit.MINUTES);
    assertThat(info1.getYarnApplicationState(), is(YarnApplicationState.FINISHED));

    List<Resource> resources1 = ContainerLogUtils.queryContainerLogs(getYarnCluster(),
            info1.getApplicationId());//from w w  w  .  j a  v a  2s  .  c  om
    assertThat(resources1, notNullValue());
    assertThat(resources1.size(), is(10));

    int ok = 0;
    int fail = 0;
    for (Resource res : resources1) {
        File file = res.getFile();
        if (file.getName().endsWith("stdout")) {
            // there has to be some content in stdout file
            assertThat(file.length(), greaterThan(0l));
            if (file.getName().equals("Container.stdout")) {
                Scanner scanner = new Scanner(file);
                String content = scanner.useDelimiter("\\A").next();
                scanner.close();
                // this is what container will log in stdout
                assertThat(content, containsString("Hello from HdfsTasklet"));
                if (content.contains("Hello from HdfsTasklet ok")) {
                    ok++;
                }
                if (content.contains("Hello from HdfsTasklet fail")) {
                    fail++;
                }
            }
        } else if (file.getName().endsWith("stderr")) {
            String content = "";
            if (file.length() > 0) {
                Scanner scanner = new Scanner(file);
                content = scanner.useDelimiter("\\A").next();
                scanner.close();
            }
            // can't have anything in stderr files
            assertThat("stderr file is not empty: " + content, file.length(), is(0l));
        }
    }
    assertThat("Failed to find ok's from logs", ok, is(2));
    assertThat("Failed to find fail's from logs", fail, is(2));

    shell.touchz("/tmp/remoteStep2partition0");
    shell.touchz("/tmp/remoteStep2partition1");

    ApplicationInfo info2 = submitApplicationAndWait(BatchClientApplication.class, args, 4, TimeUnit.MINUTES);
    assertThat(info2.getYarnApplicationState(), is(YarnApplicationState.FINISHED));

    List<Resource> resources2 = ContainerLogUtils.queryContainerLogs(getYarnCluster(),
            info2.getApplicationId());
    assertThat(resources2, notNullValue());
    assertThat(resources2.size(), is(6));

    ok = 0;
    fail = 0;
    for (Resource res : resources2) {
        File file = res.getFile();
        if (file.getName().endsWith("stdout")) {
            // there has to be some content in stdout file
            assertThat(file.length(), greaterThan(0l));
            if (file.getName().equals("Container.stdout")) {
                Scanner scanner = new Scanner(file);
                String content = scanner.useDelimiter("\\A").next();
                scanner.close();
                // this is what container will log in stdout
                assertThat(content, containsString("Hello from HdfsTasklet"));
                if (content.contains("Hello from HdfsTasklet ok")) {
                    ok++;
                }
                if (content.contains("Hello from HdfsTasklet fail")) {
                    fail++;
                }
            }
        } else if (file.getName().endsWith("stderr")) {
            String content = "";
            if (file.length() > 0) {
                Scanner scanner = new Scanner(file);
                content = scanner.useDelimiter("\\A").next();
                scanner.close();
            }
            // can't have anything in stderr files
            assertThat("stderr file is not empty: " + content, file.length(), is(0l));
        }
    }
    assertThat("Failed to find ok's from logs", ok, is(2));
    assertThat("Found fail's from logs", fail, is(0));
}

From source file:org.springframework.yarn.examples.CustomAmserviceTests.java

License:Apache License

@Test
@Timed(millis = 130000)/*w  ww .j  a  v  a2 s.  c  om*/
public void testAppSubmission() throws Exception {
    ApplicationInfo info = submitApplicationAndWait(120, TimeUnit.SECONDS);
    assertThat(info, notNullValue());
    assertThat(info.getYarnApplicationState(), notNullValue());
    assertThat(info.getApplicationId(), notNullValue());
    assertThat(info.getYarnApplicationState(), is(YarnApplicationState.FINISHED));

    List<Resource> resources = ContainerLogUtils.queryContainerLogs(getYarnCluster(), info.getApplicationId());

    assertThat(resources, notNullValue());
    assertThat(resources.size(), greaterThan(2));

    for (Resource res : resources) {
        File file = res.getFile();
        if (file.getName().endsWith("stderr")) {
            String content = "";
            if (file.length() > 0) {
                Scanner scanner = new Scanner(file);
                content = scanner.useDelimiter("\\A").next();
                scanner.close();
            }
            // can't have anything in stderr files
            assertThat("stderr file is not empty: " + content, file.length(), is(0l));
        }
    }
}

From source file:org.springframework.yarn.examples.MultiContextTests.java

License:Apache License

@Test
@Timed(millis = 120000)//from w  ww . j  a v  a  2 s . c  om
public void testAppSubmission() throws Exception {
    ApplicationInfo info = submitApplicationAndWait();
    assertNotNull(info.getYarnApplicationState());
    assertThat(info.getYarnApplicationState(), is(YarnApplicationState.FINISHED));

    List<Resource> resources = ContainerLogUtils.queryContainerLogs(getYarnCluster(), info.getApplicationId());

    // appmaster and 4 containers should
    // make it 10 log files
    assertThat(resources, notNullValue());
    assertThat(resources.size(), is(10));

    for (Resource res : resources) {
        File file = res.getFile();
        if (file.getName().endsWith("stdout")) {
            // there has to be some content in stdout file
            assertThat(file.length(), greaterThan(0l));
            if (file.getName().equals("Container.stdout")) {
                Scanner scanner = new Scanner(file);
                String content = scanner.useDelimiter("\\A").next();
                scanner.close();
                // this is what container will log in stdout
                assertThat(content, containsString("Hello from MultiContextContainer"));
            }
        } else if (file.getName().endsWith("stderr")) {
            // can't have anything in stderr files
            assertThat(file.length(), is(0l));
        }
    }
}

From source file:org.springframework.yarn.examples.RestartContextTests.java

License:Apache License

@Test
@Timed(millis = 70000)/*from w  ww . j av  a 2 s .com*/
public void testAppSubmission() throws Exception {
    ApplicationInfo info = submitApplicationAndWait();
    assertThat(info, notNullValue());
    assertThat(info.getYarnApplicationState(), notNullValue());
    assertThat(info.getApplicationId(), notNullValue());
    assertThat(info.getYarnApplicationState(), is(YarnApplicationState.FINISHED));

    List<Resource> resources = ContainerLogUtils.queryContainerLogs(getYarnCluster(), info.getApplicationId());

    assertThat(resources, notNullValue());
    assertThat(resources.size(), is(6));

    for (Resource res : resources) {
        File file = res.getFile();
        if (file.getName().endsWith("stdout")) {
            // there has to be some content in stdout file
            assertThat(file.length(), greaterThan(0l));
        } else if (file.getName().endsWith("stderr")) {
            String content = "";
            if (file.length() > 0) {
                Scanner scanner = new Scanner(file);
                content = scanner.useDelimiter("\\A").next();
                scanner.close();
            }
            // can't have anything in stderr files
            assertThat("stderr file is not empty: " + content, file.length(), is(0l));
        }
    }
}

From source file:org.springframework.yarn.examples.SimpleCommandTests.java

License:Apache License

@Test
@Timed(millis = 70000)//from w w w.ja v a2s.co  m
public void testAppSubmission() throws Exception {
    ApplicationInfo info = submitApplicationAndWait();
    assertThat(info, notNullValue());
    assertThat(info.getYarnApplicationState(), notNullValue());
    assertThat(info.getApplicationId(), notNullValue());
    assertThat(info.getYarnApplicationState(), is(YarnApplicationState.FINISHED));

    List<Resource> resources = ContainerLogUtils.queryContainerLogs(getYarnCluster(), info.getApplicationId());

    // appmaster and 4 containers should
    // make it 10 log files
    assertThat(resources, notNullValue());
    assertThat(resources.size(), is(10));

    for (Resource res : resources) {
        File file = res.getFile();
        if (file.getName().endsWith("stdout")) {
            // there has to be some content in stdout file
            assertThat(file.length(), greaterThan(0l));
        } else if (file.getName().endsWith("stderr")) {
            String content = "";
            if (file.length() > 0) {
                Scanner scanner = new Scanner(file);
                content = scanner.useDelimiter("\\A").next();
                scanner.close();
            }
            // can't have anything in stderr files
            assertThat("stderr file is not empty: " + content, file.length(), is(0l));
        }
    }
}

From source file:org.springframework.yarn.test.junit.AbstractYarnClusterTests.java

License:Apache License

/**
 * Submits application and wait state.//from   ww  w  . jav  a 2 s .c  o  m
 *
 * @param timeout the timeout for wait
 * @param unit the unit for timeout
 * @return Application info for submit
 * @throws Exception if exception occurred
 * @see ApplicationInfo
 * @see #submitApplicationAndWaitState(long, TimeUnit, YarnApplicationState...)
 */
protected ApplicationInfo submitApplicationAndWait(long timeout, TimeUnit unit) throws Exception {
    return submitApplicationAndWaitState(timeout, unit, YarnApplicationState.FINISHED,
            YarnApplicationState.FAILED);
}

From source file:org.springframework.yarn.test.junit.ClusterBaseTestClassSubmitTests.java

License:Apache License

private void doSubmitAndAssert() throws Exception {
    ApplicationInfo info = submitApplicationAndWait(120, TimeUnit.SECONDS);
    assertThat(info, notNullValue());//from   www  .j av  a  2s . c om
    assertThat(info.getYarnApplicationState(), notNullValue());
    assertThat(info.getApplicationId(), notNullValue());
    assertThat(info.getYarnApplicationState(), is(YarnApplicationState.FINISHED));

    List<Resource> resources = ContainerLogUtils.queryContainerLogs(getYarnCluster(), info.getApplicationId());

    // appmaster and 4 containers should
    // make it 10 log files
    assertThat(resources, notNullValue());
    assertThat("expecting 10 log files", resources.size(), is(10));

    for (Resource res : resources) {
        File file = res.getFile();
        if (file.getName().endsWith("stdout")) {
            // there has to be some content in stdout file
            assertThat("there has to be content in stdout file", file.length(), greaterThan(0l));
            if (file.getName().equals("Container.stdout")) {
                Scanner scanner = new Scanner(file);
                String content = scanner.useDelimiter("\\A").next();
                scanner.close();
                // check that we have a simple timestamp
                assertThat("content doesn't look like timestamp", content.length(), greaterThan(10));
                assertThat("content doesn't look like timestamp", content.length(), lessThan(40));
            }
        } else if (file.getName().endsWith("stderr")) {
            String content = "";
            if (file.length() > 0) {
                Scanner scanner = new Scanner(file);
                content = scanner.useDelimiter("\\A").next();
                scanner.close();
            }
            // can't have anything in stderr files
            assertThat("stderr file is not empty: " + content, file.length(), is(0l));
        }
    }

}

From source file:org.springframework.yarn.test.YarnClusterTests.java

License:Apache License

private void doSubmitAndAssert() throws Exception {
    YarnClient client = (YarnClient) ctx.getBean("yarnClient");
    assertThat(client, notNullValue());//from   w  w  w .j a  v  a2 s.  c  o  m

    ApplicationId applicationId = client.submitApplication();
    assertThat(applicationId, notNullValue());

    YarnApplicationState state = null;
    for (int i = 0; i < 120; i++) {
        state = findState(client, applicationId);
        if (state == null) {
            break;
        }
        if (state.equals(YarnApplicationState.FINISHED) || state.equals(YarnApplicationState.FAILED)) {
            break;
        }
        Thread.sleep(1000);
    }
    assertThat(state, notNullValue());

    YarnCluster cluster = (YarnCluster) ctx.getBean("yarnCluster");
    File testWorkDir = cluster.getYarnWorkDir();

    PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
    String locationPattern = "file:" + testWorkDir.getAbsolutePath() + "/**/" + applicationId.toString()
            + "/**/*.std*";
    Resource[] resources = resolver.getResources(locationPattern);

    // get possible appmaster error from stderr file
    StringBuilder masterFailReason = new StringBuilder();
    for (Resource res : resources) {
        File file = res.getFile();
        if (file.getName().endsWith("Appmaster.stderr") && file.length() > 0) {
            Scanner scanner = new Scanner(file);
            masterFailReason.append("[Appmaster.stderr=");
            masterFailReason.append(scanner.useDelimiter("\\A").next());
            masterFailReason.append("]");
            scanner.close();
            break;
        }
    }

    masterFailReason.append(", [ApplicationReport Diagnostics=");
    masterFailReason.append(client.getApplicationReport(applicationId).getDiagnostics());
    masterFailReason.append("], [Num of log files=");
    masterFailReason.append(resources.length);
    masterFailReason.append("]");

    assertThat(masterFailReason.toString(), state, is(YarnApplicationState.FINISHED));

    // appmaster and 4 containers should
    // make it 10 log files
    assertThat(resources, notNullValue());
    assertThat(resources.length, is(10));

    for (Resource res : resources) {
        File file = res.getFile();
        if (file.getName().endsWith("stdout")) {
            assertThat(file.length(), greaterThan(0l));
        } else if (file.getName().endsWith("stderr")) {
            assertThat(file.length(), is(0l));
        }
    }
}

From source file:org.starschema.hadoop.yarn.applications.distributedshell.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion. 
 * Kill application if time expires. /*from  w w  w  .j ava  2s.c om*/
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        //      if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
        //        LOG.info("Reached client specified timeout for application. Killing application");
        //        forceKillApplication(appId);
        //        return false;            
        //      }
    }

}

From source file:proxyyarn.ProxyYarn.java

License:Apache License

private boolean monitorApplication(YarnClient yarnClient, ApplicationId appId)
        throws YarnException, IOException {

    long clientStartTime = System.currentTimeMillis();
    while (true) {

        // Check app status every 1 second.
        try {/*from w w w. j a  v  a 2 s  .  co  m*/
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            log.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        log.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                log.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                log.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            log.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            log.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(yarnClient, appId);
            return false;
        }
    }
}