Example usage for org.apache.hadoop.yarn.api.records FinalApplicationStatus SUCCEEDED

List of usage examples for org.apache.hadoop.yarn.api.records FinalApplicationStatus SUCCEEDED

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. FinalApplicationStatus
  6. SUCCEEDED

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records FinalApplicationStatus SUCCEEDED.

Prototype

FinalApplicationStatus SUCCEEDED

To view the source code for org.apache.hadoop.yarn.api.records FinalApplicationStatus SUCCEEDED.

Click Source Link

Document

Application which finished successfully.

Usage

From source file:ml.shifu.guagua.yarn.GuaguaYarnClient.java

License:Apache License

private int printFinalJobReport() throws YarnException, IOException {
    try {/*w ww .  j av a  2  s .co m*/
        ApplicationReport report = this.yarnClient.getApplicationReport(getAppId());
        FinalApplicationStatus finalAppStatus = report.getFinalApplicationStatus();
        final long secs = (report.getFinishTime() - report.getStartTime()) / 1000L;
        final String time = String.format("%d minutes, %d seconds.", secs / 60L, secs % 60L);
        LOG.info("Completed {}: {}, total running time: {}", getAppName(), finalAppStatus.name(), time);
        return finalAppStatus == FinalApplicationStatus.SUCCEEDED ? 0 : -1;
    } catch (YarnException yre) {
        LOG.error(String.format("Exception encountered while attempting to request a final job report for %s.",
                getAppId()), yre);
        return -1;
    }
}

From source file:org.apache.drill.yarn.appMaster.AMYarnFacadeImpl.java

License:Apache License

@Override
public void finish(boolean succeeded, String msg) throws YarnFacadeException {
    // Stop the Node Manager client.

    nodeMgr.stop();// ww  w.  ja  v  a  2s  . c  o m

    // Deregister the app from YARN.

    String appMsg = "Drill Cluster Shut-Down";
    FinalApplicationStatus status = FinalApplicationStatus.SUCCEEDED;
    if (!succeeded) {
        appMsg = "Drill Cluster Fatal Error - check logs";
        status = FinalApplicationStatus.FAILED;
    }
    if (msg != null) {
        appMsg = msg;
    }
    try {
        resourceMgr.unregisterApplicationMaster(status, appMsg, "");
    } catch (YarnException | IOException e) {
        throw new YarnFacadeException("Deregister AM failed", e);
    }

    // Stop the Resource Manager client

    resourceMgr.stop();
}

From source file:org.apache.flink.yarn.AbstractYarnClusterTest.java

License:Apache License

/**
 * Tests that the cluster retrieval of a finished YARN application fails.
 *///from  www .  j  a  v  a  2s  . c  o m
@Test(expected = ClusterRetrieveException.class)
public void testClusterClientRetrievalOfFinishedYarnApplication() throws Exception {
    final ApplicationId applicationId = ApplicationId.newInstance(System.currentTimeMillis(), 42);
    final ApplicationReport applicationReport = createApplicationReport(applicationId,
            YarnApplicationState.FINISHED, FinalApplicationStatus.SUCCEEDED);

    final YarnClient yarnClient = new TestingYarnClient(
            Collections.singletonMap(applicationId, applicationReport));
    final YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnClient.init(yarnConfiguration);
    yarnClient.start();

    final TestingAbstractYarnClusterDescriptor clusterDescriptor = new TestingAbstractYarnClusterDescriptor(
            new Configuration(), yarnConfiguration, temporaryFolder.newFolder().getAbsolutePath(), yarnClient,
            false);

    try {
        clusterDescriptor.retrieve(applicationId);
    } finally {
        clusterDescriptor.close();
    }
}

From source file:org.apache.flink.yarn.ApplicationMaster.java

License:Apache License

private void run() throws Exception {
    //Utils.logFilesInCurrentDirectory(LOG);
    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = Utils.initializeYarnConfiguration();
    FileSystem fs = FileSystem.get(conf);
    Map<String, String> envs = System.getenv();
    final String currDir = envs.get(Environment.PWD.key());
    final String logDirs = envs.get(Environment.LOG_DIRS.key());
    final String ownHostname = envs.get(Environment.NM_HOST.key());
    final String appId = envs.get(Client.ENV_APP_ID);
    final String clientHomeDir = envs.get(Client.ENV_CLIENT_HOME_DIR);
    final String applicationMasterHost = envs.get(Environment.NM_HOST.key());
    final String remoteFlinkJarPath = envs.get(Client.FLINK_JAR_PATH);
    final String shipListString = envs.get(Client.ENV_CLIENT_SHIP_FILES);
    final String yarnClientUsername = envs.get(Client.ENV_CLIENT_USERNAME);
    final int taskManagerCount = Integer.valueOf(envs.get(Client.ENV_TM_COUNT));
    final int memoryPerTaskManager = Integer.valueOf(envs.get(Client.ENV_TM_MEMORY));
    final int coresPerTaskManager = Integer.valueOf(envs.get(Client.ENV_TM_CORES));

    int heapLimit = Utils.calculateHeapSize(memoryPerTaskManager);

    if (currDir == null) {
        throw new RuntimeException("Current directory unknown");
    }/*from w w w .java  2s . c om*/
    if (ownHostname == null) {
        throw new RuntimeException("Own hostname (" + Environment.NM_HOST + ") not set.");
    }
    LOG.info("Working directory " + currDir);

    // load Flink configuration.
    Utils.getFlinkConfiguration(currDir);

    final String localWebInterfaceDir = currDir + "/resources/"
            + ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME;

    // Update yaml conf -> set jobManager address to this machine's address.
    FileInputStream fis = new FileInputStream(currDir + "/flink-conf.yaml");
    BufferedReader br = new BufferedReader(new InputStreamReader(fis));
    Writer output = new BufferedWriter(new FileWriter(currDir + "/flink-conf-modified.yaml"));
    String line;
    while ((line = br.readLine()) != null) {
        if (line.contains(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY)) {
            output.append(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY + ": " + ownHostname + "\n");
        } else if (line.contains(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY)) {
            output.append(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY + ": " + "\n");
        } else {
            output.append(line + "\n");
        }
    }
    // just to make sure.
    output.append(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY + ": " + ownHostname + "\n");
    output.append(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY + ": " + localWebInterfaceDir + "\n");
    output.append(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY + ": " + logDirs + "\n");
    output.close();
    br.close();
    File newConf = new File(currDir + "/flink-conf-modified.yaml");
    if (!newConf.exists()) {
        LOG.warn("modified yaml does not exist!");
    }

    Utils.copyJarContents("resources/" + ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME,
            ApplicationMaster.class.getProtectionDomain().getCodeSource().getLocation().getPath());

    JobManager jm;
    {
        String pathToNepheleConfig = currDir + "/flink-conf-modified.yaml";
        String[] args = { "-executionMode", "cluster", "-configDir", pathToNepheleConfig };

        // start the job manager
        jm = JobManager.initialize(args);

        // Start info server for jobmanager
        jm.startInfoServer();
    }

    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    LOG.info("registering ApplicationMaster");
    rmClient.registerApplicationMaster(applicationMasterHost, 0, "http://" + applicationMasterHost + ":"
            + GlobalConfiguration.getString(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, "undefined"));

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memoryPerTaskManager);
    capability.setVirtualCores(coresPerTaskManager);

    // Make container requests to ResourceManager
    for (int i = 0; i < taskManagerCount; ++i) {
        ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
        LOG.info("Requesting TaskManager container " + i);
        rmClient.addContainerRequest(containerAsk);
    }

    LocalResource flinkJar = Records.newRecord(LocalResource.class);
    LocalResource flinkConf = Records.newRecord(LocalResource.class);

    // register Flink Jar with remote HDFS
    final Path remoteJarPath = new Path(remoteFlinkJarPath);
    Utils.registerLocalResource(fs, remoteJarPath, flinkJar);

    // register conf with local fs.
    Path remoteConfPath = Utils.setupLocalResource(conf, fs, appId,
            new Path("file://" + currDir + "/flink-conf-modified.yaml"), flinkConf, new Path(clientHomeDir));
    LOG.info("Prepared localresource for modified yaml: " + flinkConf);

    boolean hasLog4j = new File(currDir + "/log4j.properties").exists();
    // prepare the files to ship
    LocalResource[] remoteShipRsc = null;
    String[] remoteShipPaths = shipListString.split(",");
    if (!shipListString.isEmpty()) {
        remoteShipRsc = new LocalResource[remoteShipPaths.length];
        { // scope for i
            int i = 0;
            for (String remoteShipPathStr : remoteShipPaths) {
                if (remoteShipPathStr == null || remoteShipPathStr.isEmpty()) {
                    continue;
                }
                remoteShipRsc[i] = Records.newRecord(LocalResource.class);
                Path remoteShipPath = new Path(remoteShipPathStr);
                Utils.registerLocalResource(fs, remoteShipPath, remoteShipRsc[i]);
                i++;
            }
        }
    }

    // respect custom JVM options in the YAML file
    final String javaOpts = GlobalConfiguration.getString(ConfigConstants.FLINK_JVM_OPTIONS, "");

    // Obtain allocated containers and launch
    int allocatedContainers = 0;
    int completedContainers = 0;
    while (allocatedContainers < taskManagerCount) {
        AllocateResponse response = rmClient.allocate(0);
        for (Container container : response.getAllocatedContainers()) {
            LOG.info("Got new Container for TM " + container.getId() + " on host "
                    + container.getNodeId().getHost());
            ++allocatedContainers;

            // Launch container by create ContainerLaunchContext
            ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

            String tmCommand = "$JAVA_HOME/bin/java -Xmx" + heapLimit + "m " + javaOpts;
            if (hasLog4j) {
                tmCommand += " -Dlog.file=\"" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                        + "/taskmanager-log4j.log\" -Dlog4j.configuration=file:log4j.properties";
            }
            tmCommand += " org.apache.flink.yarn.YarnTaskManagerRunner -configDir . " + " 1>"
                    + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-stdout.log" + " 2>"
                    + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-stderr.log";
            ctx.setCommands(Collections.singletonList(tmCommand));

            LOG.info("Starting TM with command=" + tmCommand);

            // copy resources to the TaskManagers.
            Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(2);
            localResources.put("flink.jar", flinkJar);
            localResources.put("flink-conf.yaml", flinkConf);

            // add ship resources
            if (!shipListString.isEmpty()) {
                Preconditions.checkNotNull(remoteShipRsc);
                for (int i = 0; i < remoteShipPaths.length; i++) {
                    localResources.put(new Path(remoteShipPaths[i]).getName(), remoteShipRsc[i]);
                }
            }

            ctx.setLocalResources(localResources);

            // Setup CLASSPATH for Container (=TaskTracker)
            Map<String, String> containerEnv = new HashMap<String, String>();
            Utils.setupEnv(conf, containerEnv); //add flink.jar to class path.
            containerEnv.put(Client.ENV_CLIENT_USERNAME, yarnClientUsername);

            ctx.setEnvironment(containerEnv);

            UserGroupInformation user = UserGroupInformation.getCurrentUser();
            try {
                Credentials credentials = user.getCredentials();
                DataOutputBuffer dob = new DataOutputBuffer();
                credentials.writeTokenStorageToStream(dob);
                ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
                ctx.setTokens(securityTokens);
            } catch (IOException e) {
                LOG.warn("Getting current user info failed when trying to launch the container"
                        + e.getMessage());
            }

            LOG.info("Launching container " + allocatedContainers);
            nmClient.startContainer(container, ctx);
        }
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            LOG.info("Completed container (while allocating) " + status.getContainerId() + ". Total Completed:"
                    + completedContainers);
            LOG.info("Diagnostics " + status.getDiagnostics());
        }
        Thread.sleep(100);
    }

    // Now wait for containers to complete

    while (completedContainers < taskManagerCount) {
        AllocateResponse response = rmClient.allocate(completedContainers / taskManagerCount);
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            LOG.info("Completed container " + status.getContainerId() + ". Total Completed:"
                    + completedContainers);
            LOG.info("Diagnostics " + status.getDiagnostics());
        }
        Thread.sleep(5000);
    }
    LOG.info("Shutting down JobManager");
    jm.shutdown();

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");

}

From source file:org.apache.flink.yarn.appMaster.ApplicationMaster.java

License:Apache License

@Override
public BooleanValue shutdownAM() throws Exception {
    LOG.info("Client requested shutdown of AM");
    FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
    String finalMessage = "";
    if (isFailed) {
        finalStatus = FinalApplicationStatus.FAILED;
        finalMessage = "Application Master failed";
        isFailed = false; // allow a proper shutdown
        isFailed.notifyAll();//from w w  w .j a  v a 2s .  c om
    }
    rmClient.unregisterApplicationMaster(finalStatus, finalMessage, "");
    this.close();
    return new BooleanValue(true);
}

From source file:org.apache.flink.yarn.CliFrontendYarnAddressConfigurationTest.java

License:Apache License

@Test(expected = IllegalConfigurationException.class)
public void testResumeFromYarnPropertiesFileWithFinishedApplication() throws Exception {

    File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

    // start CLI Frontend
    TestCLI frontend = new CustomYarnTestCLI(directoryPath.getAbsolutePath(), FinalApplicationStatus.SUCCEEDED);

    RunOptions options = CliFrontendParser.parseRunCommand(new String[] {});

    frontend.retrieveClient(options);/* w  w w.ja v  a  2s.c o  m*/
    checkJobManagerAddress(frontend.getConfiguration(), TEST_YARN_JOB_MANAGER_ADDRESS,
            TEST_YARN_JOB_MANAGER_PORT);
}

From source file:org.apache.flink.yarn.CliFrontendYarnAddressConfigurationTest.java

License:Apache License

@Test(expected = IllegalConfigurationException.class)
public void testResumeFromInvalidYarnID() throws Exception {
    File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

    // start CLI Frontend
    TestCLI frontend = new CustomYarnTestCLI(directoryPath.getAbsolutePath(), FinalApplicationStatus.SUCCEEDED);

    RunOptions options = CliFrontendParser
            .parseRunCommand(new String[] { "-yid", ApplicationId.newInstance(0, 666).toString() });

    frontend.retrieveClient(options);//from   w  ww.  j a  va2  s .c  o  m
    checkJobManagerAddress(frontend.getConfiguration(), TEST_YARN_JOB_MANAGER_ADDRESS,
            TEST_YARN_JOB_MANAGER_PORT);
}

From source file:org.apache.flink.yarn.CliFrontendYarnAddressConfigurationTest.java

License:Apache License

@Test(expected = IllegalConfigurationException.class)
public void testResumeFromYarnIDWithFinishedApplication() throws Exception {
    File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

    // start CLI Frontend
    TestCLI frontend = new CustomYarnTestCLI(directoryPath.getAbsolutePath(), FinalApplicationStatus.SUCCEEDED);

    RunOptions options = CliFrontendParser
            .parseRunCommand(new String[] { "-yid", TEST_YARN_APPLICATION_ID.toString() });

    frontend.retrieveClient(options);//from w  w  w .ja v a 2  s.c o  m

    checkJobManagerAddress(frontend.getConfiguration(), TEST_YARN_JOB_MANAGER_ADDRESS,
            TEST_YARN_JOB_MANAGER_PORT);
}

From source file:org.apache.flink.yarn.FlinkYarnCluster.java

License:Apache License

/**
 * Shutdown the YARN cluster./*from   w w  w.  j  a v  a 2  s.  c  om*/
 * @param failApplication whether we should fail the YARN application (in case of errors in Flink)
 */
@Override
public void shutdown(boolean failApplication) {
    if (!isConnected) {
        throw new IllegalStateException("The cluster has been connected to the ApplicationMaster.");
    }

    if (hasBeenShutDown.getAndSet(true)) {
        return;
    }

    try {
        Runtime.getRuntime().removeShutdownHook(clientShutdownHook);
    } catch (IllegalStateException e) {
        // we are already in the shutdown hook
    }

    if (actorSystem != null) {
        LOG.info("Sending shutdown request to the Application Master");
        if (applicationClient != ActorRef.noSender()) {
            try {
                FinalApplicationStatus finalStatus;
                if (failApplication) {
                    finalStatus = FinalApplicationStatus.FAILED;
                } else {
                    finalStatus = FinalApplicationStatus.SUCCEEDED;
                }
                Future<Object> response = Patterns.ask(applicationClient, new YarnMessages.LocalStopYarnSession(
                        finalStatus, "Flink YARN Client requested shutdown"), new Timeout(akkaDuration));
                Await.ready(response, akkaDuration);
            } catch (Exception e) {
                LOG.warn("Error while stopping YARN Application Client", e);
            }
        }

        actorSystem.shutdown();
        actorSystem.awaitTermination();

        actorSystem = null;
    }

    LOG.info("Deleting files in " + sessionFilesDir);
    try {
        FileSystem shutFS = FileSystem.get(hadoopConfig);
        shutFS.delete(sessionFilesDir, true); // delete conf and jar file.
        shutFS.close();
    } catch (IOException e) {
        LOG.error("Could not delete the Flink jar and configuration files in HDFS..", e);
    }

    try {
        actorRunner.join(1000); // wait for 1 second
    } catch (InterruptedException e) {
        LOG.warn("Shutdown of the actor runner was interrupted", e);
        Thread.currentThread().interrupt();
    }
    try {
        pollingRunner.stopRunner();
        pollingRunner.join(1000);
    } catch (InterruptedException e) {
        LOG.warn("Shutdown of the polling runner was interrupted", e);
        Thread.currentThread().interrupt();
    }

    LOG.info("YARN Client is shutting down");
    yarnClient.stop(); // actorRunner is using the yarnClient.
    yarnClient = null; // set null to clearly see if somebody wants to access it afterwards.
}

From source file:org.apache.flink.yarn.YarnFlinkResourceManager.java

License:Apache License

/**
 * Converts a Flink application status enum to a YARN application status enum.
 * @param status The Flink application status.
 * @return The corresponding YARN application status.
 *///from w  w w . j  a v a 2  s  .  c  om
private FinalApplicationStatus getYarnStatus(ApplicationStatus status) {
    if (status == null) {
        return FinalApplicationStatus.UNDEFINED;
    } else {
        switch (status) {
        case SUCCEEDED:
            return FinalApplicationStatus.SUCCEEDED;
        case FAILED:
            return FinalApplicationStatus.FAILED;
        case CANCELED:
            return FinalApplicationStatus.KILLED;
        default:
            return FinalApplicationStatus.UNDEFINED;
        }
    }
}