Example usage for org.apache.hadoop.yarn.api.records ApplicationReport getTrackingUrl

List of usage examples for org.apache.hadoop.yarn.api.records ApplicationReport getTrackingUrl

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. ApplicationReport
  6. getTrackingUrl

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationReport getTrackingUrl.

Prototype

@Public
@Stable
public abstract String getTrackingUrl();

Source Link

Document

Get the tracking url for the application.

Usage

From source file:com.datatorrent.stram.StramClient.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if time expires.
 *
 * @return true if application completed successfully
 * @throws YarnException/*w w w. j a va  2 s.  c o m*/
 * @throws IOException
 */
public boolean monitorApplication() throws YarnException, IOException {
    ClientRMHelper.AppStatusCallback callback = new ClientRMHelper.AppStatusCallback() {
        @Override
        public boolean exitLoop(ApplicationReport report) {
            LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToken="
                    + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                    + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                    + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                    + ", yarnAppState=" + report.getYarnApplicationState().toString()
                    + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
                    + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());
            return false;
        }

    };
    ClientRMHelper rmClient = new ClientRMHelper(yarnClient);
    return rmClient.waitForCompletion(appId, callback, clientTimeout);
}

From source file:com.datatorrent.stram.StramMiniClusterTest.java

License:Apache License

/**
 * Verify the web service deployment and lifecycle functionality
 *
 * @throws Exception//from   w ww  .  ja va  2s.com
 */
@Ignore //disabled due to web service init delay issue
@Test
public void testWebService() throws Exception {

    // single container topology of inline input and module
    Properties props = new Properties();
    props.put(StreamingApplication.DT_PREFIX + "stream.input.classname",
            TestGeneratorInputOperator.class.getName());
    props.put(StreamingApplication.DT_PREFIX + "stream.input.outputNode", "module1");
    props.put(StreamingApplication.DT_PREFIX + "module.module1.classname", GenericTestOperator.class.getName());

    LOG.info("Initializing Client");
    LogicalPlanConfiguration tb = new LogicalPlanConfiguration(new Configuration(false));
    tb.addFromProperties(props, null);

    StramClient client = new StramClient(new Configuration(yarnCluster.getConfig()), createDAG(tb));
    if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
        client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
    }
    try {
        client.start();
        client.startApplication();

        // attempt web service connection
        ApplicationReport appReport = client.getApplicationReport();
        Thread.sleep(5000); // delay to give web service time to fully initialize
        Client wsClient = Client.create();
        wsClient.setFollowRedirects(true);
        WebResource r = wsClient.resource("http://" + appReport.getTrackingUrl()).path(StramWebServices.PATH)
                .path(StramWebServices.PATH_INFO);
        LOG.info("Requesting: " + r.getURI());
        ClientResponse response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
        JSONObject json = response.getEntity(JSONObject.class);
        LOG.info("Got response: " + json.toString());
        assertEquals("incorrect number of elements", 1, json.length());
        assertEquals("appId", appReport.getApplicationId().toString(), json.get("id"));
        r = wsClient.resource("http://" + appReport.getTrackingUrl()).path(StramWebServices.PATH)
                .path(StramWebServices.PATH_PHYSICAL_PLAN_OPERATORS);
        LOG.info("Requesting: " + r.getURI());
        response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
        json = response.getEntity(JSONObject.class);
        LOG.info("Got response: " + json.toString());

    } finally {
        //LOG.info("waiting...");
        //synchronized (this) {
        //  this.wait();
        //}
        //boolean result = client.monitorApplication();
        client.killApplication();
        client.stop();
    }

}

From source file:com.epam.hadoop.nv.yarn.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if
 * time expires./*from w w  w .  j av a 2s . c  o  m*/
 * 
 * @param appId
 *            Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }

}

From source file:com.flyhz.avengers.framework.AvengersClient.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if
 * time expires.//from   w w  w  .j  av a 2  s.  com
 * 
 * @param appId
 *            Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 10 second.
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }

}

From source file:com.ibm.bi.dml.yarn.DMLYarnClient.java

License:Open Source License

/**
 * Method to launch the dml yarn app master and execute the given dml script
 * with the given configuration and jar file.
 * //  w w w . j  av a  2  s  . co m
 * NOTE: on launching the yarn app master, we do not explicitly probe if we
 *     are running on a yarn or MR1 cluster. In case of MR1, already the class 
 *     YarnConfiguration will not be found and raise a classnotfound. In case of any 
 *     exception we fall back to run CP directly in the client process.
 * 
 * @return true if dml program successfully executed as yarn app master
 * @throws IOException 
 */
protected boolean launchDMLYarnAppmaster() throws IOException, DMLScriptException {
    boolean ret = false;
    String hdfsWD = null;

    try {
        Timing time = new Timing(true);

        // load yarn configuration
        YarnConfiguration yconf = new YarnConfiguration();

        // create yarn client
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(yconf);
        yarnClient.start();

        // create application and get the ApplicationID
        YarnClientApplication app = yarnClient.createApplication();
        ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
        ApplicationId appId = appContext.getApplicationId();
        LOG.debug("Created application (applicationID: " + appId + ")");

        // prepare hdfs working directory via ApplicationID
        // copy script, config, jar file to hdfs
        hdfsWD = DMLAppMasterUtils.constructHDFSWorkingDir(_dmlConfig, appId);
        copyResourcesToHdfsWorkingDir(yconf, hdfsWD);

        //construct command line argument
        String command = constructAMCommand(_args, _dmlConfig);
        LOG.debug("Constructed application master command: \n" + command);

        // set up the container launch context for the application master
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
        amContainer.setCommands(Collections.singletonList(command));
        amContainer.setLocalResources(constructLocalResourceMap(yconf));
        amContainer.setEnvironment(constructEnvionmentMap(yconf));

        // Set up resource type requirements for ApplicationMaster
        int memHeap = _dmlConfig.getIntValue(DMLConfig.YARN_APPMASTERMEM);
        int memAlloc = (int) computeMemoryAllocation(memHeap);
        Resource capability = Records.newRecord(Resource.class);
        capability.setMemory(memAlloc);
        capability.setVirtualCores(NUM_CORES);
        LOG.debug("Requested application resources: memory=" + memAlloc + ", vcores=" + NUM_CORES);

        // Finally, set-up ApplicationSubmissionContext for the application
        String qname = _dmlConfig.getTextValue(DMLConfig.YARN_APPQUEUE);
        appContext.setApplicationName(APPMASTER_NAME); // application name
        appContext.setAMContainerSpec(amContainer);
        appContext.setResource(capability);
        appContext.setQueue(qname); // queue
        LOG.debug("Configured application meta data: name=" + APPMASTER_NAME + ", queue=" + qname);

        // submit application (non-blocking)
        yarnClient.submitApplication(appContext);

        // Check application status periodically (and output web ui address)
        ApplicationReport appReport = yarnClient.getApplicationReport(appId);
        LOG.info("Application tracking-URL: " + appReport.getTrackingUrl());
        YarnApplicationState appState = appReport.getYarnApplicationState();
        YarnApplicationState oldState = appState;
        LOG.info("Application state: " + appState);
        while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED
                && appState != YarnApplicationState.FAILED) {
            Thread.sleep(APP_STATE_INTERVAL); //wait for 200ms
            appReport = yarnClient.getApplicationReport(appId);
            appState = appReport.getYarnApplicationState();
            if (appState != oldState) {
                oldState = appState;
                LOG.info("Application state: " + appState);
            }
        }
        //check final status (failed or succeeded)
        FinalApplicationStatus finalState = appReport.getFinalApplicationStatus();
        LOG.info("Application final status: " + finalState);

        //show application and total runtime
        double appRuntime = (double) (appReport.getFinishTime() - appReport.getStartTime()) / 1000;
        LOG.info("Application runtime: " + appRuntime + " sec.");
        LOG.info("Total runtime: " + String.format("%.3f", time.stop() / 1000) + " sec.");

        //raised script-level error in case of failed final status
        if (finalState != FinalApplicationStatus.SUCCEEDED) {
            //propagate script-level stop call message
            String stop_msg = readMessageToHDFSWorkingDir(_dmlConfig, yconf, appId);
            if (stop_msg != null)
                throw new DMLScriptException(stop_msg);

            //generic failure message
            throw new DMLRuntimeException(
                    "DML yarn app master finished with final status: " + finalState + ".");
        }

        ret = true;
    } catch (DMLScriptException ex) {
        //rethrow DMLScriptException to propagate stop call
        throw ex;
    } catch (Exception ex) {
        LOG.error("Failed to run DML yarn app master.", ex);
        ret = false;
    } finally {
        //cleanup working directory
        if (hdfsWD != null)
            MapReduceTool.deleteFileIfExistOnHDFS(hdfsWD);
    }

    return ret;
}

From source file:com.scistor.dshell.ScistorClient.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if
 * time expires./*from w  w w . ja  v a  2s.  c  om*/
 * 
 * @param appId
 *            Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 2 second.
        try {
            Thread.sleep(2000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }

}

From source file:com.sogou.dockeronyarn.client.DockerClient.java

License:Apache License

/**
 * Monitor the submitted application for completion. 
 * Kill application if time expires. //from w w w . j a  va2  s  . com
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }
        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not shutdown." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }

}

From source file:com.toy.Client.java

License:Apache License

private boolean monitorApplication(ApplicationId appId) throws Exception {

    while (true) {

        // Check app status every second.
        try {//from  w w  w . ja  va 2  s . co  m
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = yarn.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appDiagnostics="
                + report.getDiagnostics() + ", yarnAppState=" + report.getYarnApplicationState().toString()
                + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
                + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        switch (state) {
        case RUNNING:
            LOG.info(
                    "TOY is running\n" + "To stop : ./toy.sh -stop -zookeeper {}\n"
                            + "Get containers status : ./toy.sh -status -zookeeper {}",
                    toyConfig.zookeeper, toyConfig.zookeeper);
            return true;
        case FAILED:
        case KILLED:
            LOG.error("Ooops something went wrong");
            return false;
        case FINISHED:
            LOG.info("YARN app has already finished");
            return false;
        default:
            break;

        }
    }
}

From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusChecker.java

License:Apache License

private String buildXmlUrl(ApplicationReport appReport) {
    String trackingUrl = appReport.getTrackingUrl();
    String xmlUrl = trackingUrl + "conf";
    return xmlUrl;
}

From source file:com.twitter.hraven.hadoopJobMonitor.notification.Notifier.java

License:Apache License

public static String littleProgressMsg(ApplicationReport appReport, TaskReport taskReport,
        TaskAttemptID taskAttemptId, float progress, float progressThreshold, long max) {
    String msg = "The task attempt " + taskAttemptId + " of  " + appReport.getName() + " run by "
            + appReport.getUser() + " on " + CLUSTER + " did not have enough progress: total progress is "
            + progress + ", lagged progress threshold is " + progressThreshold + " and max run time is "
            + durationStr(max) + " hours\n";
    msg += "Tracking url is " + appReport.getTrackingUrl() + "\n";
    return msg;/*from  w w  w. j  a v a2 s.co m*/
}