Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YARN_LOG_SERVER_URL

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YARN_LOG_SERVER_URL.

Prototype

String YARN_LOG_SERVER_URL

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration YARN_LOG_SERVER_URL.

Click Source Link

Usage

From source file:org.apache.hoya.yarn.appmaster.state.AppState.java

License:Apache License

/**
 * handle completed node in the CD -move something from the live
 * server list to the completed server list
 * @param amConf YarnConfiguration/*from  w  w w .j  ava2s. c om*/
 * @param status the node that has just completed
 * @return NodeCompletionResult
 */
public synchronized NodeCompletionResult onCompletedNode(YarnConfiguration amConf, ContainerStatus status) {
    ContainerId containerId = status.getContainerId();
    NodeCompletionResult result = new NodeCompletionResult();
    RoleInstance roleInstance;

    if (containersBeingReleased.containsKey(containerId)) {
        log.info("Container was queued for release");
        Container container = containersBeingReleased.remove(containerId);
        RoleStatus roleStatus = lookupRoleStatus(container);
        log.info("decrementing role count for role {}", roleStatus.getName());
        roleStatus.decReleasing();
        roleStatus.decActual();
        roleStatus.incCompleted();
        roleHistory.onReleaseCompleted(container);

    } else if (surplusNodes.remove(containerId)) {
        //its a surplus one being purged
        result.surplusNode = true;
    } else {
        //a container has failed 
        result.containerFailed = true;
        roleInstance = activeContainers.remove(containerId);
        if (roleInstance != null) {
            //it was active, move it to failed 
            incFailedCountainerCount();
            failedNodes.put(containerId, roleInstance);
        } else {
            // the container may have been noted as failed already, so look
            // it up
            roleInstance = failedNodes.get(containerId);
        }
        if (roleInstance != null) {
            int roleId = roleInstance.roleId;
            log.info("Failed container in role {}", roleId);
            try {
                RoleStatus roleStatus = lookupRoleStatus(roleId);
                roleStatus.decActual();
                boolean shortLived = isShortLived(roleInstance);
                String message;
                if (roleInstance.container != null) {
                    String user = null;
                    try {
                        user = HoyaUtils.getCurrentUser().getShortUserName();
                    } catch (IOException ioe) {
                    }
                    String completedLogsUrl = null;
                    Container c = roleInstance.container;
                    String url = null;
                    if (amConf != null) {
                        url = amConf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
                    }
                    if (user != null && url != null) {
                        completedLogsUrl = url + "/" + c.getNodeId() + "/" + roleInstance.getContainerId()
                                + "/ctx/" + user;
                    }
                    message = String.format(
                            "Failure %s on host %s" + (completedLogsUrl != null ? ", see %s" : ""),
                            roleInstance.getContainerId(), c.getNodeId().getHost(), completedLogsUrl);
                } else {
                    message = String.format("Failure %s", containerId.toString());
                }
                roleStatus.noteFailed(message);
                //have a look to see if it short lived
                if (shortLived) {
                    roleStatus.incStartFailed();
                }

                if (roleInstance.container != null) {
                    roleHistory.onFailedContainer(roleInstance.container, shortLived);
                }

            } catch (YarnRuntimeException e1) {
                log.error("Failed container of unknown role {}", roleId);
            }
        } else {
            //this isn't a known container.

            log.error("Notified of completed container {} that is not in the list"
                    + " of active or failed containers", containerId);
            completionOfUnknownContainerEvent.incrementAndGet();
        }
    }

    if (result.surplusNode) {
        //a surplus node
        return result;
    }

    //record the complete node's details; this pulls it from the livenode set 
    //remove the node
    ContainerId id = status.getContainerId();
    RoleInstance node = getLiveNodes().remove(id);
    if (node == null) {
        log.warn("Received notification of completion of unknown node {}", id);
        completionOfNodeNotInLiveListEvent.incrementAndGet();

    } else {
        node.state = ClusterDescription.STATE_DESTROYED;
        node.exitCode = status.getExitStatus();
        node.diagnostics = status.getDiagnostics();
        getCompletedNodes().put(id, node);
        result.roleInstance = node;
    }
    return result;
}

---

From source file:org.apache.slider.server.appmaster.state.AppState.java

License:Apache License

/**
 * Build up the application state//w w w  . ja v  a2s .  co  m
 * @param instanceDefinition definition of the applicatin instance
 * @param appmasterConfig
 * @param publishedProviderConf any configuration info to be published by a provider
 * @param providerRoles roles offered by a provider
 * @param fs filesystem
 * @param historyDir directory containing history files
 * @param liveContainers list of live containers supplied on an AM restart
 * @param applicationInfo
 * @param releaseSelector
 */
public synchronized void buildInstance(AggregateConf instanceDefinition, Configuration appmasterConfig,
        Configuration publishedProviderConf, List<ProviderRole> providerRoles, FileSystem fs, Path historyDir,
        List<Container> liveContainers, Map<String, String> applicationInfo,
        SimpleReleaseSelector releaseSelector)
        throws BadClusterStateException, BadConfigException, IOException {
    Preconditions.checkArgument(instanceDefinition != null);
    Preconditions.checkArgument(releaseSelector != null);

    this.publishedProviderConf = publishedProviderConf;
    this.applicationInfo = applicationInfo != null ? applicationInfo : new HashMap<String, String>();

    clientProperties = new HashMap<String, String>();
    containerReleaseSelector = releaseSelector;

    Set<String> confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf);

    //  Add the -site configuration properties
    for (String key : confKeys) {
        String val = publishedProviderConf.get(key);
        clientProperties.put(key, val);
    }

    // set the cluster specification (once its dependency the client properties
    // is out the way
    updateInstanceDefinition(instanceDefinition);

    //build the initial role list
    for (ProviderRole providerRole : providerRoles) {
        buildRole(providerRole);
    }

    ConfTreeOperations resources = instanceDefinition.getResourceOperations();

    Set<String> roleNames = resources.getComponentNames();
    for (String name : roleNames) {
        if (!roles.containsKey(name)) {
            // this is a new value
            log.info("Adding role {}", name);
            MapOperations resComponent = resources.getComponent(name);
            ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent);
            buildRole(dynamicRole);
            providerRoles.add(dynamicRole);
        }
    }
    //then pick up the requirements
    buildRoleRequirementsFromResources();

    //set the livespan
    MapOperations globalResOpts = instanceDefinition.getResourceOperations().getGlobalOptions();

    startTimeThreshold = globalResOpts.getOptionInt(InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
            InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);

    failureThreshold = globalResOpts.getOptionInt(ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
            ResourceKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD);
    initClusterStatus();

    // add the roles
    roleHistory = new RoleHistory(providerRoles);
    roleHistory.onStart(fs, historyDir);

    //rebuild any live containers
    rebuildModelFromRestart(liveContainers);

    // any am config options to pick up

    logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, "");
    //mark as live
    applicationLive = true;
}

---

From source file:org.apache.tez.dag.app.dag.impl.TaskAttemptImpl.java

License:Apache License

protected void logJobHistoryAttemptStarted() {
    final String containerIdStr = containerId.toString();
    String inProgressLogsUrl = nodeHttpAddress + "/" + "node/containerlogs" + "/" + containerIdStr + "/"
            + this.appContext.getUser();
    String completedLogsUrl = "";
    if (conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
            YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)
            && conf.get(YarnConfiguration.YARN_LOG_SERVER_URL) != null) {
        String contextStr = "v_" + getTask().getVertex().getName() + "_" + this.attemptId.toString();
        completedLogsUrl = conf.get(YarnConfiguration.YARN_LOG_SERVER_URL) + "/" + containerNodeId.toString()
                + "/" + containerIdStr + "/" + contextStr + "/" + this.appContext.getUser();
    }//w ww  . j a va2s  .  c o m
    TaskAttemptStartedEvent startEvt = new TaskAttemptStartedEvent(attemptId, getTask().getVertex().getName(),
            launchTime, containerId, containerNodeId, inProgressLogsUrl, completedLogsUrl, nodeHttpAddress);
    this.appContext.getHistoryHandler().handle(new DAGHistoryEvent(getDAGID(), startEvt));
}

---