org.apache.falcon.service.EntitySLAMonitoringService.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.falcon.service.EntitySLAMonitoringService.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.falcon.service;

import com.google.common.annotations.VisibleForTesting;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.LifeCycle;
import org.apache.falcon.Pair;
import org.apache.falcon.entity.ClusterHelper;
import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.FeedHelper;
import org.apache.falcon.entity.FeedInstanceStatus;
import org.apache.falcon.entity.ProcessHelper;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.feed.Cluster;
import org.apache.falcon.entity.v0.feed.Feed;
import org.apache.falcon.entity.v0.feed.Sla;
import org.apache.falcon.entity.v0.process.Clusters;
import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.expression.ExpressionHelper;
import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.falcon.jdbc.MonitoringJdbcStateStore;
import org.apache.falcon.persistence.MonitoredEntityBean;
import org.apache.falcon.persistence.PendingInstanceBean;
import org.apache.falcon.resource.InstancesResult;
import org.apache.falcon.resource.SchedulableEntityInstance;
import org.apache.falcon.security.CurrentUser;
import org.apache.falcon.util.DateUtil;
import org.apache.falcon.util.DeploymentUtil;
import org.apache.falcon.util.StartupProperties;
import org.apache.falcon.workflow.WorkflowEngineFactory;
import org.apache.falcon.workflow.engine.AbstractWorkflowEngine;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.falcon.entity.EntityUtil.getStartTime;
import static org.apache.falcon.util.DateUtil.now;

/**
 * Service to monitor Feed SLAs.
 */
public final class EntitySLAMonitoringService implements ConfigurationChangeListener, FalconService {
    private static final Logger LOG = LoggerFactory.getLogger(EntitySLAMonitoringService.class);

    private static final MonitoringJdbcStateStore MONITORING_JDBC_STATE_STORE = new MonitoringJdbcStateStore();

    private static final int ONE_MS = 1;

    private static final EntitySLAMonitoringService SERVICE = new EntitySLAMonitoringService();

    static final String TAG_CRITICAL = "Missed-SLA-High";
    static final String TAG_WARN = "Missed-SLA-Low";
    private static final long MINUTE_DELAY = 60000L;

    private static final List<LifeCycle> PROCESS_LIFE_CYCLE = Collections
            .singletonList(LifeCycle.valueOf(LifeCycle.EXECUTION.name()));

    private EntitySLAMonitoringService() {

    }

    public static EntitySLAMonitoringService get() {
        return SERVICE;
    }

    /**
     * Permissions for storePath.
     */
    private static final FsPermission STORE_PERMISSION = new FsPermission(FsAction.ALL, FsAction.NONE,
            FsAction.NONE);

    /**
     * Frequency in seconds of "status check" for pending entity instances.
     */
    private int statusCheckFrequencySeconds; // 10 minutes

    /**
     * Time Duration (in milliseconds) in future for generating pending entity instances.
     *
     * In every cycle pending entity instances are added for monitoring, till this time in future.
     */
    private int lookAheadWindowMillis; // 15 MINUTES

    /**
     * Filesystem used for serializing and deserializing.
     */
    private FileSystem fileSystem;

    /**
     * Working directory for the entity sla monitoring service.
     */
    private Path storePath;

    /**
     * Path to store the state of the monitoring service.
     */
    private Path filePath;

    @Override
    public void onAdd(Entity entity) throws FalconException {
        startEntityMonitoring(entity, false);
    }

    private void startEntityMonitoring(Entity entity, boolean isEntityUpdated) throws FalconException {
        Set<String> currentClusters = DeploymentUtil.getCurrentClusters();
        Set<String> clustersDefined = EntityUtil.getClustersDefined(entity);
        if (entity.getEntityType() == EntityType.FEED) {
            Feed feed = (Feed) entity;
            // currently sla service for feed is enabled only for fileSystemStorage
            if (feed.getLocations() != null || feed.getSla() != null || checkFeedClusterSLA(feed)) {
                for (String cluster : clustersDefined) {
                    if (currentClusters.contains(cluster)) {
                        if (FeedHelper.getSLA(cluster, feed) != null) {
                            LOG.debug("Adding feed:{} for monitoring", feed.getName());
                            if (isEntityUpdated) {
                                MONITORING_JDBC_STATE_STORE.putMonitoredEntity(feed.getName(),
                                        EntityType.FEED.toString(), now());
                            } else {
                                MONITORING_JDBC_STATE_STORE.putMonitoredEntity(feed.getName(),
                                        EntityType.FEED.toString(), getStartTime(entity, cluster));
                            }
                        }
                    }
                }
            }
        } else if (entity.getEntityType() == EntityType.PROCESS) {
            Process process = (Process) entity;
            if (process.getSla() != null || checkProcessClusterSLA(process)) {
                for (String cluster : clustersDefined) {
                    if (currentClusters.contains(cluster)) {
                        LOG.debug("Adding process:{} for monitoring", process.getName());
                        if (isEntityUpdated) {
                            MONITORING_JDBC_STATE_STORE.putMonitoredEntity(process.getName(),
                                    EntityType.PROCESS.toString(), now());
                        } else {
                            MONITORING_JDBC_STATE_STORE.putMonitoredEntity(process.getName(),
                                    EntityType.PROCESS.toString(), getStartTime(entity, cluster));
                        }
                    }
                }
            }
        }
    }

    private Boolean checkFeedClusterSLA(Feed feed) {
        for (Cluster cluster : feed.getClusters().getClusters()) {
            Sla sla = FeedHelper.getSLA(cluster, feed);
            if (sla != null) {
                return true;
            }
        }
        return false;
    }

    private Boolean checkProcessClusterSLA(Process process) {
        Clusters clusters = process.getClusters();
        for (org.apache.falcon.entity.v0.process.Cluster cluster : clusters.getClusters()) {
            org.apache.falcon.entity.v0.process.Sla sla = ProcessHelper.getSLA(cluster, process);
            if (sla != null) {
                return true;
            }
        }
        return false;
    }

    @Override
    public void onRemove(Entity entity) throws FalconException {
        Set<String> currentClusters = DeploymentUtil.getCurrentClusters();
        if (entity.getEntityType() == EntityType.FEED) {
            Feed feed = (Feed) entity;
            // currently sla service is enabled only for fileSystemStorage
            if (feed.getSla() != null && feed.getLocations() != null) {
                for (Cluster cluster : feed.getClusters().getClusters()) {
                    if (currentClusters.contains(cluster.getName()) && FeedHelper.getSLA(cluster, feed) != null) {
                        MONITORING_JDBC_STATE_STORE.deleteMonitoringEntity(feed.getName(),
                                EntityType.FEED.toString());
                        MONITORING_JDBC_STATE_STORE.deletePendingInstances(feed.getName(), cluster.getName(),
                                EntityType.FEED.toString());
                    }
                }
            }
        }
        if (entity.getEntityType() == EntityType.PROCESS) {
            Process process = (Process) entity;
            if (process.getSla() != null) {
                for (org.apache.falcon.entity.v0.process.Cluster cluster : process.getClusters().getClusters()) {
                    if (currentClusters.contains(cluster.getName())) {
                        MONITORING_JDBC_STATE_STORE.deleteMonitoringEntity(process.getName(),
                                EntityType.PROCESS.toString());
                        MONITORING_JDBC_STATE_STORE.deletePendingInstances(process.getName(), cluster.getName(),
                                EntityType.PROCESS.toString());
                    }
                }
            }
        }
    }

    private boolean isSLAMonitoringEnabledInCurrentColo(Feed feed) {
        if (feed.getLocations() != null) {
            Set<String> currentClusters = DeploymentUtil.getCurrentClusters();
            for (Cluster cluster : feed.getClusters().getClusters()) {
                if (currentClusters.contains(cluster.getName()) && FeedHelper.getSLA(cluster, feed) != null) {
                    return true;
                }
            }
        }
        return false;
    }

    private boolean isSLAMonitoringEnabledInCurrentColo(Process process) {

        Set<String> currentClusters = DeploymentUtil.getCurrentClusters();
        for (org.apache.falcon.entity.v0.process.Cluster cluster : process.getClusters().getClusters()) {
            if (currentClusters.contains(cluster.getName()) && ProcessHelper.getSLA(cluster, process) != null) {
                return true;
            }
        }
        return false;
    }

    @Override
    public void onChange(Entity oldEntity, Entity newEntity) throws FalconException {
        if (newEntity.getEntityType() == EntityType.FEED) {
            Feed oldFeed = (Feed) oldEntity;
            Feed newFeed = (Feed) newEntity;
            if (!isSLAMonitoringEnabledInCurrentColo(newFeed)) {
                onRemove(oldFeed);
            } else if (!isSLAMonitoringEnabledInCurrentColo(oldFeed)) {
                onAdd(newFeed);
            } else {
                List<String> slaRemovedClusters = new ArrayList<>();
                for (String oldCluster : EntityUtil.getClustersDefinedInColos(oldFeed)) {
                    if (FeedHelper.getSLA(oldCluster, oldFeed) != null
                            && FeedHelper.getSLA(oldCluster, newFeed) == null) {
                        slaRemovedClusters.add(oldCluster);
                    }
                }
                updatePendingInstances(newFeed.getName(), slaRemovedClusters, EntityType.FEED.toString());
            }
        }
        if (newEntity.getEntityType() == EntityType.PROCESS) {
            Process oldProcess = (Process) oldEntity;
            Process newProcess = (Process) newEntity;
            if (!isSLAMonitoringEnabledInCurrentColo(newProcess)) {
                onRemove(oldProcess);
            } else if (!isSLAMonitoringEnabledInCurrentColo(oldProcess)) {
                onAdd(newProcess);
            } else {
                List<String> slaRemovedClusters = new ArrayList<>();
                for (String oldCluster : EntityUtil.getClustersDefined(oldProcess)) {
                    if (ProcessHelper.getSLA(oldCluster, oldProcess) != null
                            && ProcessHelper.getSLA(oldCluster, newProcess) == null) {
                        slaRemovedClusters.add(oldCluster);
                    }
                }
                updatePendingInstances(newProcess.getName(), slaRemovedClusters, EntityType.PROCESS.toString());
            }
        }
    }

    private void updatePendingInstances(String entityName, List<String> slaRemovedClusters, String entityType) {
        for (String clusterName : slaRemovedClusters) {
            MONITORING_JDBC_STATE_STORE.deletePendingInstances(entityName, clusterName, entityType);
        }
    }

    @Override
    public void onReload(Entity entity) throws FalconException {
    }

    @Override
    public String getName() {
        return EntitySLAMonitoringService.class.getSimpleName();
    }

    @Override
    public void init() throws FalconException {

        String freq = StartupProperties.get().getProperty("entity.sla.statusCheck.frequency.seconds", "600");
        statusCheckFrequencySeconds = Integer.parseInt(freq);

        freq = StartupProperties.get().getProperty("entity.sla.lookAheadWindow.millis", "900000");
        lookAheadWindowMillis = Integer.parseInt(freq);
        ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
        addPendingEntityInstances(now());
        executor.scheduleWithFixedDelay(new Monitor(), 0, statusCheckFrequencySeconds, TimeUnit.SECONDS);
    }

    public void makeFeedInstanceAvailable(String feedName, String clusterName, Date nominalTime)
            throws FalconException {
        LOG.debug("Removing {} feed's instance {} in cluster {} from pendingSLA", feedName, clusterName,
                nominalTime);
        List<Date> instances = (MONITORING_JDBC_STATE_STORE.getNominalInstances(feedName, clusterName,
                EntityType.FEED.toString()));
        // Slas for feeds not having sla tag are not stored.
        if (CollectionUtils.isEmpty(instances)) {
            MONITORING_JDBC_STATE_STORE.deletePendingInstance(feedName, clusterName, nominalTime,
                    EntityType.FEED.toString());
        }
    }

    private FileSystem initializeFileSystem() {
        try {
            fileSystem = HadoopClientFactory.get().createFalconFileSystem(storePath.toUri());
            if (!fileSystem.exists(storePath)) {
                LOG.info("Creating directory for pending entity instances: {}", storePath);
                // set permissions so config store dir is owned by falcon alone
                HadoopClientFactory.mkdirs(fileSystem, storePath, STORE_PERMISSION);
            }
            return fileSystem;
        } catch (Exception e) {
            throw new RuntimeException("Unable to bring up entity sla store for path: " + storePath, e);
        }
    }

    @Override
    public void destroy() throws FalconException {
    }

    //Periodically update status of pending instances, add new instances and take backup.
    private class Monitor implements Runnable {

        @Override
        public void run() {
            try {
                if (MONITORING_JDBC_STATE_STORE.getAllMonitoredEntities().size() > 0) {
                    checkPendingInstanceAvailability();

                    // add Instances from last checked time to 10 minutes from now(some buffer for status check)
                    Date newCheckPointTime = new Date(now().getTime() + lookAheadWindowMillis);
                    addPendingEntityInstances(newCheckPointTime);
                }
            } catch (Throwable e) {
                LOG.error("Feed SLA monitoring failed: ", e);
            }
        }
    }

    private void addPendingInstances(String entityType, Entity entity, String clusterName, List<Date> instances)
            throws FalconException {
        if (instances != null && !instances.isEmpty()) {
            for (Date date : instances) {
                LOG.debug("Adding pending instance ={} for entity= {} in cluster>={} and entityType={}", date,
                        entity.getName(), clusterName, entityType);
                MONITORING_JDBC_STATE_STORE.putPendingInstances(entity.getName(), clusterName, date, entityType);
            }
        }
    }

    private void addPendingEntityInstances(Date checkPointTime) throws FalconException {
        Set<String> currentClusters = DeploymentUtil.getCurrentClusters();
        List<MonitoredEntityBean> entityBeanList = MONITORING_JDBC_STATE_STORE.getAllMonitoredEntities();
        for (MonitoredEntityBean monitoredEntityBean : entityBeanList) {
            String entityName = monitoredEntityBean.getEntityName();
            String entityType = monitoredEntityBean.getEntityType();
            if (EntityType.FEED.name().equalsIgnoreCase(entityType)
                    || isEntityRunning(EntityUtil.getEntity(entityType, entityName))) {
                Date lastMonitoredInstanceTime = monitoredEntityBean.getLastMonitoredTime();
                Date newCheckPointTime = checkPointTime;
                Entity entity = EntityUtil.getEntity(entityType, entityName);
                Set<String> clustersDefined = EntityUtil.getClustersDefined(entity);
                List<org.apache.falcon.entity.v0.cluster.Cluster> clusters = new ArrayList();
                for (String cluster : clustersDefined) {
                    clusters.add(ClusterHelper.getCluster(cluster));
                }
                for (org.apache.falcon.entity.v0.cluster.Cluster entityCluster : clusters) {
                    if (currentClusters.contains(entityCluster.getName())) {
                        Date endTime = EntityUtil.getEndTime(entity, entityCluster.getName());
                        if (endTime.before(now())) {
                            newCheckPointTime = endTime;
                        }
                        List<Date> instances = EntityUtil.getEntityInstanceTimesInBetween(entity,
                                entityCluster.getName(), lastMonitoredInstanceTime, newCheckPointTime);
                        addPendingInstances(entityType, entity, entityCluster.getName(), instances);
                        // update last monitored time with the new checkpoint time
                        MONITORING_JDBC_STATE_STORE.updateLastMonitoredTime(entityName, entityType,
                                new Date(newCheckPointTime.getTime() + MINUTE_DELAY));
                    }
                }
            }
        }
    }

    /**
     * Checks the availability of all the pendingInstances and removes the ones which have become available.
     */
    private void checkPendingInstanceAvailability() throws FalconException {
        List<PendingInstanceBean> pendingInstanceBeans = MONITORING_JDBC_STATE_STORE.getAllPendingInstances();
        if (pendingInstanceBeans.isEmpty()) {
            LOG.info("No pending instances to be checked");
            return;
        }
        for (PendingInstanceBean pendingInstanceBean : pendingInstanceBeans) {
            boolean status = checkEntityInstanceAvailability(pendingInstanceBean.getEntityName(),
                    pendingInstanceBean.getClusterName(), pendingInstanceBean.getNominalTime(),
                    pendingInstanceBean.getEntityType());
            if (status) {
                MONITORING_JDBC_STATE_STORE.deletePendingInstance(pendingInstanceBean.getEntityName(),
                        pendingInstanceBean.getClusterName(), pendingInstanceBean.getNominalTime(),
                        pendingInstanceBean.getEntityType());
            }
        }
    }

    // checks whether a given entity instance is available or not
    private boolean checkEntityInstanceAvailability(String entityName, String clusterName, Date nominalTime,
            String entityType) throws FalconException {
        Entity entity = EntityUtil.getEntity(entityType, entityName);
        authenticateUser(entity);
        try {
            if (entityType.equalsIgnoreCase(EntityType.PROCESS.toString())) {
                LOG.trace("Checking instance availability status for entity:{}, cluster:{}, " + "instanceTime:{}",
                        entity.getName(), clusterName, nominalTime, entityType);
                AbstractWorkflowEngine wfEngine = WorkflowEngineFactory.getWorkflowEngine();

                InstancesResult instancesResult = wfEngine.getStatus(entity, nominalTime,
                        new Date(nominalTime.getTime() + 200), PROCESS_LIFE_CYCLE, false);
                if (instancesResult.getInstances().length > 0) {
                    if (instancesResult.getInstances()[0].status.equals(InstancesResult.WorkflowStatus.SUCCEEDED)) {
                        LOG.trace("Entity instance(Process:{}, cluster:{}, instanceTime:{}) is available.",
                                entity.getName(), clusterName, nominalTime);
                        return true;
                    }
                }
                return false;
            }
            if (entityType.equalsIgnoreCase(EntityType.FEED.toString())) {
                LOG.trace("Checking instance availability status for feed:{}, cluster:{}, instanceTime:{}",
                        entity.getName(), clusterName, nominalTime);

                FeedInstanceStatus.AvailabilityStatus status = FeedHelper.getFeedInstanceStatus((Feed) entity,
                        clusterName, nominalTime);
                if (status.equals(FeedInstanceStatus.AvailabilityStatus.AVAILABLE)
                        || status.equals(FeedInstanceStatus.AvailabilityStatus.EMPTY)) {
                    LOG.trace("Feed instance(feed:{}, cluster:{}, instanceTime:{}) is available.", entity.getName(),
                            clusterName, nominalTime);
                    return true;
                }
            }
        } catch (Throwable e) {
            LOG.error("Couldn't find status for Entity:{}, cluster:{}, entityType{}, nominalTime{}", entityName,
                    clusterName, entityType, nominalTime, e);
        }
        LOG.debug("Entity instance(entity:{}, cluster:{}, instanceTime:{}) is not available.", entity.getName(),
                clusterName, nominalTime);
        return false;
    }

    /**
     * Returns all the instances between given time range which have missed slaLow or slaHigh for given entity.
     *
     * Only entities which have defined sla in their definition are considered.
     * Only the entity instances between the given time range are considered.
     * Start time and end time are both inclusive.
     * @param start start time, inclusive
     * @param end end time, inclusive
     * @return Set of pending entity instances belonging to the given range which have missed SLA
     * @throws FalconException
     */
    public Set<SchedulableEntityInstance> getEntitySLAMissPendingAlerts(Date start, Date end)
            throws FalconException {
        Set<SchedulableEntityInstance> result = new HashSet<>();
        for (PendingInstanceBean pendingInstanceBean : MONITORING_JDBC_STATE_STORE.getAllPendingInstances()) {
            Pair<String, String> entityClusterPair = new Pair<>(pendingInstanceBean.getEntityName(),
                    pendingInstanceBean.getClusterName());
            String entityType = pendingInstanceBean.getEntityType();
            if (entityType.equalsIgnoreCase(EntityType.FEED.toString())) {
                Feed feed = EntityUtil.getEntity(entityType, entityClusterPair.first);
                Cluster cluster = FeedHelper.getCluster(feed, entityClusterPair.second);
                Sla sla = FeedHelper.getSLA(cluster, feed);
                if (sla != null) {
                    Set<Pair<Date, String>> slaStatus = getFeedSLAStatus(sla, start, end,
                            MONITORING_JDBC_STATE_STORE.getNominalInstances(entityClusterPair.first,
                                    entityClusterPair.second, entityType));
                    for (Pair<Date, String> status : slaStatus) {
                        SchedulableEntityInstance instance = new SchedulableEntityInstance(entityClusterPair.first,
                                entityClusterPair.second, status.first, EntityType.FEED);
                        instance.setTags(status.second);
                        result.add(instance);
                    }
                }
            } else {
                Process process = EntityUtil.getEntity(entityType, entityClusterPair.first);
                org.apache.falcon.entity.v0.process.Cluster cluster = ProcessHelper.getCluster(process,
                        entityClusterPair.second);
                org.apache.falcon.entity.v0.process.Sla sla = ProcessHelper.getSLA(cluster, process);
                if (sla != null && isEntityRunning(process)) {
                    Set<Pair<Date, String>> slaStatus = getProcessSLAStatus(sla, start, end,
                            MONITORING_JDBC_STATE_STORE.getNominalInstances(entityClusterPair.first,
                                    entityClusterPair.second, entityType));
                    for (Pair<Date, String> status : slaStatus) {
                        SchedulableEntityInstance instance = new SchedulableEntityInstance(entityClusterPair.first,
                                entityClusterPair.second, status.first, EntityType.PROCESS);
                        instance.setTags(status.second);
                        result.add(instance);
                    }
                }
            }
        }
        return result;
    }

    /**
     * Returns all the instances of a given entity between the given time range
     * which missed sla.Only those instances are included which have missed either slaLow or slaHigh.
     * @param entityName name of the feed
     * @param clusterName cluster name
     * @param start start time, inclusive
     * @param end end time, inclusive
     * @return Pending instances of the given entity which belong to the given time range and have missed SLA.
     * @throws FalconException
     */
    public Set<SchedulableEntityInstance> getEntitySLAMissPendingAlerts(String entityName, String clusterName,
            Date start, Date end, String entityType) throws FalconException {
        Set<SchedulableEntityInstance> result = new HashSet<>();
        List<Date> missingInstances = MONITORING_JDBC_STATE_STORE.getNominalInstances(entityName, clusterName,
                entityType);
        if (missingInstances == null) {
            return result;
        }
        Entity entity = EntityUtil.getEntity(entityType, entityName);

        if (entityType.equals(EntityType.FEED.toString())) {
            Sla sla = FeedHelper.getSLA(clusterName, (Feed) entity);

            if (sla != null) {
                Set<Pair<Date, String>> slaStatus = getFeedSLAStatus(sla, start, end, missingInstances);
                for (Pair<Date, String> status : slaStatus) {
                    SchedulableEntityInstance instance = new SchedulableEntityInstance(entityName, clusterName,
                            status.first, EntityType.FEED);
                    instance.setTags(status.second);
                    result.add(instance);
                }
            }
            return result;
        } else {
            org.apache.falcon.entity.v0.process.Sla sla = ProcessHelper.getSLA(clusterName, (Process) entity);
            if (sla != null) {
                Set<Pair<Date, String>> slaStatus = getProcessSLAStatus(sla, start, end, missingInstances);
                for (Pair<Date, String> status : slaStatus) {
                    SchedulableEntityInstance instance = new SchedulableEntityInstance(entityName, clusterName,
                            status.first, EntityType.PROCESS);
                    instance.setTags(status.second);
                    result.add(instance);
                }
            }
        }
        return result;
    }

    Set<Pair<Date, String>> getFeedSLAStatus(Sla sla, Date start, Date end, List<Date> missingInstances)
            throws FalconException {
        Date now = new Date();
        Frequency slaLow = sla.getSlaLow();
        Frequency slaHigh = sla.getSlaHigh();
        Set<Pair<Date, String>> result = new HashSet<>();
        for (Date nominalTime : missingInstances) {
            if (!nominalTime.before(start) && !nominalTime.after(end)) {
                ExpressionHelper.setReferenceDate(nominalTime);
                ExpressionHelper evaluator = ExpressionHelper.get();
                Long slaHighDuration = evaluator.evaluate(slaHigh.toString(), Long.class);
                Long slaLowDuration = evaluator.evaluate(slaLow.toString(), Long.class);
                Date slaCriticalTime = new Date(nominalTime.getTime() + slaHighDuration);
                Date slaWarnTime = new Date(nominalTime.getTime() + slaLowDuration);
                if (slaCriticalTime.before(now)) {
                    result.add(new Pair<>(nominalTime, TAG_CRITICAL));
                } else if (slaWarnTime.before(now)) {
                    result.add(new Pair<>(nominalTime, TAG_WARN));
                }
            }
        }
        return result;
    }

    private Set<Pair<Date, String>> getProcessSLAStatus(org.apache.falcon.entity.v0.process.Sla sla, Date start,
            Date end, List<Date> missingInstances) throws FalconException {
        Date now = new Date();
        Frequency slaHigh = sla.getShouldEndIn();
        Set<Pair<Date, String>> result = new HashSet<>();
        for (Date nominalTime : missingInstances) {
            if (!nominalTime.before(start) && !nominalTime.after(end)) {
                ExpressionHelper.setReferenceDate(nominalTime);
                ExpressionHelper evaluator = ExpressionHelper.get();
                Long slaHighDuration = evaluator.evaluate(slaHigh.toString(), Long.class);
                Date slaCriticalTime = new Date(nominalTime.getTime() + slaHighDuration);
                if (slaCriticalTime.before(now)) {
                    result.add(new Pair<>(nominalTime, TAG_CRITICAL));
                }
            }
        }
        return result;
    }

    @VisibleForTesting
    Date getInitialStartTime(Entity entity, String clusterName, String entityType) throws FalconException {
        if (entityType.equals(EntityType.FEED.toString())) {
            Sla sla = FeedHelper.getSLA(clusterName, (Feed) entity);
            if (sla == null) {
                throw new IllegalStateException("InitialStartTime can not be determined as the feed: "
                        + entity.getName() + " and cluster: " + clusterName + " does not have any sla");
            }
            Date startTime = FeedHelper.getFeedValidityStart((Feed) entity, clusterName);
            Frequency slaLow = sla.getSlaLow();
            Date slaTime = new Date(now().getTime() - DateUtil.getFrequencyInMillis(slaLow));
            return startTime.before(slaTime) ? startTime : slaTime;
        } else {
            org.apache.falcon.entity.v0.process.Sla sla = ProcessHelper.getSLA(clusterName, (Process) entity);
            if (sla == null) {
                throw new IllegalStateException("InitialStartTime can not be determined as the feed: "
                        + entity.getName() + " and cluster: " + clusterName + " does not have any sla");
            }
            Date startTime = ProcessHelper.getProcessValidityStart((Process) entity, clusterName);
            Frequency slaLow = sla.getShouldEndIn();
            Date slaTime = new Date(now().getTime() - DateUtil.getFrequencyInMillis(slaLow));
            return startTime.before(slaTime) ? startTime : slaTime;
        }
    }

    public void makeProcessInstanceAvailable(String clusterName, String entityName, String date,
            String entityType) {
        Date nominalTime = null;
        try {
            nominalTime = DateUtil.parseDateFalconTZ(date);
        } catch (ParseException e) {
            LOG.error("Exception while translating the date:", e);
        }
        if (nominalTime != null) {
            List<Date> instances = (MONITORING_JDBC_STATE_STORE.getNominalInstances(entityName, clusterName,
                    entityType));
            if (!CollectionUtils.isEmpty(instances)) {
                MONITORING_JDBC_STATE_STORE.deletePendingInstance(entityName, clusterName, nominalTime, entityType);
            }
        }
    }

    // Authenticate user only if not already authenticated.
    private void authenticateUser(Entity entity) {
        if (!CurrentUser.isAuthenticated()) {
            if (StringUtils.isNotBlank(entity.getACL().getOwner())) {
                CurrentUser.authenticate(entity.getACL().getOwner());
            } else {
                CurrentUser.authenticate(System.getProperty("user.name"));
            }
        }
    }

    private boolean isEntityRunning(Entity entity) throws FalconException {
        authenticateUser(entity);
        AbstractWorkflowEngine workflowEngine = WorkflowEngineFactory.getWorkflowEngine();
        return workflowEngine.isActive(entity) && !workflowEngine.isSuspended(entity)
                && !workflowEngine.isCompleted(entity);
    }
}