org.jboss.pnc.environment.openshift.OpenshiftStartedEnvironment.java Source code

Java tutorial

Introduction

Here is the source code for org.jboss.pnc.environment.openshift.OpenshiftStartedEnvironment.java

Source

/**
 * JBoss, Home of Professional Open Source.
 * Copyright 2014-2019 Red Hat, Inc., and individual contributors
 * as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jboss.pnc.environment.openshift;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.io.JsonStringEncoder;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.openshift.internal.restclient.model.Pod;
import com.openshift.internal.restclient.model.Route;
import com.openshift.internal.restclient.model.Service;
import com.openshift.internal.restclient.model.properties.ResourcePropertiesRegistry;
import com.openshift.restclient.ClientBuilder;
import com.openshift.restclient.IClient;
import com.openshift.restclient.NotFoundException;
import com.openshift.restclient.ResourceKind;
import com.openshift.restclient.model.IResource;
import org.apache.commons.lang.RandomStringUtils;
import org.jboss.dmr.ModelNode;
import org.jboss.pnc.common.json.moduleconfig.OpenshiftBuildAgentConfig;
import org.jboss.pnc.common.json.moduleconfig.OpenshiftEnvironmentDriverModuleConfig;
import org.jboss.pnc.common.monitor.PullingMonitor;
import org.jboss.pnc.common.monitor.RunningTask;
import org.jboss.pnc.common.util.RandomUtils;
import org.jboss.pnc.common.util.StringUtils;
import org.jboss.pnc.environment.openshift.exceptions.PodFailedStartException;
import org.jboss.pnc.pncmetrics.GaugeMetric;
import org.jboss.pnc.pncmetrics.MetricsConfiguration;
import org.jboss.pnc.spi.builddriver.DebugData;
import org.jboss.pnc.spi.environment.RunningEnvironment;
import org.jboss.pnc.spi.environment.StartedEnvironment;
import org.jboss.pnc.spi.repositorymanager.model.RepositorySession;
import org.jboss.util.StringPropertyReplacer;
import org.jboss.util.collection.ConcurrentSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.function.Consumer;
import java.util.regex.Pattern;

/**
 * @author <a href="mailto:matejonnet@gmail.com">Matej Lazar</a>
 */
public class OpenshiftStartedEnvironment implements StartedEnvironment {

    private static final Logger logger = LoggerFactory.getLogger(OpenshiftStartedEnvironment.class);
    private static final String SSH_SERVICE_PORT_NAME = "2222-ssh";
    private static final String POD_USERNAME = "worker";
    private static final String POD_USER_PASSWD = "workerUserPassword";
    private static final String OSE_API_VERSION = "v1";
    private static final Pattern SECURE_LOG_PATTERN = Pattern
            .compile("\"name\":\\s*\"accessToken\",\\s*\"value\":\\s*\"\\p{Print}+\"");

    private static final String METRICS_POD_STARTED_KEY = "openshift-environment-driver.started.pod";
    private static final String METRICS_POD_STARTED_ATTEMPTED_KEY = METRICS_POD_STARTED_KEY + ".attempts";
    private static final String METRICS_POD_STARTED_SUCCESS_KEY = METRICS_POD_STARTED_KEY + ".success";
    private static final String METRICS_POD_STARTED_FAILED_KEY = METRICS_POD_STARTED_KEY + ".failed";
    private static final String METRICS_POD_STARTED_RETRY_KEY = METRICS_POD_STARTED_KEY + ".retries";
    private static final String METRICS_POD_STARTED_FAILED_REASON_KEY = METRICS_POD_STARTED_KEY + ".failed_reason";

    private static final int DEFAULT_CREATION_POD_RETRY = 1;

    private int creationPodRetry;

    /**
     * From: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
     *
     * ErrImagePull and ImagePullBackOff added to that list. The pod.getStatus() call will return the *reason* of failure,
     * and if the reason is not available, then it'll return the regular status (as mentioned in the link)
     *
     * For pod creation, the failure reason we expect when docker registry is not behaving is 'ErrImagePull' or 'ImagePullBackOff'
     */
    private static final String[] POD_FAILED_STATUSES = { "Failed", "Unknown", "CrashLoopBackOff", "ErrImagePull",
            "ImagePullBackOff" };

    /**
     * Parameter specifying override for the builder pod memory size.
     */
    private static final String BUILDER_POD_MEMORY = "BUILDER_POD_MEMORY";

    private final IClient client;
    private final RepositorySession repositorySession;
    private final OpenshiftBuildAgentConfig openshiftBuildAgentConfig;
    private final OpenshiftEnvironmentDriverModuleConfig environmentConfiguration;
    private final PullingMonitor pullingMonitor;
    private final String imageId;
    private final DebugData debugData;
    private final Set<Selector> initialized = new HashSet<>();
    private final Map<String, String> runtimeProperties;

    private final ExecutorService executor;
    private Optional<GaugeMetric> gaugeMetric = Optional.empty();

    private Pod pod;
    private Service service;
    private Route route;
    private Service sshService;

    private ConcurrentSet<RunningTask> runningTaskMonitors = new ConcurrentSet<>();

    private String buildAgentContextPath;

    private final boolean createRoute;

    private Runnable cancelHook;
    private boolean cancelRequested = false;

    private Optional<CompletableFuture<Void>> creatingPod = Optional.empty();
    private Optional<CompletableFuture<Void>> creatingService = Optional.empty();
    private Optional<CompletableFuture<Void>> creatingRoute = Optional.empty();

    // Used to track whether all the futures for creation are completed, or failed with an exception
    private CompletableFuture<Void> creationCompletableFutures;

    public OpenshiftStartedEnvironment(ExecutorService executor,
            OpenshiftBuildAgentConfig openshiftBuildAgentConfig,
            OpenshiftEnvironmentDriverModuleConfig environmentConfiguration, PullingMonitor pullingMonitor,
            RepositorySession repositorySession, String systemImageId, DebugData debugData, String accessToken,
            boolean tempBuild, Instant temporaryBuildExpireDate, MetricsConfiguration metricsConfiguration,
            Map<String, String> parameters) {

        creationPodRetry = DEFAULT_CREATION_POD_RETRY;

        if (environmentConfiguration.getCreationPodRetry() != null) {
            try {
                creationPodRetry = Integer.parseInt(environmentConfiguration.getCreationPodRetry());
            } catch (NumberFormatException e) {
                logger.error(
                        "Couldn't parse the value of creation pod retry from the configuration. Using default");
            }
        }

        logger.info("Creating new build environment using image id: " + environmentConfiguration.getImageId());

        this.executor = executor;
        this.openshiftBuildAgentConfig = openshiftBuildAgentConfig;
        this.environmentConfiguration = environmentConfiguration;
        this.pullingMonitor = pullingMonitor;
        this.repositorySession = repositorySession;
        this.imageId = systemImageId == null ? environmentConfiguration.getImageId() : systemImageId;
        this.debugData = debugData;
        if (metricsConfiguration != null) {
            this.gaugeMetric = Optional.of(metricsConfiguration.getGaugeMetric());
        }

        createRoute = environmentConfiguration.getExposeBuildAgentOnPublicUrl();

        client = new ClientBuilder(environmentConfiguration.getRestEndpointUrl())
                .usingToken(environmentConfiguration.getRestAuthToken()).build();
        client.getServerReadyStatus(); // make sure client is connected

        runtimeProperties = new HashMap<>();

        final String buildAgentHost = environmentConfiguration.getBuildAgentHost();
        String expiresDateStamp = Long.toString(temporaryBuildExpireDate.toEpochMilli());

        runtimeProperties.put("build-agent-host", buildAgentHost);
        runtimeProperties.put("containerPort", environmentConfiguration.getContainerPort());
        runtimeProperties.put("buildContentId", repositorySession.getBuildRepositoryId());
        runtimeProperties.put("accessToken", accessToken);
        runtimeProperties.put("tempBuild", Boolean.toString(tempBuild));
        runtimeProperties.put("expiresDate", "ts" + expiresDateStamp);
        runtimeProperties.put("resourcesMemory", builderPodMemory(environmentConfiguration, parameters));

        createEnvironment();
    }

    private void createEnvironment() {

        List<CompletableFuture<Void>> trackCreationFutures = new ArrayList<>();

        String randString = RandomUtils.randString(6);//note the 24 char limit
        buildAgentContextPath = "pnc-ba-" + randString;

        runtimeProperties.put("pod-name", "pnc-ba-pod-" + randString);
        runtimeProperties.put("service-name", "pnc-ba-service-" + randString);
        runtimeProperties.put("ssh-service-name", "pnc-ba-ssh-" + randString);
        runtimeProperties.put("route-name", "pnc-ba-route-" + randString);
        runtimeProperties.put("route-path", "/" + buildAgentContextPath);
        runtimeProperties.put("buildAgentContextPath", "/" + buildAgentContextPath);

        initDebug();

        ModelNode podConfigurationNode = createModelNode(
                Configurations.getContentAsString(Resource.PNC_BUILDER_POD, openshiftBuildAgentConfig),
                runtimeProperties);
        pod = new Pod(podConfigurationNode, client,
                ResourcePropertiesRegistry.getInstance().get(OSE_API_VERSION, ResourceKind.POD));
        pod.setNamespace(environmentConfiguration.getPncNamespace());
        Runnable createPod = () -> {
            try {
                client.create(pod, pod.getNamespace());
            } catch (Throwable e) {
                logger.error("Cannot create pod.", e);
                throw e;
            }
        };
        creatingPod = Optional.of(CompletableFuture.runAsync(createPod, executor));
        trackCreationFutures.add(creatingPod.get());

        ModelNode serviceConfigurationNode = createModelNode(
                Configurations.getContentAsString(Resource.PNC_BUILDER_SERVICE, openshiftBuildAgentConfig),
                runtimeProperties);
        service = new Service(serviceConfigurationNode, client,
                ResourcePropertiesRegistry.getInstance().get(OSE_API_VERSION, ResourceKind.SERVICE));
        service.setNamespace(environmentConfiguration.getPncNamespace());
        Runnable createService = () -> {
            try {
                client.create(service, service.getNamespace());
            } catch (Throwable e) {
                logger.error("Cannot create service.", e);
                throw e;
            }
        };
        creatingService = Optional.of(CompletableFuture.runAsync(createService, executor));
        trackCreationFutures.add(creatingService.get());

        if (createRoute) {
            ModelNode routeConfigurationNode = createModelNode(
                    Configurations.getContentAsString(Resource.PNC_BUILDER_ROUTE, openshiftBuildAgentConfig),
                    runtimeProperties);
            route = new Route(routeConfigurationNode, client,
                    ResourcePropertiesRegistry.getInstance().get(OSE_API_VERSION, ResourceKind.ROUTE));
            route.setNamespace(environmentConfiguration.getPncNamespace());
            Runnable createRoute = () -> {
                try {
                    client.create(route, route.getNamespace());
                } catch (Throwable e) {
                    logger.error("Cannot create route.", e);
                    throw e;
                }
            };
            creatingRoute = Optional.of(CompletableFuture.runAsync(createRoute, executor));
            trackCreationFutures.add(creatingRoute.get());
        }
        gaugeMetric.ifPresent(g -> g.incrementMetric(METRICS_POD_STARTED_ATTEMPTED_KEY));
    }

    private String builderPodMemory(OpenshiftEnvironmentDriverModuleConfig environmentConfiguration1,
            Map<String, String> parameters) {
        double builderPodMemory = environmentConfiguration1.getBuilderPodMemory();
        String builderPodMemoryOverride = parameters.get(BUILDER_POD_MEMORY);
        if (builderPodMemoryOverride != null) {
            try {
                builderPodMemory = Double.parseDouble(builderPodMemoryOverride);
            } catch (NumberFormatException ex) {
                throw new IllegalArgumentException("Failed to parse memory size '" + builderPodMemoryOverride
                        + "' from " + BUILDER_POD_MEMORY + " parameter.", ex);
            }
            logger.info("Using override for builder pod memory size: " + builderPodMemoryOverride);
        }
        return ((int) Math.ceil(builderPodMemory * 1024)) + "Mi";
    }

    static String secureLog(String message) {
        return SECURE_LOG_PATTERN.matcher(message)
                .replaceAll("\"name\": \"accessToken\",\n" + "            \"value\": \"***\"");
    }

    private void initDebug() {
        if (debugData.isEnableDebugOnFailure()) {
            String password = RandomStringUtils.randomAlphanumeric(10);
            debugData.setSshPassword(password);
            runtimeProperties.put(POD_USER_PASSWD, password);

            debugData.setSshServiceInitializer(d -> {
                Integer port = startSshService();
                d.setSshCommand("ssh " + POD_USERNAME + "@" + route.getHost() + " -p " + port);
            });
        }
    }

    private ModelNode createModelNode(String resourceDefinition, Map<String, String> runtimeProperties) {
        String definition = replaceConfigurationVariables(resourceDefinition, runtimeProperties);
        if (logger.isTraceEnabled()) {
            logger.trace("Node definition: " + secureLog(definition));
        }

        return ModelNode.fromJSONString(definition);
    }

    /**
     * Method to retry creating the whole Openshift environment in case of failure
     *
     * @param e exception thrown
     * @param onComplete consumer to call if successful
     * @param onError consumer to call if no more retries
     * @param retries how many times will we retry starting the build environment
     */
    private void retryPod(Exception e, Consumer<RunningEnvironment> onComplete, Consumer<Exception> onError,
            int retries) {

        gaugeMetric.ifPresent(g -> g.incrementMetric(METRICS_POD_STARTED_FAILED_KEY));

        logger.debug("Cancelling existing monitors for this build environment");
        cancelAndClearMonitors();

        // no more retries, execute the onError consumer
        if (retries == 0) {
            onError.accept(e);

        } else {
            logger.error("Creating build environment failed! Retrying...");
            gaugeMetric.ifPresent(g -> g.incrementMetric(METRICS_POD_STARTED_RETRY_KEY));

            // since deletion runs in an executor, it might run *after* the createEnvironment() is finished.
            // createEnvironment()  will overwrite the Openshift object fields. So we need to capture the existing
            // openshift objects to delete before they get overwritten by createEnvironment()
            Route routeToDestroy = route;
            Service serviceToDestroy = service;
            Service sshServiceToDestroy = sshService;
            Pod podToDestroy = pod;

            executor.submit(() -> {
                try {
                    logger.debug("Destroying old build environment");
                    destroyEnvironment(routeToDestroy, serviceToDestroy, sshServiceToDestroy, podToDestroy, true);
                } catch (Exception ex) {
                    logger.error("Error deleting previous environment", ex);
                }
            });

            logger.debug("Creating new build environment");
            createEnvironment();

            // restart the process again
            monitorInitialization(onComplete, onError, retries - 1);
            // at this point the running task running this is finished. New ones are created to monitor pod /service/route creation
        }

    }

    /**
     * Call stack:
     *   monitorInitialization:
     *       -> setup monitors, track them and return
     *
     * -> pullingMonitor.monitor(<pod>) [in background]
     *    -> Success: signal via executing onComplete consumer
     *                finish
     *    -> Failure: call retryPod consumer
     *       -> if retries == 0: call onError consumer. no more retries
     *       -> else: cancel and clear monitors,
     *                delete existing build environment (if any),
     *                recreate build environment,
     *                call monitorInitialization again with retries decremented
     *                finish
     *
     *  While the call stack may appear recursive, it's not in fact recursive due to the fact that we are using RunningTask
     *  to figure out if the pod / route /service are online or not and they run in the background
     */
    @Override
    public void monitorInitialization(Consumer<RunningEnvironment> onComplete, Consumer<Exception> onError) {
        monitorInitialization(onComplete, onError, creationPodRetry);
    }

    /**
     * retries is decremented in retryPod in case of pod failing to start
     *
     * @param onComplete
     * @param onError
     * @param retries
     */
    private void monitorInitialization(Consumer<RunningEnvironment> onComplete, Consumer<Exception> onError,
            int retries) {

        Consumer<RunningEnvironment> onCompleteInternal = (runningEnvironment) -> {
            logger.info("New build environment available on internal url: {}", getInternalEndpointUrl());

            try {
                Runnable onUrlAvailable = () -> onComplete.accept(runningEnvironment);

                URL url = new URL(getInternalEndpointUrl());
                addMonitors(pullingMonitor.monitor(onUrlAvailable, onError, () -> isServletAvailable(url)));
            } catch (IOException e) {
                onError.accept(e);
            }
        };

        Consumer<Exception> onErrorInternal = (exception) -> {
            cancelAndClearMonitors();
            onError.accept(exception);
        };

        cancelHook = () -> onComplete.accept(null);

        creatingPod.ifPresent((f) -> f.thenRunAsync(() -> {
            addMonitors(pullingMonitor.monitor(onEnvironmentInitComplete(onCompleteInternal, Selector.POD),
                    (t) -> this.retryPod(t, onComplete, onError, retries), this::isPodRunning));
        }));

        creatingService.ifPresent((f) -> f.thenRunAsync(() -> {
            addMonitors(pullingMonitor.monitor(onEnvironmentInitComplete(onCompleteInternal, Selector.SERVICE),
                    onErrorInternal, this::isServiceRunning));
        }));

        logger.info("Waiting to initialize environment. Pod [{}]; Service [{}].", pod.getName(), service.getName());

        creatingRoute.ifPresent((f) -> f.thenRunAsync(() -> {
            addMonitors(pullingMonitor.monitor(onEnvironmentInitComplete(onCompleteInternal, Selector.ROUTE),
                    onErrorInternal, this::isRouteRunning));
            logger.info("Route [{}].", route.getName());
        }));

        // monitor creation errors after all other monitors to make sure we cancel all of them on failure
        addMonitors(pullingMonitor.monitor(() -> {
        }, onErrorInternal, this::checkOpenshiftObjectCreation));
    }

    private void addMonitors(RunningTask task) {
        runningTaskMonitors.add(task);
    }

    private void cancelAndClearMonitors() {
        runningTaskMonitors.stream().forEach(pullingMonitor::cancelRunningTask);
        runningTaskMonitors.clear();
    }

    private boolean isServletAvailable(URL servletUrl) {
        try {
            return connectToPingUrl(servletUrl);
        } catch (IOException e) {
            return false;
        }
    }

    private Runnable onEnvironmentInitComplete(Consumer<RunningEnvironment> onComplete, Selector selector) {
        return () -> {
            synchronized (this) {
                initialized.add(selector);
                if (createRoute) {
                    if (!initialized.containsAll(Arrays.asList(Selector.POD, Selector.SERVICE, Selector.ROUTE))) {
                        return;
                    }
                } else {
                    if (!initialized.containsAll(Arrays.asList(Selector.POD, Selector.SERVICE))) {
                        return;
                    }
                }
            }

            logger.info("Environment successfully initialized. Pod [{}]; Service [{}].", pod.getName(),
                    service.getName());
            if (createRoute) {
                logger.info("Route initialized [{}].", route.getName());
            }

            RunningEnvironment runningEnvironment = RunningEnvironment.createInstance(pod.getName(),
                    Integer.parseInt(environmentConfiguration.getContainerPort()), route.getHost(),
                    getPublicEndpointUrl(), getInternalEndpointUrl(), repositorySession,
                    Paths.get(environmentConfiguration.getWorkingDirectory()), this::destroyEnvironment, debugData);

            gaugeMetric.ifPresent(g -> g.incrementMetric(METRICS_POD_STARTED_SUCCESS_KEY));
            onComplete.accept(runningEnvironment);
        };
    }

    private String getPublicEndpointUrl() {
        if (createRoute) {
            return "http://" + route.getHost() + "" + route.getPath() + "/"
                    + environmentConfiguration.getBuildAgentBindPath();
        } else {
            return getInternalEndpointUrl();
        }
    }

    private String getInternalEndpointUrl() {
        return "http://" + service.getClusterIP() + "/" + buildAgentContextPath + "/"
                + environmentConfiguration.getBuildAgentBindPath();
    }

    /**
     * check if the Openshift object futures are completed successfully.
     *
     * If completed successfully, return true
     * If completed, but not successfully, the exception in the future is thrown.
     * If not completed, return false
     *
     * @return boolean
     */
    private boolean checkOpenshiftObjectCreation() {

        if (!creationCompletableFutures.isDone()) {
            logger.debug("All openshift creating completable futures not done yet!");
            return false;
        } else {

            // creation futures are done. Check if there was an exception or not
            if (creationCompletableFutures.isCompletedExceptionally()) {
                // capturing the exception
                try {
                    creationCompletableFutures.join();
                } catch (Exception e) {
                    logger.debug("Exception in one of the openshift creating completable future", e);
                    throw new PodFailedStartException(e.getMessage());
                }
                return false;
            } else {
                return true;
            }
        }
    }

    /**
     * Check if pod is in running state.
     * If pod is in one of the failure statuses (as specified in POD_FAILED_STATUSES, PodFailedStartException is thrown
     *
     * @return boolean: is pod running?
     */
    private boolean isPodRunning() {

        pod = client.get(pod.getKind(), pod.getName(), environmentConfiguration.getPncNamespace());

        String podStatus = pod.getStatus();
        logger.debug("Pod {} status: {}", pod.getName(), podStatus);

        if (Arrays.asList(POD_FAILED_STATUSES).contains(podStatus)) {
            gaugeMetric.ifPresent(g -> g.incrementMetric(METRICS_POD_STARTED_FAILED_REASON_KEY + "." + podStatus));
            throw new PodFailedStartException("Pod failed with status: " + podStatus);
        }

        boolean isRunning = "Running".equals(pod.getStatus());
        if (isRunning) {
            logger.debug("Pod {} running.", pod.getName());
            return true;
        }
        return false;
    }

    private boolean isServiceRunning() {

        service = client.get(service.getKind(), service.getName(), environmentConfiguration.getPncNamespace());
        boolean isRunning = service.getPods().size() > 0;
        if (isRunning) {
            logger.debug("Service {} running.", service.getName());
            return true;
        }
        return false;
    }

    private boolean isRouteRunning() {

        try {
            if (connectToPingUrl(new URL(getPublicEndpointUrl()))) {
                route = client.get(route.getKind(), route.getName(), environmentConfiguration.getPncNamespace());
                logger.debug("Route {} running.", route.getName());
                return true;
            } else {
                return false;
            }
        } catch (IOException e) {
            logger.error("Cannot open URL " + getPublicEndpointUrl(), e);
            return false;
        }
    }

    @Override
    public String getId() {
        return pod.getName();
    }

    @Override
    public void cancel() {
        cancelRequested = true;

        creatingPod.ifPresent(f -> f.cancel(false));
        creatingService.ifPresent(f -> f.cancel(false));
        creatingRoute.ifPresent(f -> f.cancel(false));

        if (cancelHook != null) {
            cancelHook.run();
        } else {
            logger.warn("Trying to cancel operation while no cancel hook is defined.");
        }
        destroyEnvironment();
    }

    @Override
    public void destroyEnvironment() {
        destroyEnvironment(route, service, sshService, pod, false);
    }

    private void destroyEnvironment(Route routeLocal, Service serviceLocal, Service sshServiceLocal, Pod podLocal,
            boolean force) {

        if (!debugData.isDebugEnabled() || force) {
            if (!environmentConfiguration.getKeepBuildAgentInstance()) {
                if (createRoute) {
                    tryOpenshiftDeleteResource(routeLocal);
                }
                tryOpenshiftDeleteResource(serviceLocal);
                if (sshService != null) {
                    tryOpenshiftDeleteResource(sshServiceLocal);
                }
                tryOpenshiftDeleteResource(podLocal);
            }
        }
    }

    /**
     * Try to delete an openshift resource. If it doesn't exist, it's fine
     *
     * @param resource Openshift resource to delete
     * @param <T>
     */
    private <T extends IResource> void tryOpenshiftDeleteResource(T resource) {

        try {
            client.delete(resource);
        } catch (NotFoundException e) {
            logger.warn("Couldn't delete the Openshift resource since it does not exist", e);
        }
    }

    private String replaceConfigurationVariables(String podConfiguration, Map runtimeProperties) {
        Boolean proxyActive = !StringUtils.isEmpty(environmentConfiguration.getProxyServer())
                && !StringUtils.isEmpty(environmentConfiguration.getProxyPort());

        Properties properties = new Properties();
        properties.put("image", imageId);
        properties.put("containerPort", environmentConfiguration.getContainerPort());
        properties.put("firewallAllowedDestinations", environmentConfiguration.getFirewallAllowedDestinations());
        // This property sent as Json
        properties.put("allowedHttpOutgoingDestinations",
                toEscapedJsonString(environmentConfiguration.getAllowedHttpOutgoingDestinations()));
        properties.put("isHttpActive", proxyActive.toString().toLowerCase());
        properties.put("proxyServer", environmentConfiguration.getProxyServer());
        properties.put("proxyPort", environmentConfiguration.getProxyPort());
        properties.put("nonProxyHosts", environmentConfiguration.getNonProxyHosts());

        properties.put("AProxDependencyUrl", repositorySession.getConnectionInfo().getDependencyUrl());
        properties.put("AProxDeployUrl", repositorySession.getConnectionInfo().getDeployUrl());

        properties.putAll(runtimeProperties);

        return StringPropertyReplacer.replaceProperties(podConfiguration, properties);
    }

    /**
     * Enable ssh forwarding
     *
     * @return port, to which ssh is forwarded
     */
    private Integer startSshService() {
        ModelNode serviceConfigurationNode = createModelNode(
                Configurations.getContentAsString(Resource.PNC_BUILDER_SSH_SERVICE, openshiftBuildAgentConfig),
                runtimeProperties);
        sshService = new Service(serviceConfigurationNode, client,
                ResourcePropertiesRegistry.getInstance().get(OSE_API_VERSION, ResourceKind.SERVICE));
        sshService.setNamespace(environmentConfiguration.getPncNamespace());
        try {
            Service resultService = client.create(this.sshService, sshService.getNamespace());
            return resultService.getNode().get("spec").get("ports").asList().stream()
                    .filter(m -> m.get("name").asString().equals(SSH_SERVICE_PORT_NAME)).findAny()
                    .orElseThrow(() -> new RuntimeException(
                            "No ssh service in response! Service data: " + describeService(resultService)))
                    .get("nodePort").asInt();
        } catch (Throwable e) {
            logger.error("Cannot create service.", e);
            return null;
        }
    }

    private String describeService(Service resultService) {
        if (resultService == null)
            return null;

        ModelNode node = resultService.getNode();
        return "Service[" + "name = " + resultService.getName() + ", node= '"
                + (node == null ? null : node.toJSONString(false)) + "]";
    }

    private enum Selector {
        POD, SERVICE, ROUTE
    }

    private boolean connectToPingUrl(URL url) throws IOException {
        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
        connection.setConnectTimeout(500);
        connection.setRequestMethod("GET");
        connection.setDoOutput(true);
        connection.setDoInput(true);
        connection.connect();

        int responseCode = connection.getResponseCode();
        connection.disconnect();

        logger.debug("Got {} from {}.", responseCode, url);
        return responseCode == 200;
    }

    /**
     * Return an escaped string of the JSON representation of the object
     *
     * By 'escaped', it means that strings like '"' are escaped to '\"'
     * @param object object to marshall
     * @return Escaped Json String
     */
    private String toEscapedJsonString(Object object) {
        ObjectMapper mapper = new ObjectMapper();
        JsonStringEncoder jsonStringEncoder = JsonStringEncoder.getInstance();
        try {
            return new String(jsonStringEncoder.quoteAsString(mapper.writeValueAsString(object)));
        } catch (JsonProcessingException e) {
            logger.error("Could not parse object: " + object, e);
            throw new RuntimeException(e);
        }
    }
}