Java tutorial
/* * Copyright 2017-2019 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.glowroot.central; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.concurrent.GuardedBy; import com.google.common.base.MoreObjects; import com.google.common.base.Ticker; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.machinepublishers.jbrowserdriver.JBrowserDriver; import com.machinepublishers.jbrowserdriver.ProxyConfig; import com.machinepublishers.jbrowserdriver.RequestHeaders; import com.machinepublishers.jbrowserdriver.Settings; import com.machinepublishers.jbrowserdriver.UserAgent; import com.machinepublishers.jbrowserdriver.UserAgent.Family; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.auth.AUTH; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.concurrent.FutureCallback; import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.TrustSelfSignedStrategy; import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClients; import org.apache.http.message.BasicHeader; import org.apache.http.ssl.SSLContextBuilder; import org.checkerframework.checker.nullness.qual.Nullable; import org.immutables.value.Value; import org.openqa.selenium.WebDriver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.glowroot.agent.api.Instrumentation; import org.glowroot.agent.api.Instrumentation.AlreadyInTransactionBehavior; import org.glowroot.central.RollupService.AgentRollupConsumer; import org.glowroot.central.repo.ActiveAgentDao; import org.glowroot.central.repo.AlertingDisabledDao; import org.glowroot.central.repo.ConfigRepositoryImpl; import org.glowroot.central.repo.IncidentDao; import org.glowroot.central.repo.SyntheticResultDao; import org.glowroot.central.repo.SyntheticResultDao.SyntheticResultRollup0; import org.glowroot.central.util.ClusterManager; import org.glowroot.central.util.MoreExecutors2; import org.glowroot.common.util.Clock; import org.glowroot.common.util.Styles; import org.glowroot.common.util.Throwables; import org.glowroot.common.util.Version; import org.glowroot.common2.config.HttpProxyConfig; import org.glowroot.common2.config.MoreConfigDefaults; import org.glowroot.common2.repo.ActiveAgentRepository.AgentRollup; import org.glowroot.common2.repo.ConfigRepository.AgentConfigNotFoundException; import org.glowroot.common2.repo.IncidentRepository.OpenIncident; import org.glowroot.common2.repo.util.AlertingService; import org.glowroot.common2.repo.util.Compilations; import org.glowroot.common2.repo.util.Encryption; import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.AlertConfig; import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.AlertConfig.AlertCondition; import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.AlertConfig.AlertCondition.SyntheticMonitorCondition; import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.SyntheticMonitorConfig; import org.glowroot.wire.api.model.AgentConfigOuterClass.AgentConfig.SyntheticMonitorConfig.SyntheticMonitorKind; import static com.google.common.base.Preconditions.checkNotNull; import static java.util.concurrent.TimeUnit.DAYS; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; class SyntheticMonitorService implements Runnable { private static final Logger logger = LoggerFactory.getLogger(SyntheticMonitorService.class); private static final Pattern encryptedPattern = Pattern.compile("\"ENCRYPTED:([^\"]*)\""); private static final int PING_TIMEOUT_MILLIS = 60000; private static final long PING_TIMEOUT_NANOS = MILLISECONDS.toNanos(PING_TIMEOUT_MILLIS); private static final RequestHeaders REQUEST_HEADERS; static { // this list is from com.machinepublishers.jbrowserdriver.RequestHeaders.CHROME, // with added Glowroot-Transaction-Type header LinkedHashMap<String, String> headersTmp = new LinkedHashMap<>(); headersTmp.put("Host", RequestHeaders.DYNAMIC_HEADER); headersTmp.put("Connection", "keep-alive"); headersTmp.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); headersTmp.put("Upgrade-Insecure-Requests", "1"); headersTmp.put("User-Agent", RequestHeaders.DYNAMIC_HEADER); headersTmp.put("Referer", RequestHeaders.DYNAMIC_HEADER); headersTmp.put("Accept-Encoding", "gzip, deflate, sdch"); headersTmp.put("Accept-Language", "en-US,en;q=0.8"); headersTmp.put("Cookie", RequestHeaders.DYNAMIC_HEADER); headersTmp.put("Glowroot-Transaction-Type", "Synthetic"); REQUEST_HEADERS = new RequestHeaders(headersTmp); } private final ActiveAgentDao activeAgentDao; private final ConfigRepositoryImpl configRepository; private final AlertingDisabledDao alertingDisabledDao; private final IncidentDao incidentDao; private final AlertingService alertingService; private final SyntheticResultDao syntheticResponseDao; private final Ticker ticker; private final Clock clock; private final ConcurrentMap<String, Boolean> executionRateLimiter; private final CloseableHttpAsyncClient asyncHttpClient; @GuardedBy("syncHttpClientHolder") private volatile SyncHttpClientHolder syncHttpClientHolder; private final Object syncHttpClientHolderLock = new Object(); private final String shortVersion; private final UserAgent userAgent; private final ExecutorService mainLoopExecutor; private final ExecutorService workerExecutor; private final ListeningExecutorService subWorkerExecutor; private final Set<SyntheticMonitorUniqueKey> activeSyntheticMonitors = Sets.newConcurrentHashSet(); private volatile boolean closed; SyntheticMonitorService(ActiveAgentDao activeAgentDao, ConfigRepositoryImpl configRepository, AlertingDisabledDao alertingDisabledDao, IncidentDao incidentDao, AlertingService alertingService, SyntheticResultDao syntheticResponseDao, ClusterManager clusterManager, Ticker ticker, Clock clock, String version) throws Exception { this.activeAgentDao = activeAgentDao; this.configRepository = configRepository; this.alertingDisabledDao = alertingDisabledDao; this.incidentDao = incidentDao; this.alertingService = alertingService; this.syntheticResponseDao = syntheticResponseDao; this.ticker = ticker; this.clock = clock; executionRateLimiter = clusterManager.createReplicatedMap("syntheticMonitorExecutionRateLimiter", 30, SECONDS); if (version.equals(Version.UNKNOWN_VERSION)) { shortVersion = ""; } else { int index = version.indexOf(", built "); if (index == -1) { shortVersion = "/" + version; } else { shortVersion = "/" + version.substring(0, index); } } // asyncHttpClient is only used for pings, so safe to ignore cert errors asyncHttpClient = HttpAsyncClients.custom().setUserAgent("GlowrootCentral" + shortVersion) .setDefaultHeaders(Arrays.asList(new BasicHeader("Glowroot-Transaction-Type", "Synthetic"))) .setMaxConnPerRoute(10) // increasing from default 2 .setMaxConnTotal(1000) // increasing from default 20 .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(new TrustSelfSignedStrategy()).build()) .setSSLHostnameVerifier(new NoopHostnameVerifier()).build(); asyncHttpClient.start(); syncHttpClientHolder = createSyncHttpClientHolder(shortVersion, configRepository.getHttpProxyConfig()); // these parameters are from com.machinepublishers.jbrowserdriver.UserAgent.CHROME // with added GlowrootCentral/<version> for identification purposes userAgent = new UserAgent(Family.WEBKIT, "Google Inc.", "Win32", "Windows NT 6.1", "5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)" + " Chrome/45.0.2454.85 Safari/537.36 GlowrootCentral" + shortVersion, "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)" + " Chrome/45.0.2454.85 Safari/537.36 GlowrootCentral" + shortVersion); // there is one subworker per worker, so using same max subWorkerExecutor = MoreExecutors .listeningDecorator(MoreExecutors2.newCachedThreadPool("Synthetic-Monitor-Sub-Worker-%d")); workerExecutor = MoreExecutors2.newCachedThreadPool("Synthetic-Monitor-Worker-%d"); mainLoopExecutor = MoreExecutors2.newSingleThreadExecutor("Synthetic-Monitor-Main-Loop"); mainLoopExecutor.execute(castInitialized(this)); } @Override public void run() { while (!closed) { try { long currMillis = clock.currentTimeMillis(); long nextMillis = (long) Math.ceil(currMillis / 60000.0) * 60000; // scheduling for 5 seconds after the minute (just to avoid exactly on the minute) MILLISECONDS.sleep(nextMillis - currMillis + 5000); runInternal(); } catch (InterruptedException e) { // probably shutdown requested (see close method below) logger.debug(e.getMessage(), e); continue; } catch (Throwable t) { logger.error(t.getMessage(), t); } } } void close() throws Exception { closed = true; // shutdownNow() is needed to send interrupt to SyntheticMonitorService user test threads subWorkerExecutor.shutdownNow(); if (!subWorkerExecutor.awaitTermination(10, SECONDS)) { throw new IllegalStateException("Timed out waiting for synthetic user test threads to terminate"); } // shutdownNow() is needed to send interrupt to SyntheticMonitorService check threads workerExecutor.shutdownNow(); if (!workerExecutor.awaitTermination(10, SECONDS)) { throw new IllegalStateException("Timed out waiting for synthetic monitor check threads to terminate"); } // shutdownNow() is needed to send interrupt to SyntheticMonitorService main thread mainLoopExecutor.shutdownNow(); if (!mainLoopExecutor.awaitTermination(10, SECONDS)) { throw new IllegalStateException("Timed out waiting for synthetic monitor loop thread to terminate"); } syncHttpClientHolder.syncHttpClient().close(); asyncHttpClient.close(); } @Instrumentation.Transaction(transactionType = "Background", transactionName = "Outer synthetic monitor loop", traceHeadline = "Outer synthetic monitor loop", timer = "outer synthetic monitor loop") private void runInternal() throws Exception { for (AgentRollup agentRollup : activeAgentDao.readRecentlyActiveAgentRollups(DAYS.toMillis(7))) { consumeAgentRollups(agentRollup, this::runSyntheticMonitors); } } private void runSyntheticMonitors(AgentRollup agentRollup) throws InterruptedException { List<SyntheticMonitorConfig> syntheticMonitorConfigs; try { syntheticMonitorConfigs = configRepository.getSyntheticMonitorConfigs(agentRollup.id()); } catch (InterruptedException e) { // probably shutdown requested throw e; } catch (AgentConfigNotFoundException e) { // be lenient if agent_config table is messed up logger.debug(e.getMessage(), e); return; } catch (Exception e) { logger.error("{} - {}", agentRollup.id(), e.getMessage(), e); return; } if (syntheticMonitorConfigs.isEmpty()) { return; } for (SyntheticMonitorConfig syntheticMonitorConfig : syntheticMonitorConfigs) { List<AlertConfig> alertConfigs; try { alertConfigs = configRepository.getAlertConfigsForSyntheticMonitorId(agentRollup.id(), syntheticMonitorConfig.getId()); } catch (InterruptedException e) { // probably shutdown requested throw e; } catch (AgentConfigNotFoundException e) { // be lenient if agent_config table is messed up logger.debug(e.getMessage(), e); return; } catch (Exception e) { logger.error(e.getMessage(), e); continue; } String uniqueId = syntheticMonitorConfig.getId() + agentRollup.id(); if (executionRateLimiter.putIfAbsent(uniqueId, true) != null) { // was run in the last 30 seconds (probably on a different cluster node) continue; } workerExecutor.execute(() -> { try { switch (syntheticMonitorConfig.getKind()) { case PING: runPing(agentRollup, syntheticMonitorConfig, alertConfigs); break; case JAVA: runJava(agentRollup, syntheticMonitorConfig, alertConfigs); break; default: throw new IllegalStateException( "Unexpected synthetic kind: " + syntheticMonitorConfig.getKind()); } } catch (InterruptedException e) { // probably shutdown requested (see close method above) logger.debug(e.getMessage(), e); } catch (Throwable t) { logger.error("{} - {}", agentRollup.id(), t.getMessage(), t); } }); } } @Instrumentation.Transaction(transactionType = "Background", transactionName = "Synthetic monitor", traceHeadline = "Synthetic monitor: {{0.id}}", timer = "synthetic monitor", alreadyInTransactionBehavior = AlreadyInTransactionBehavior.CAPTURE_NEW_TRANSACTION) private void runPing(AgentRollup agentRollup, SyntheticMonitorConfig syntheticMonitorConfig, List<AlertConfig> alertConfigs) throws Exception { runSyntheticMonitor(agentRollup, syntheticMonitorConfig, alertConfigs, () -> runPing(syntheticMonitorConfig.getPingUrl())); } @Instrumentation.Transaction(transactionType = "Background", transactionName = "Synthetic monitor", traceHeadline = "Synthetic monitor: {{0.id}}", timer = "synthetic monitor") private void runJava(AgentRollup agentRollup, SyntheticMonitorConfig syntheticMonitorConfig, List<AlertConfig> alertConfigs) throws Exception { Matcher matcher = encryptedPattern.matcher(syntheticMonitorConfig.getJavaSource()); StringBuffer sb = new StringBuffer(); while (matcher.find()) { String encryptedPassword = checkNotNull(matcher.group(1)); matcher.appendReplacement(sb, "\"" + Encryption.decrypt(encryptedPassword, configRepository.getLazySecretKey()) + "\""); } matcher.appendTail(sb); runSyntheticMonitor(agentRollup, syntheticMonitorConfig, alertConfigs, () -> runJava(sb.toString())); } private FutureWithStartTick runJava(String javaSource) throws Exception { Class<?> syntheticUserTestClass = Compilations.compile(javaSource); // validation for default constructor and test method occurs on save Constructor<?> defaultConstructor = syntheticUserTestClass.getConstructor(); Method testMethod; try { testMethod = syntheticUserTestClass.getMethod("test", CloseableHttpClient.class); } catch (NoSuchMethodException e) { testMethod = syntheticUserTestClass.getMethod("test", WebDriver.class); } if (testMethod.getParameterTypes()[0] == CloseableHttpClient.class) { synchronized (syncHttpClientHolderLock) { HttpProxyConfig httpProxyConfig = configRepository.getHttpProxyConfig(); if (!syncHttpClientHolder.httpProxyConfig().equals(httpProxyConfig)) { syncHttpClientHolder = createSyncHttpClientHolder(shortVersion, httpProxyConfig); // don't close the old http client since other threads may still be using it // (it will be garbage collected when no longer in use) } } return runJava(defaultConstructor, testMethod, syncHttpClientHolder.syncHttpClient()); } else { return runWebDriverJava(defaultConstructor, testMethod); } } private FutureWithStartTick runWebDriverJava(Constructor<?> defaultConstructor, Method testMethod) throws Exception { Settings.Builder settings = Settings.builder().requestHeaders(REQUEST_HEADERS).userAgent(userAgent); HttpProxyConfig httpProxyConfig = configRepository.getHttpProxyConfig(); if (!httpProxyConfig.host().isEmpty()) { int proxyPort = MoreObjects.firstNonNull(httpProxyConfig.port(), 80); settings.proxy(new ProxyConfig(ProxyConfig.Type.HTTP, httpProxyConfig.host(), proxyPort, httpProxyConfig.username(), httpProxyConfig.encryptedPassword())); } JBrowserDriver driver = new JBrowserDriver(settings.build()); return runJava(defaultConstructor, testMethod, driver); } private FutureWithStartTick runJava(Constructor<?> defaultConstructor, Method testMethod, Object testArg) { long startTick = ticker.read(); FutureWithStartTick future = new FutureWithStartTick(startTick); subWorkerExecutor.execute(() -> { try { future.complete(runJava(defaultConstructor, testMethod, testArg, startTick)); } catch (Throwable t) { // unexpected exception future.completeExceptionally(t); } }); return future; } public SyntheticRunResult runJava(Constructor<?> defaultConstructor, Method testMethod, Object testArg, long startTick) throws Exception { try { return runJavaInternal(defaultConstructor, testMethod, testArg, startTick); } catch (InterruptedException e) { // probably shutdown requested (see close method above) logger.debug(e.getMessage(), e); throw e; } catch (Throwable t) { // unexpected exception logger.error(t.getMessage(), t); throw t; } } private SyntheticRunResult runJavaInternal(Constructor<?> defaultConstructor, Method testMethod, Object testArg, long startTick) throws Exception { long captureTime; long durationNanos; try { testMethod.invoke(defaultConstructor.newInstance(), testArg); // capture time and duration before calling driver.quit() captureTime = clock.currentTimeMillis(); durationNanos = ticker.read() - startTick; } catch (InvocationTargetException e) { logger.debug(e.getMessage(), e); Throwable throwable = e.getTargetException(); if (throwable instanceof InterruptedException) { // probably shutdown requested (see close method above) throw (InterruptedException) throwable; } return ImmutableSyntheticRunResult.builder().captureTime(clock.currentTimeMillis()) .durationNanos(ticker.read() - startTick).throwable(throwable).build(); } finally { if (testArg instanceof JBrowserDriver) { // quit() can hang (easily reproducible on test that doesn't use testArg at all) ((JBrowserDriver) testArg).kill(); } } return ImmutableSyntheticRunResult.builder().captureTime(captureTime).durationNanos(durationNanos).build(); } private FutureWithStartTick runPing(String url) throws Exception { HttpGet httpGet = new HttpGet(url); RequestConfig.Builder config = RequestConfig.custom() // wait an extra second to make sure no edge case where // SocketTimeoutException occurs with elapsed time < PING_TIMEOUT_MILLIS .setSocketTimeout(PING_TIMEOUT_MILLIS + 1000); HttpProxyConfig httpProxyConfig = configRepository.getHttpProxyConfig(); if (!httpProxyConfig.host().isEmpty()) { int proxyPort = MoreObjects.firstNonNull(httpProxyConfig.port(), 80); config.setProxy(new HttpHost(httpProxyConfig.host(), proxyPort)); } httpGet.setConfig(config.build()); long startTick = ticker.read(); FutureWithStartTick future = new FutureWithStartTick(startTick); subWorkerExecutor.execute(() -> { try { asyncHttpClient.execute(httpGet, getHttpClientContext(), new CompletingFutureCallback(future)); } catch (Throwable t) { logger.debug(t.getMessage(), t); future.complete(ImmutableSyntheticRunResult.builder().captureTime(clock.currentTimeMillis()) .durationNanos(ticker.read() - startTick).throwable(t).build()); } }); return future; } private void runSyntheticMonitor(AgentRollup agentRollup, SyntheticMonitorConfig syntheticMonitorConfig, List<AlertConfig> alertConfigs, Callable<FutureWithStartTick> callable) throws Exception { SyntheticMonitorUniqueKey uniqueKey = ImmutableSyntheticMonitorUniqueKey.of(agentRollup.id(), syntheticMonitorConfig.getId()); if (!activeSyntheticMonitors.add(uniqueKey)) { return; } FutureWithStartTick future = callable.call(); // important that uniqueKey is always removed on completion even on unexpected errors future.whenComplete((v, t) -> activeSyntheticMonitors.remove(uniqueKey)); OnRunComplete onRunComplete = new OnRunComplete(agentRollup, syntheticMonitorConfig); if (alertConfigs.isEmpty()) { future.thenAccept(onRunComplete); return; } int maxAlertThresholdMillis = 0; for (AlertConfig alertConfig : alertConfigs) { maxAlertThresholdMillis = Math.max(maxAlertThresholdMillis, alertConfig.getCondition().getSyntheticMonitorCondition().getThresholdMillis()); } long captureTime; long durationNanos; boolean success; String errorMessage; try { // wait an extra second to make sure no edge case where TimeoutException occurs with // stopwatch.elapsed(MILLISECONDS) < maxAlertThresholdMillis SyntheticRunResult result = future.get(maxAlertThresholdMillis + 1000L, MILLISECONDS); captureTime = result.captureTime(); durationNanos = result.durationNanos(); Throwable throwable = result.throwable(); if (throwable == null) { success = true; errorMessage = null; } else { success = false; errorMessage = getBestMessageForSyntheticFailure(throwable); } } catch (TimeoutException e) { logger.debug(e.getMessage(), e); captureTime = clock.currentTimeMillis(); durationNanos = 0; // durationNanos is only used below when success is true success = false; errorMessage = null; } if (!isCurrentlyDisabled(agentRollup.id())) { if (success) { for (AlertConfig alertConfig : alertConfigs) { AlertCondition alertCondition = alertConfig.getCondition(); SyntheticMonitorCondition condition = alertCondition.getSyntheticMonitorCondition(); boolean currentlyTriggered = durationNanos >= MILLISECONDS .toNanos(condition.getThresholdMillis()); sendAlertIfStatusChanged(agentRollup, syntheticMonitorConfig, alertConfig, condition, captureTime, currentlyTriggered, null); } } else { sendAlertOnErrorIfStatusChanged(agentRollup, syntheticMonitorConfig, alertConfigs, errorMessage, captureTime); } } // need to run at end to ensure new synthetic response doesn't get stored before consecutive // count is checked in sendAlertOnErrorIfStatusChanged() future.thenAccept(onRunComplete); } private boolean isCurrentlyDisabled(String agentRollupId) throws Exception { Long disabledUntilTime = alertingDisabledDao.getAlertingDisabledUntilTime(agentRollupId); return disabledUntilTime != null && disabledUntilTime > clock.currentTimeMillis(); } private static String getBestMessageForSyntheticFailure(Throwable throwable) { String message = throwable.getMessage(); if (throwable.getClass() == Exception.class && throwable.getCause() == null && message != null) { // special case so synthetic monitors can display a simple error message without the // exception class name clutter return message; } else { return Throwables.getBestMessage(throwable); } } private class OnRunComplete implements Consumer<SyntheticRunResult> { private final AgentRollup agentRollup; private final SyntheticMonitorConfig syntheticMonitorConfig; private OnRunComplete(AgentRollup agentRollup, SyntheticMonitorConfig syntheticMonitorConfig) { this.agentRollup = agentRollup; this.syntheticMonitorConfig = syntheticMonitorConfig; } @Override public void accept(SyntheticRunResult syntheticRunResult) { String errorMessage = null; long durationNanos = syntheticRunResult.durationNanos(); Throwable t = syntheticRunResult.throwable(); if (syntheticMonitorConfig.getKind() == SyntheticMonitorKind.PING && durationNanos >= PING_TIMEOUT_NANOS) { durationNanos = PING_TIMEOUT_NANOS; errorMessage = "Timeout"; } else if (t != null) { errorMessage = getBestMessageForSyntheticFailure(t); } try { syntheticResponseDao.store(agentRollup.id(), syntheticMonitorConfig.getId(), MoreConfigDefaults.getDisplayOrDefault(syntheticMonitorConfig), syntheticRunResult.captureTime(), durationNanos, errorMessage); } catch (InterruptedException e) { // probably shutdown requested (see close method above) logger.debug(e.getMessage(), e); return; } catch (Exception e) { logger.error(e.getMessage(), e); } } } private void sendAlertOnErrorIfStatusChanged(AgentRollup agentRollup, SyntheticMonitorConfig syntheticMonitorConfig, List<AlertConfig> alertConfigs, @Nullable String errorMessage, long captureTime) throws Exception { for (AlertConfig alertConfig : alertConfigs) { AlertCondition alertCondition = alertConfig.getCondition(); SyntheticMonitorCondition condition = alertCondition.getSyntheticMonitorCondition(); sendAlertIfStatusChanged(agentRollup, syntheticMonitorConfig, alertConfig, condition, captureTime, true, errorMessage); } } private void sendAlertIfStatusChanged(AgentRollup agentRollup, SyntheticMonitorConfig syntheticMonitorConfig, AlertConfig alertConfig, SyntheticMonitorCondition condition, long endTime, boolean currentlyTriggered, @Nullable String errorMessage) throws Exception { AlertCondition alertCondition = alertConfig.getCondition(); OpenIncident openIncident = incidentDao.readOpenIncident(agentRollup.id(), alertCondition, alertConfig.getSeverity()); if (openIncident != null && !currentlyTriggered) { incidentDao.resolveIncident(openIncident, endTime); sendAlert(agentRollup.id(), agentRollup.display(), syntheticMonitorConfig, alertConfig, condition, endTime, true, null); } else if (openIncident == null && currentlyTriggered && consecutiveCountHit(agentRollup.id(), syntheticMonitorConfig, condition)) { // the start time for the incident is the end time of the interval evaluated above incidentDao.insertOpenIncident(agentRollup.id(), alertCondition, alertConfig.getSeverity(), alertConfig.getNotification(), endTime); sendAlert(agentRollup.id(), agentRollup.display(), syntheticMonitorConfig, alertConfig, condition, endTime, false, errorMessage); } } private boolean consecutiveCountHit(String agentRollupId, SyntheticMonitorConfig syntheticMonitorConfig, SyntheticMonitorCondition condition) throws Exception { int consecutiveCount = condition.getConsecutiveCount(); if (consecutiveCount == 1) { return true; } List<SyntheticResultRollup0> syntheticResults = syntheticResponseDao.readLastFromRollup0(agentRollupId, syntheticMonitorConfig.getId(), consecutiveCount - 1); if (syntheticResults.size() < consecutiveCount - 1) { return false; } for (SyntheticResultRollup0 syntheticResult : syntheticResults) { if (!syntheticResult.error() && syntheticResult.totalDurationNanos() < MILLISECONDS .toNanos(condition.getThresholdMillis())) { return false; } } return true; } private void sendAlert(String agentRollupId, String agentRollupDisplay, SyntheticMonitorConfig syntheticMonitorConfig, AlertConfig alertConfig, SyntheticMonitorCondition condition, long endTime, boolean ok, @Nullable String errorMessage) throws Exception { // subject is the same between initial and ok messages so they will be threaded by gmail String subject = MoreConfigDefaults.getDisplayOrDefault(syntheticMonitorConfig); StringBuilder sb = new StringBuilder(); sb.append(MoreConfigDefaults.getDisplayOrDefault(syntheticMonitorConfig)); if (errorMessage == null) { sb.append(" time"); sb.append(AlertingService.getPreUpperBoundText(ok)); sb.append(AlertingService.getWithUnit(condition.getThresholdMillis(), "millisecond")); sb.append("."); } else { sb.append(" resulted in error: "); sb.append(errorMessage); } alertingService.sendNotification(configRepository.getCentralAdminGeneralConfig().centralDisplayName(), agentRollupId, agentRollupDisplay, alertConfig, endTime, subject, sb.toString(), ok); } private HttpClientContext getHttpClientContext() throws Exception { HttpProxyConfig httpProxyConfig = configRepository.getHttpProxyConfig(); if (httpProxyConfig.host().isEmpty() || httpProxyConfig.username().isEmpty()) { return HttpClientContext.create(); } // perform preemptive proxy authentication int proxyPort = MoreObjects.firstNonNull(httpProxyConfig.port(), 80); HttpHost proxyHost = new HttpHost(httpProxyConfig.host(), proxyPort); BasicScheme basicScheme = new BasicScheme(); basicScheme.processChallenge(new BasicHeader(AUTH.PROXY_AUTH, "BASIC realm=")); BasicAuthCache authCache = new BasicAuthCache(); authCache.put(proxyHost, basicScheme); String password = httpProxyConfig.encryptedPassword(); if (!password.isEmpty()) { password = Encryption.decrypt(password, configRepository.getLazySecretKey()); } CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(proxyHost), new UsernamePasswordCredentials(httpProxyConfig.username(), password)); HttpClientContext context = HttpClientContext.create(); context.setAuthCache(authCache); context.setCredentialsProvider(credentialsProvider); return context; } private static SyncHttpClientHolder createSyncHttpClientHolder(String shortVersion, HttpProxyConfig httpProxyConfig) { HttpClientBuilder builder = HttpClients.custom().setUserAgent("GlowrootCentral" + shortVersion) .setDefaultHeaders(Arrays.asList(new BasicHeader("Glowroot-Transaction-Type", "Synthetic"))); if (!httpProxyConfig.host().isEmpty()) { int proxyPort = MoreObjects.firstNonNull(httpProxyConfig.port(), 80); builder.setProxy(new HttpHost(httpProxyConfig.host(), proxyPort)); } CloseableHttpClient httpClient = builder.setMaxConnPerRoute(10) // increasing from default 2 .setMaxConnTotal(1000) // increasing from default 20 .build(); return ImmutableSyncHttpClientHolder.of(httpClient, httpProxyConfig); } private static void consumeAgentRollups(AgentRollup agentRollup, AgentRollupConsumer agentRollupConsumer) throws Exception { for (AgentRollup childAgentRollup : agentRollup.children()) { consumeAgentRollups(childAgentRollup, agentRollupConsumer); } agentRollupConsumer.accept(agentRollup); } @SuppressWarnings("return.type.incompatible") private static <T> /*@Initialized*/ T castInitialized(/*@UnderInitialization*/ T obj) { return obj; } @Value.Immutable @Styles.AllParameters interface SyncHttpClientHolder { CloseableHttpClient syncHttpClient(); HttpProxyConfig httpProxyConfig(); } @Value.Immutable @Styles.AllParameters interface SyntheticMonitorUniqueKey { String agentRollupId(); String syntheticMonitorId(); } private static class FutureWithStartTick extends CompletableFuture<SyntheticRunResult> { private final long startTick; private FutureWithStartTick(long startTick) { this.startTick = startTick; } } @Value.Immutable interface SyntheticRunResult { long captureTime(); long durationNanos(); @Nullable Throwable throwable(); } private class CompletingFutureCallback implements FutureCallback<HttpResponse> { private final FutureWithStartTick future; private CompletingFutureCallback(FutureWithStartTick future) { this.future = future; } @Override public void completed(HttpResponse response) { ImmutableSyntheticRunResult.Builder builder = ImmutableSyntheticRunResult.builder() .captureTime(clock.currentTimeMillis()).durationNanos(ticker.read() - future.startTick); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode < 400) { future.complete(builder.build()); } else { future.complete( builder.throwable(new Exception("Unexpected response status code: " + statusCode)).build()); } } @Override public void failed(Exception ex) { future.complete(ImmutableSyntheticRunResult.builder().captureTime(clock.currentTimeMillis()) .durationNanos(ticker.read() - future.startTick).throwable(ex).build()); } @Override public void cancelled() { future.complete(ImmutableSyntheticRunResult.builder().captureTime(clock.currentTimeMillis()) .durationNanos(ticker.read() - future.startTick) .throwable(new RuntimeException("Unexpected cancellation")).build()); } } }