diff --git a/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDao.java b/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDao.java index 66dcf68f81fd..5f9ef766ce54 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDao.java @@ -28,6 +28,12 @@ public interface RouterHealthCheckResultDao extends GenericDao getHealthCheckResults(long routerId); + /** + * @param routerIds + * @return Returns all the health checks in the database for the given router ids, in a single query + */ + List getHealthCheckResults(List routerIds); + boolean expungeHealthChecks(long routerId); /** diff --git a/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDaoImpl.java index b1292aebfd14..9b8edea41cd0 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDaoImpl.java @@ -17,6 +17,7 @@ package com.cloud.network.dao; +import java.util.Collections; import java.util.List; import org.springframework.stereotype.Component; @@ -30,9 +31,14 @@ public class RouterHealthCheckResultDaoImpl extends GenericDaoBase RouterChecksSearchBuilder; private SearchBuilder IsRouterFailingSearchBuilder; + private SearchBuilder RoutersChecksSearchBuilder; protected RouterHealthCheckResultDaoImpl() { super(); + RoutersChecksSearchBuilder = createSearchBuilder(); + RoutersChecksSearchBuilder.and("routerIds", RoutersChecksSearchBuilder.entity().getRouterId(), SearchCriteria.Op.IN); + RoutersChecksSearchBuilder.done(); + RouterChecksSearchBuilder = createSearchBuilder(); RouterChecksSearchBuilder.and("routerId", RouterChecksSearchBuilder.entity().getRouterId(), SearchCriteria.Op.EQ); RouterChecksSearchBuilder.and("checkName", RouterChecksSearchBuilder.entity().getCheckName(), SearchCriteria.Op.EQ); @@ -52,6 +58,16 @@ public List getHealthCheckResults(long routerId) { return listBy(sc); } + @Override + public List getHealthCheckResults(List routerIds) { + if (routerIds == null || routerIds.isEmpty()) { + return Collections.emptyList(); + } + SearchCriteria sc = RoutersChecksSearchBuilder.create(); + sc.setParameters("routerIds", routerIds.toArray()); + return listBy(sc); + } + @Override public boolean expungeHealthChecks(long routerId) { SearchCriteria sc = RouterChecksSearchBuilder.create(); diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index abb94d786793..2d6495465a65 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -59,7 +59,12 @@ import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostTagsDao; +import com.cloud.network.VirtualNetworkApplianceService; import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.RouterHealthCheckResultDao; +import com.cloud.network.dao.RouterHealthCheckResultVO; +import com.cloud.network.router.VirtualNetworkApplianceManager; +import com.cloud.network.router.VirtualRouter; import com.cloud.storage.ImageStore; import com.cloud.storage.StorageStats; import com.cloud.storage.Volume; @@ -70,7 +75,9 @@ import com.cloud.utils.Ternary; import com.cloud.utils.component.Manager; import com.cloud.utils.component.ManagerBase; +import com.cloud.vm.DomainRouterVO; import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; @@ -137,6 +144,10 @@ public String toString() { private HostTagsDao _hostTagsDao; @Inject private CAManager caManager; + @Inject + private DomainRouterDao domainRouterDao; + @Inject + private RouterHealthCheckResultDao routerHealthCheckResultDao; public PrometheusExporterImpl() { super(); @@ -501,10 +512,47 @@ private void addVMsBySizeMetrics(final List metricsList, final long dcId, } } + protected void addVirtualRouterHealthCheckMetrics(final List metricsList, final long dcId, final String zoneName, final String zoneUuid) { + List routers = domainRouterDao.listByDataCenter(dcId); + if (routers == null) { + return; + } + // only running virtual routers; results of stopped or destroyed routers are stale + Map runningRouters = routers.stream() + .filter(router -> router.getRole() == VirtualRouter.Role.VIRTUAL_ROUTER && router.getState() == State.Running) + .collect(Collectors.toMap(DomainRouterVO::getId, router -> router)); + if (runningRouters.isEmpty()) { + return; + } + // one query for all routers in the zone instead of one per router + List checks = routerHealthCheckResultDao.getHealthCheckResults(new ArrayList<>(runningRouters.keySet())); + if (checks == null) { + return; + } + for (RouterHealthCheckResultVO check : checks) { + DomainRouterVO router = runningRouters.get(check.getRouterId()); + if (router == null) { + continue; + } + int resultValue = (check.getCheckResult() == + VirtualNetworkApplianceService.RouterHealthStatus.SUCCESS) ? 1 : 0; + metricsList.add(new ItemVRHealthCheckResult( + zoneName, router.getInstanceName(), + check.getCheckName(), check.getCheckType(), resultValue)); + if (check.getLastUpdateTime() != null) { + long epochSeconds = check.getLastUpdateTime().getTime() / 1000L; + metricsList.add(new ItemVRHealthCheckLastCheck( + zoneName, router.getInstanceName(), + check.getCheckName(), check.getCheckType(), epochSeconds)); + } + } + } + @Override public void updateMetrics() { final List latestMetricsItems = new ArrayList(); try { + final boolean routerHealthChecksEnabled = Boolean.TRUE.equals(VirtualNetworkApplianceManager.RouterHealthChecksEnabled.value()); for (final DataCenterVO dc : dcDao.listAll()) { final String zoneName = dc.getName(); final String zoneUuid = dc.getUuid(); @@ -518,6 +566,9 @@ public void updateMetrics() { addDomainMetrics(latestMetricsItems, zoneName, zoneUuid); addAccountMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); addVMsBySizeMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); + if (routerHealthChecksEnabled) { + addVirtualRouterHealthCheckMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid); + } } addDomainLimits(latestMetricsItems); addDomainResourceCount(latestMetricsItems); @@ -1180,4 +1231,56 @@ public String toMetricsString() { return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\"} %d", name, zoneName, hostName, hostIp, expiryTimestamp); } } + + class ItemVRHealthCheckResult extends Item { + String zoneName; + String routerName; + String checkName; + String checkType; + int result; + + public ItemVRHealthCheckResult(String zn, String rn, String cn, String ct, int res) { + super("cloudstack_virtualrouter_healthcheck_result", + "Virtual Router Health Check result (1=success, 0=failure/unknown)", + "gauge"); + zoneName = zn; + routerName = rn; + checkName = cn; + checkType = ct; + result = res; + } + + @Override + public String toMetricsString() { + return String.format( + "%s{checkname=\"%s\",checktype=\"%s\",zone=\"%s\",virtualrouter=\"%s\"} %d", + name, checkName, checkType, zoneName, routerName, result); + } + } + + class ItemVRHealthCheckLastCheck extends Item { + String zoneName; + String routerName; + String checkName; + String checkType; + long lastCheckEpochSeconds; + + public ItemVRHealthCheckLastCheck(String zn, String rn, String cn, String ct, long ts) { + super("cloudstack_virtualrouter_healthcheck_lastcheck", + "Virtual Router Health Check last check timestamp (Unix seconds)", + "gauge"); + zoneName = zn; + routerName = rn; + checkName = cn; + checkType = ct; + lastCheckEpochSeconds = ts; + } + + @Override + public String toMetricsString() { + return String.format( + "%s{checkname=\"%s\",checktype=\"%s\",zone=\"%s\",virtualrouter=\"%s\"} %d", + name, checkName, checkType, zoneName, routerName, lastCheckEpochSeconds); + } + } } diff --git a/plugins/integrations/prometheus/src/test/java/org/apache/cloudstack/metrics/PrometheusExporterImplVRHealthCheckTest.java b/plugins/integrations/prometheus/src/test/java/org/apache/cloudstack/metrics/PrometheusExporterImplVRHealthCheckTest.java new file mode 100644 index 000000000000..23202b83d544 --- /dev/null +++ b/plugins/integrations/prometheus/src/test/java/org/apache/cloudstack/metrics/PrometheusExporterImplVRHealthCheckTest.java @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.metrics; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.network.VirtualNetworkApplianceService.RouterHealthStatus; +import com.cloud.network.dao.RouterHealthCheckResultDao; +import com.cloud.network.dao.RouterHealthCheckResultVO; +import com.cloud.network.router.VirtualRouter; +import com.cloud.vm.DomainRouterVO; +import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.dao.DomainRouterDao; + +@RunWith(MockitoJUnitRunner.class) +public class PrometheusExporterImplVRHealthCheckTest { + + private static final long ZONE_ID = 1L; + private static final String ZONE_NAME = "zone1"; + private static final String ZONE_UUID = "zone-uuid-1"; + + @Mock + private DomainRouterDao domainRouterDao; + @Mock + private RouterHealthCheckResultDao routerHealthCheckResultDao; + + private PrometheusExporterImpl exporter; + + @Before + public void setUp() { + exporter = new PrometheusExporterImpl(); + ReflectionTestUtils.setField(exporter, "domainRouterDao", domainRouterDao); + ReflectionTestUtils.setField(exporter, "routerHealthCheckResultDao", routerHealthCheckResultDao); + } + + private DomainRouterVO router(long id, String name, VirtualRouter.Role role, State state) { + DomainRouterVO router = Mockito.mock(DomainRouterVO.class); + Mockito.lenient().when(router.getId()).thenReturn(id); + Mockito.lenient().when(router.getInstanceName()).thenReturn(name); + Mockito.lenient().when(router.getRole()).thenReturn(role); + Mockito.lenient().when(router.getState()).thenReturn(state); + return router; + } + + private RouterHealthCheckResultVO check(long routerId, String name, RouterHealthStatus result, Date lastUpdate) { + RouterHealthCheckResultVO check = Mockito.mock(RouterHealthCheckResultVO.class); + Mockito.when(check.getRouterId()).thenReturn(routerId); + Mockito.lenient().when(check.getCheckName()).thenReturn(name); + Mockito.lenient().when(check.getCheckType()).thenReturn("basic"); + Mockito.lenient().when(check.getCheckResult()).thenReturn(result); + Mockito.lenient().when(check.getLastUpdateTime()).thenReturn(lastUpdate); + return check; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private List collectMetrics() { + List metrics = new ArrayList(); + exporter.addVirtualRouterHealthCheckMetrics(metrics, ZONE_ID, ZONE_NAME, ZONE_UUID); + return ((List) metrics).stream().map(item -> { + if (item instanceof PrometheusExporterImpl.ItemVRHealthCheckResult) { + return ((PrometheusExporterImpl.ItemVRHealthCheckResult) item).toMetricsString(); + } + return ((PrometheusExporterImpl.ItemVRHealthCheckLastCheck) item).toMetricsString(); + }).collect(Collectors.toList()); + } + + @Test + @SuppressWarnings("unchecked") + public void onlyRunningVirtualRoutersAreQueriedInASingleCall() { + DomainRouterVO running = router(10L, "r-10-VM", VirtualRouter.Role.VIRTUAL_ROUTER, State.Running); + DomainRouterVO stopped = router(11L, "r-11-VM", VirtualRouter.Role.VIRTUAL_ROUTER, State.Stopped); + DomainRouterVO internalLb = router(12L, "r-12-VM", VirtualRouter.Role.INTERNAL_LB_VM, State.Running); + Mockito.when(domainRouterDao.listByDataCenter(ZONE_ID)).thenReturn(List.of(running, stopped, internalLb)); + Mockito.when(routerHealthCheckResultDao.getHealthCheckResults(Mockito.anyList())).thenReturn(List.of()); + + collectMetrics(); + + ArgumentCaptor> ids = ArgumentCaptor.forClass(List.class); + Mockito.verify(routerHealthCheckResultDao, Mockito.times(1)).getHealthCheckResults(ids.capture()); + Mockito.verify(routerHealthCheckResultDao, Mockito.never()).getHealthCheckResults(Mockito.anyLong()); + assertEquals(List.of(10L), ids.getValue()); + } + + @Test + public void noRunningRoutersSkipsHealthCheckQuery() { + DomainRouterVO stopped = router(11L, "r-11-VM", VirtualRouter.Role.VIRTUAL_ROUTER, State.Stopped); + Mockito.when(domainRouterDao.listByDataCenter(ZONE_ID)).thenReturn(List.of(stopped)); + + assertTrue(collectMetrics().isEmpty()); + Mockito.verifyNoInteractions(routerHealthCheckResultDao); + } + + @Test + public void resultsAreExportedPerRouterAndCheck() { + DomainRouterVO r1 = router(10L, "r-10-VM", VirtualRouter.Role.VIRTUAL_ROUTER, State.Running); + DomainRouterVO r2 = router(20L, "r-20-VM", VirtualRouter.Role.VIRTUAL_ROUTER, State.Running); + Mockito.when(domainRouterDao.listByDataCenter(ZONE_ID)).thenReturn(List.of(r1, r2)); + List checks = List.of( + check(10L, "dns_check.py", RouterHealthStatus.SUCCESS, new Date(1735689600000L)), + check(20L, "disk_space_check.py", RouterHealthStatus.FAILED, null)); + Mockito.when(routerHealthCheckResultDao.getHealthCheckResults(Mockito.anyList())).thenReturn(checks); + + List metrics = collectMetrics(); + + assertEquals(List.of( + "cloudstack_virtualrouter_healthcheck_result{checkname=\"dns_check.py\",checktype=\"basic\",zone=\"zone1\",virtualrouter=\"r-10-VM\"} 1", + "cloudstack_virtualrouter_healthcheck_lastcheck{checkname=\"dns_check.py\",checktype=\"basic\",zone=\"zone1\",virtualrouter=\"r-10-VM\"} 1735689600", + "cloudstack_virtualrouter_healthcheck_result{checkname=\"disk_space_check.py\",checktype=\"basic\",zone=\"zone1\",virtualrouter=\"r-20-VM\"} 0"), + metrics); + } +}