add VR health checks to prometheus output

engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDao.java

@@ -28,6 +28,12 @@ public interface RouterHealthCheckResultDao extends GenericDao<RouterHealthCheck
      */
     List<RouterHealthCheckResultVO> getHealthCheckResults(long routerId);
 
+    /**
+     * @param routerIds
+     * @return Returns all the health checks in the database for the given router ids, in a single query
+     */
+    List<RouterHealthCheckResultVO> getHealthCheckResults(List<Long> routerIds);
+
     boolean expungeHealthChecks(long routerId);
 
     /**

engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultDaoImpl.java

@@ -17,6 +17,7 @@
 
 package com.cloud.network.dao;
 
+import java.util.Collections;
 import java.util.List;
 
 import org.springframework.stereotype.Component;
@@ -30,9 +31,14 @@ public class RouterHealthCheckResultDaoImpl extends GenericDaoBase<RouterHealthC
 
     private SearchBuilder<RouterHealthCheckResultVO> RouterChecksSearchBuilder;
     private SearchBuilder<RouterHealthCheckResultVO> IsRouterFailingSearchBuilder;
+    private SearchBuilder<RouterHealthCheckResultVO> RoutersChecksSearchBuilder;
 
     protected RouterHealthCheckResultDaoImpl() {
         super();
+        RoutersChecksSearchBuilder = createSearchBuilder();
+        RoutersChecksSearchBuilder.and("routerIds", RoutersChecksSearchBuilder.entity().getRouterId(), SearchCriteria.Op.IN);
+        RoutersChecksSearchBuilder.done();
+
         RouterChecksSearchBuilder = createSearchBuilder();
         RouterChecksSearchBuilder.and("routerId", RouterChecksSearchBuilder.entity().getRouterId(), SearchCriteria.Op.EQ);
         RouterChecksSearchBuilder.and("checkName", RouterChecksSearchBuilder.entity().getCheckName(), SearchCriteria.Op.EQ);
@@ -52,6 +58,16 @@ public List<RouterHealthCheckResultVO> getHealthCheckResults(long routerId) {
         return listBy(sc);
     }
 
+    @Override
+    public List<RouterHealthCheckResultVO> getHealthCheckResults(List<Long> routerIds) {
+        if (routerIds == null || routerIds.isEmpty()) {
+            return Collections.emptyList();
+        }
+        SearchCriteria<RouterHealthCheckResultVO> sc = RoutersChecksSearchBuilder.create();
+        sc.setParameters("routerIds", routerIds.toArray());
+        return listBy(sc);
+    }
+
     @Override
     public boolean expungeHealthChecks(long routerId) {
         SearchCriteria<RouterHealthCheckResultVO> sc = RouterChecksSearchBuilder.create();

plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java

@@ -59,7 +59,12 @@
 import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.host.dao.HostTagsDao;
+import com.cloud.network.VirtualNetworkApplianceService;
 import com.cloud.network.dao.IPAddressDao;
+import com.cloud.network.dao.RouterHealthCheckResultDao;
+import com.cloud.network.dao.RouterHealthCheckResultVO;
+import com.cloud.network.router.VirtualNetworkApplianceManager;
+import com.cloud.network.router.VirtualRouter;
 import com.cloud.storage.ImageStore;
 import com.cloud.storage.StorageStats;
 import com.cloud.storage.Volume;
@@ -70,7 +75,9 @@
 import com.cloud.utils.Ternary;
 import com.cloud.utils.component.Manager;
 import com.cloud.utils.component.ManagerBase;
+import com.cloud.vm.DomainRouterVO;
 import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.VMInstanceDao;
 
@@ -137,6 +144,10 @@ public String toString() {
     private HostTagsDao _hostTagsDao;
     @Inject
     private CAManager caManager;
+    @Inject
+    private DomainRouterDao domainRouterDao;
+    @Inject
+    private RouterHealthCheckResultDao routerHealthCheckResultDao;
 
     public PrometheusExporterImpl() {
         super();
@@ -501,10 +512,47 @@ private void addVMsBySizeMetrics(final List<Item> metricsList, final long dcId,
         }
     }
 
+    protected void addVirtualRouterHealthCheckMetrics(final List<Item> metricsList, final long dcId, final String zoneName, final String zoneUuid) {
+        List<DomainRouterVO> routers = domainRouterDao.listByDataCenter(dcId);
+        if (routers == null) {
+            return;
+        }
+        // only running virtual routers; results of stopped or destroyed routers are stale
+        Map<Long, DomainRouterVO> runningRouters = routers.stream()
+                .filter(router -> router.getRole() == VirtualRouter.Role.VIRTUAL_ROUTER && router.getState() == State.Running)
+                .collect(Collectors.toMap(DomainRouterVO::getId, router -> router));
+        if (runningRouters.isEmpty()) {
+            return;
+        }
+        // one query for all routers in the zone instead of one per router
+        List<RouterHealthCheckResultVO> checks = routerHealthCheckResultDao.getHealthCheckResults(new ArrayList<>(runningRouters.keySet()));
+        if (checks == null) {
+            return;
+        }
+        for (RouterHealthCheckResultVO check : checks) {
+            DomainRouterVO router = runningRouters.get(check.getRouterId());
+            if (router == null) {
+                continue;
+            }
+            int resultValue = (check.getCheckResult() ==
+                    VirtualNetworkApplianceService.RouterHealthStatus.SUCCESS) ? 1 : 0;
+            metricsList.add(new ItemVRHealthCheckResult(
+                    zoneName, router.getInstanceName(),
+                    check.getCheckName(), check.getCheckType(), resultValue));
+            if (check.getLastUpdateTime() != null) {
+                long epochSeconds = check.getLastUpdateTime().getTime() / 1000L;
+                metricsList.add(new ItemVRHealthCheckLastCheck(
+                        zoneName, router.getInstanceName(),
+                        check.getCheckName(), check.getCheckType(), epochSeconds));
+            }
+        }
+    }
+
     @Override
     public void updateMetrics() {
         final List<Item> latestMetricsItems = new ArrayList<Item>();
         try {
+            final boolean routerHealthChecksEnabled = Boolean.TRUE.equals(VirtualNetworkApplianceManager.RouterHealthChecksEnabled.value());
             for (final DataCenterVO dc : dcDao.listAll()) {
                 final String zoneName = dc.getName();
                 final String zoneUuid = dc.getUuid();
@@ -518,6 +566,9 @@ public void updateMetrics() {
                 addDomainMetrics(latestMetricsItems, zoneName, zoneUuid);
                 addAccountMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid);
                 addVMsBySizeMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid);
+                if (routerHealthChecksEnabled) {
+                    addVirtualRouterHealthCheckMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid);
+                }
             }
             addDomainLimits(latestMetricsItems);
             addDomainResourceCount(latestMetricsItems);
@@ -1180,4 +1231,56 @@ public String toMetricsString() {
             return String.format("%s{zone=\"%s\",hostname=\"%s\",ip=\"%s\"} %d", name, zoneName, hostName, hostIp, expiryTimestamp);
         }
     }
+
+    class ItemVRHealthCheckResult extends Item {
+        String zoneName;
+        String routerName;
+        String checkName;
+        String checkType;
+        int result;
+
+        public ItemVRHealthCheckResult(String zn, String rn, String cn, String ct, int res) {
+            super("cloudstack_virtualrouter_healthcheck_result",
+                    "Virtual Router Health Check result (1=success, 0=failure/unknown)",
+                    "gauge");
+            zoneName = zn;
+            routerName = rn;
+            checkName = cn;
+            checkType = ct;
+            result = res;
+        }
+
+        @Override
+        public String toMetricsString() {
+            return String.format(
+                    "%s{checkname=\"%s\",checktype=\"%s\",zone=\"%s\",virtualrouter=\"%s\"} %d",
+                    name, checkName, checkType, zoneName, routerName, result);
+        }
+    }
+
+    class ItemVRHealthCheckLastCheck extends Item {
+        String zoneName;
+        String routerName;
+        String checkName;
+        String checkType;
+        long lastCheckEpochSeconds;
+
+        public ItemVRHealthCheckLastCheck(String zn, String rn, String cn, String ct, long ts) {
+            super("cloudstack_virtualrouter_healthcheck_lastcheck",
+                    "Virtual Router Health Check last check timestamp (Unix seconds)",
+                    "gauge");
+            zoneName = zn;
+            routerName = rn;
+            checkName = cn;
+            checkType = ct;
+            lastCheckEpochSeconds = ts;
+        }
+
+        @Override
+        public String toMetricsString() {
+            return String.format(
+                    "%s{checkname=\"%s\",checktype=\"%s\",zone=\"%s\",virtualrouter=\"%s\"} %d",
+                    name, checkName, checkType, zoneName, routerName, lastCheckEpochSeconds);
+        }
+    }
 }

plugins/integrations/prometheus/src/test/java/org/apache/cloudstack/metrics/PrometheusExporterImplVRHealthCheckTest.java

@@ -0,0 +1,140 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.metrics;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import com.cloud.network.VirtualNetworkApplianceService.RouterHealthStatus;
+import com.cloud.network.dao.RouterHealthCheckResultDao;
+import com.cloud.network.dao.RouterHealthCheckResultVO;
+import com.cloud.network.router.VirtualRouter;
+import com.cloud.vm.DomainRouterVO;
+import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.dao.DomainRouterDao;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PrometheusExporterImplVRHealthCheckTest {
+
+    private static final long ZONE_ID = 1L;
+    private static final String ZONE_NAME = "zone1";
+    private static final String ZONE_UUID = "zone-uuid-1";
+
+    @Mock
+    private DomainRouterDao domainRouterDao;
+    @Mock
+    private RouterHealthCheckResultDao routerHealthCheckResultDao;
+
+    private PrometheusExporterImpl exporter;
+
+    @Before
+    public void setUp() {
+        exporter = new PrometheusExporterImpl();
+        ReflectionTestUtils.setField(exporter, "domainRouterDao", domainRouterDao);
+        ReflectionTestUtils.setField(exporter, "routerHealthCheckResultDao", routerHealthCheckResultDao);
+    }
+
+    private DomainRouterVO router(long id, String name, VirtualRouter.Role role, State state) {
+        DomainRouterVO router = Mockito.mock(DomainRouterVO.class);
+        Mockito.lenient().when(router.getId()).thenReturn(id);
+        Mockito.lenient().when(router.getInstanceName()).thenReturn(name);
+        Mockito.lenient().when(router.getRole()).thenReturn(role);
+        Mockito.lenient().when(router.getState()).thenReturn(state);
+        return router;
+    }
+
+    private RouterHealthCheckResultVO check(long routerId, String name, RouterHealthStatus result, Date lastUpdate) {
+        RouterHealthCheckResultVO check = Mockito.mock(RouterHealthCheckResultVO.class);
+        Mockito.when(check.getRouterId()).thenReturn(routerId);
+        Mockito.lenient().when(check.getCheckName()).thenReturn(name);
+        Mockito.lenient().when(check.getCheckType()).thenReturn("basic");
+        Mockito.lenient().when(check.getCheckResult()).thenReturn(result);
+        Mockito.lenient().when(check.getLastUpdateTime()).thenReturn(lastUpdate);
+        return check;
+    }
+
+    @SuppressWarnings({"rawtypes", "unchecked"})
+    private List<String> collectMetrics() {
+        List metrics = new ArrayList();
+        exporter.addVirtualRouterHealthCheckMetrics(metrics, ZONE_ID, ZONE_NAME, ZONE_UUID);
+        return ((List<Object>) metrics).stream().map(item -> {
+            if (item instanceof PrometheusExporterImpl.ItemVRHealthCheckResult) {
+                return ((PrometheusExporterImpl.ItemVRHealthCheckResult) item).toMetricsString();
+            }
+            return ((PrometheusExporterImpl.ItemVRHealthCheckLastCheck) item).toMetricsString();
+        }).collect(Collectors.toList());
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void onlyRunningVirtualRoutersAreQueriedInASingleCall() {
+        DomainRouterVO running = router(10L, "r-10-VM", VirtualRouter.Role.VIRTUAL_ROUTER, State.Running);
+        DomainRouterVO stopped = router(11L, "r-11-VM", VirtualRouter.Role.VIRTUAL_ROUTER, State.Stopped);
+        DomainRouterVO internalLb = router(12L, "r-12-VM", VirtualRouter.Role.INTERNAL_LB_VM, State.Running);
+        Mockito.when(domainRouterDao.listByDataCenter(ZONE_ID)).thenReturn(List.of(running, stopped, internalLb));
+        Mockito.when(routerHealthCheckResultDao.getHealthCheckResults(Mockito.anyList())).thenReturn(List.of());
+
+        collectMetrics();
+
+        ArgumentCaptor<List<Long>> ids = ArgumentCaptor.forClass(List.class);
+        Mockito.verify(routerHealthCheckResultDao, Mockito.times(1)).getHealthCheckResults(ids.capture());
+        Mockito.verify(routerHealthCheckResultDao, Mockito.never()).getHealthCheckResults(Mockito.anyLong());
+        assertEquals(List.of(10L), ids.getValue());
+    }
+
+    @Test
+    public void noRunningRoutersSkipsHealthCheckQuery() {
+        DomainRouterVO stopped = router(11L, "r-11-VM", VirtualRouter.Role.VIRTUAL_ROUTER, State.Stopped);
+        Mockito.when(domainRouterDao.listByDataCenter(ZONE_ID)).thenReturn(List.of(stopped));
+
+        assertTrue(collectMetrics().isEmpty());
+        Mockito.verifyNoInteractions(routerHealthCheckResultDao);
+    }
+
+    @Test
+    public void resultsAreExportedPerRouterAndCheck() {
+        DomainRouterVO r1 = router(10L, "r-10-VM", VirtualRouter.Role.VIRTUAL_ROUTER, State.Running);
+        DomainRouterVO r2 = router(20L, "r-20-VM", VirtualRouter.Role.VIRTUAL_ROUTER, State.Running);
+        Mockito.when(domainRouterDao.listByDataCenter(ZONE_ID)).thenReturn(List.of(r1, r2));
+        List<RouterHealthCheckResultVO> checks = List.of(
+                check(10L, "dns_check.py", RouterHealthStatus.SUCCESS, new Date(1735689600000L)),
+                check(20L, "disk_space_check.py", RouterHealthStatus.FAILED, null));
+        Mockito.when(routerHealthCheckResultDao.getHealthCheckResults(Mockito.anyList())).thenReturn(checks);
+
+        List<String> metrics = collectMetrics();
+
+        assertEquals(List.of(
+                "cloudstack_virtualrouter_healthcheck_result{checkname=\"dns_check.py\",checktype=\"basic\",zone=\"zone1\",virtualrouter=\"r-10-VM\"} 1",
+                "cloudstack_virtualrouter_healthcheck_lastcheck{checkname=\"dns_check.py\",checktype=\"basic\",zone=\"zone1\",virtualrouter=\"r-10-VM\"} 1735689600",
+                "cloudstack_virtualrouter_healthcheck_result{checkname=\"disk_space_check.py\",checktype=\"basic\",zone=\"zone1\",virtualrouter=\"r-20-VM\"} 0"),
+                metrics);
+    }
+}