From a55c4d901e21a34d8683ef49aee91f80c210d722 Mon Sep 17 00:00:00 2001
From: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
Date: Thu, 15 Oct 2026 15:48:19 -0700
Subject: [PATCH 1/6] Support VictoriaMetrics observability and Grafana
 datasources

Accept native VictoriaMetrics datasource types throughout dashboard validation and creation, retain the Prometheus default and Loki workflows, and document tenant-scoped query endpoints. Cover both dashboard tools and metrics transport, including live VictoriaMetrics v1.154.0 queries.

Signed-off-by: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
---
 README.md                                     |  4 +-
 docs/installation.md                          |  4 +-
 docs/victoriametrics.md                       | 77 +++++++++++++++++++
 .../openclaw-seed/extensions/grafana/index.ts | 46 ++++++-----
 .../extensions/grafana/openclaw.plugin.json   | 10 ++-
 .../extensions/observability/grafana.ts       | 42 +++++-----
 .../extensions/observability/index.ts         | 16 ++--
 .../observability/openclaw.plugin.json        | 10 ++-
 .../files/openclaw-seed/nemoclaw.json         |  2 +
 .../skills/observability/SKILL.md             | 12 ++-
 helm/mosaic-stack/tests/chart-render.test.mjs | 18 +++++
 .../tests/mixed-dashboards.test.mjs           | 14 +++-
 .../tests/victoriametrics.test.mjs            | 69 +++++++++++++++++
 helm/mosaic-stack/values.yaml                 |  2 +
 index.md                                      |  1 +
 15 files changed, 267 insertions(+), 60 deletions(-)
 create mode 100644 docs/victoriametrics.md
 create mode 100644 helm/mosaic-stack/tests/victoriametrics.test.mjs

diff --git a/README.md b/README.md
index 8f8f67a7..d629b9a1 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ AI Factory Operations Agent is an NVIDIA blueprint for deploying an extensible a
 Supported workflows include:
 
 - Read-only Kubernetes workload and cluster inspection.
-- Prometheus queries and Grafana dashboard creation.
+- Prometheus and VictoriaMetrics queries and Grafana dashboard creation.
 - Slurm job root-cause analysis from scheduler and log evidence.
 - Optional cluster-management integrations.
 - Sandboxed command execution with an auditable agent workflow.
@@ -29,7 +29,7 @@ OpenShell isolates general agent command execution. Kubernetes, observability, S
 - Runtime: Kubernetes, `kubectl`, and Helm 3.
 - LLM: an OpenAI-compatible endpoint or a cluster capable of running chart-managed vLLM.
 - GPU/driver: required only when deploying chart-managed vLLM; requirements depend on the selected model profile.
-- Optional services: Prometheus, Grafana, Slurm, or cluster-management endpoints for the corresponding modules.
+- Optional services: Prometheus or VictoriaMetrics, Grafana, Slurm, or cluster-management endpoints for the corresponding modules.
 
 ## Getting Started
 
diff --git a/docs/installation.md b/docs/installation.md
index cc07ca30..e2759c4f 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -202,7 +202,7 @@ All module changes below update an existing AI Factory Operations Agent release
 
 ### Observability
 
-To connect AI Factory Operations Agent to an existing Prometheus service, run:
+To connect AI Factory Operations Agent to an existing Prometheus-compatible metrics service, run:
 
 ```bash
 export PROMETHEUS_URL='http://kube-prometheus-stack-prometheus.prometheus.svc.cluster.local:9090'
@@ -216,6 +216,8 @@ helm upgrade mosaic "$MOSAIC_CHART" \
   --timeout 12m
 ```
 
+VictoriaMetrics uses the same endpoint setting. See [VictoriaMetrics metrics and dashboards](victoriametrics.md) for single-node and cluster URLs and native Grafana datasource configuration.
+
 ### Read-only Loki logs
 
 To connect Mosaic to an existing Loki service, run:
diff --git a/docs/victoriametrics.md b/docs/victoriametrics.md
new file mode 100644
index 00000000..f7b68864
--- /dev/null
+++ b/docs/victoriametrics.md
@@ -0,0 +1,77 @@
+<!-- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
+<!-- SPDX-License-Identifier: Apache-2.0 -->
+
+# VictoriaMetrics
+
+Connect the observability component to an existing VictoriaMetrics service through
+its Prometheus-compatible API. The metric discovery, instant query and historical
+query tools support PromQL and MetricsQL. Mosaic does not install VictoriaMetrics.
+
+## Metrics endpoint
+
+Set `observability.prometheusUrl` to the query base URL. This setting accepts
+Prometheus-compatible services, including VictoriaMetrics; no backend selector is
+required.
+
+| Deployment | Example base URL |
+| --- | --- |
+| Single-node VictoriaMetrics | `http://victoriametrics.monitoring.svc.cluster.local:8428` |
+| VictoriaMetrics cluster | `http://vmselect.monitoring.svc.cluster.local:8481/select/0/prometheus` |
+
+For a cluster, replace `0` with the intended tenant ID. Use the read endpoint
+(`vmselect`), rather than `vminsert` or `vmagent`. Include the tenant path and any
+proxy prefix, and omit `/api/v1`; the tools append the API paths themselves.
+See the [VictoriaMetrics API examples](https://docs.victoriametrics.com/url-examples.html).
+
+Use these values with your existing installation configuration:
+
+```yaml
+modules:
+  observability:
+    enabled: true
+observability:
+  prometheusUrl: http://vmselect.monitoring.svc.cluster.local:8481/select/0/prometheus
+```
+
+The endpoint must be reachable from the agent pod. Grafana credentials authenticate
+Grafana requests, not direct metrics queries. Do not put credentials in the URL or
+committed values.
+
+## Grafana dashboards
+
+VictoriaMetrics works with Grafana's Prometheus datasource and with the native
+VictoriaMetrics datasource plugin. Configure Grafana to query the same backend and
+tenant as `observability.prometheusUrl`, then use the datasource's actual UID.
+
+For an existing Prometheus datasource, keep the default type:
+
+```yaml
+observability:
+  grafanaDatasourceUid: metrics
+  grafanaDatasourceType: prometheus
+```
+
+For Grafana's native VictoriaMetrics plugin:
+
+```yaml
+observability:
+  grafanaDatasourceUid: victoria-metrics
+  grafanaDatasourceType: victoriametrics-metrics-datasource
+```
+
+The Grafana administrator installs and configures the plugin; see the
+[VictoriaMetrics Grafana integration](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/datasource/).
+Mosaic preserves this datasource type in dashboard panels, direct Grafana query
+validation and frontend proxy validation. An explicit panel datasource overrides
+the configured default. Loki panels can share a dashboard with either metrics
+datasource type.
+
+## Verify access
+
+Ask Mosaic to list the available metric names, query a discovered metric, and
+show its history for the past hour. Ask it to discover Grafana datasources before
+creating a dashboard. Use the discovered UID and type for each panel.
+
+MetricsQL extensions require a VictoriaMetrics backend. PromQL queries remain
+supported, with the differences described in the
+[MetricsQL documentation](https://docs.victoriametrics.com/victoriametrics/metricsql/).
diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/grafana/index.ts b/helm/mosaic-stack/files/openclaw-seed/extensions/grafana/index.ts
index 8efb63cd..186b4c88 100644
--- a/helm/mosaic-stack/files/openclaw-seed/extensions/grafana/index.ts
+++ b/helm/mosaic-stack/files/openclaw-seed/extensions/grafana/index.ts
@@ -8,7 +8,7 @@ declare const Buffer: { from(value: string): { toString(encoding: string): strin
 type PanelSpec = {
   title: string;
   query: string;
-  datasource?: { type: "prometheus" | "loki"; uid: string };
+  datasource?: { type: "prometheus" | "victoriametrics-metrics-datasource" | "loki"; uid: string };
   unit?: string;
   visualization?: "timeseries" | "stat" | "gauge" | "table" | "logs";
   legend?: string;
@@ -157,12 +157,12 @@ async function fetchJson(url: string, options?: RequestInit) {
   return payload as Record<string, unknown>;
 }
 
-function buildGrafanaQueryPayload(panel: PanelSpec, rangeMinutes: number, datasourceUid: string) {
+function buildGrafanaQueryPayload(panel: PanelSpec, rangeMinutes: number, defaultDatasource: NonNullable<PanelSpec["datasource"]>) {
   return {
     queries: [
       {
         refId: "A",
-        datasource: panel.datasource || { type: "prometheus", uid: datasourceUid },
+        datasource: panel.datasource || defaultDatasource,
         expr: panel.query,
         ...(panel.datasource?.type === "loki" ? {queryType: "range", maxLines: 1000} : {}),
         range: true,
@@ -210,7 +210,7 @@ async function validatePanels(
   grafanaUrl: string,
   uiUrl: string,
   headers: Record<string, string>,
-  datasourceUid: string,
+  defaultDatasource: NonNullable<PanelSpec["datasource"]>,
   panels: PanelSpec[],
   requireData: boolean,
   rangeMinutes: number,
@@ -220,8 +220,8 @@ async function validatePanels(
 
   const datasources = await fetchJson(`${grafanaUrl}/api/datasources`, {headers}) as unknown as Array<{uid: string; type: string; name: string}>;
   for (const panel of panels) {
-    const selected = panel.datasource || {type: "prometheus", uid: datasourceUid};
-    if (!["prometheus", "loki"].includes(selected.type) || !datasources.some(source => source.uid === selected.uid && source.type === selected.type)) {
+    const selected = panel.datasource || defaultDatasource;
+    if (!["prometheus", "victoriametrics-metrics-datasource", "loki"].includes(selected.type) || !datasources.some(source => source.uid === selected.uid && source.type === selected.type)) {
       issues.push({panelTitle: panel.title, query: panel.query, severity: "error", message: "Panel datasource UID or type is missing or unavailable. Select the intended datasource from availableDatasources and retry."});
       continue;
     }
@@ -232,10 +232,10 @@ async function validatePanels(
     const url = new URL(`${prometheusUrl}/api/v1/query`);
     url.searchParams.set("query", panel.query);
     try {
-      const payload = selected.type === "prometheus" ? await fetchJson(url.toString()) : {};
+      const payload = selected.type !== "loki" ? await fetchJson(url.toString()) : {};
       const data = payload.data as { result?: unknown[] } | undefined;
       const result = Array.isArray(data?.result) ? data.result : [];
-      if (requireData && selected.type === "prometheus" && result.length === 0) {
+      if (requireData && selected.type !== "loki" && result.length === 0) {
         issues.push({
           panelTitle: panel.title,
           query: panel.query,
@@ -248,7 +248,7 @@ async function validatePanels(
         const grafanaPayload = await fetchJson(`${grafanaUrl}/api/ds/query`, {
           method: "POST",
           headers,
-          body: JSON.stringify(buildGrafanaQueryPayload(panel, rangeMinutes, datasourceUid)),
+          body: JSON.stringify(buildGrafanaQueryPayload(panel, rangeMinutes, defaultDatasource)),
         });
         frameCount = grafanaFrameCount(grafanaPayload);
         if (requireData && frameCount === 0) {
@@ -271,7 +271,7 @@ async function validatePanels(
         const uiProxyPayload = await fetchJson(`${uiUrl}/api/grafana/proxy/api/ds/query`, {
           method: "POST",
           headers: { ...uiAuthHeaders(), Origin: uiUrl },
-          body: JSON.stringify(buildGrafanaQueryPayload(panel, rangeMinutes, datasourceUid)),
+          body: JSON.stringify(buildGrafanaQueryPayload(panel, rangeMinutes, defaultDatasource)),
         });
         const uiFrameCount = grafanaFrameCount(uiProxyPayload);
         if (requireData && uiFrameCount === 0) {
@@ -339,7 +339,7 @@ function appendTimeRange(path: string, rangeMinutes: number) {
   return `${path}${separator}orgId=1&from=now-${Math.trunc(rangeMinutes)}m&to=now&kiosk`;
 }
 
-function buildPanel(panel: PanelSpec, index: number, datasourceUid: string) {
+function buildPanel(panel: PanelSpec, index: number, defaultDatasource: NonNullable<PanelSpec["datasource"]>) {
   const type = panel.visualization || "timeseries";
   const width = type === "stat" || type === "gauge" ? 6 : 12;
   return {
@@ -347,7 +347,7 @@ function buildPanel(panel: PanelSpec, index: number, datasourceUid: string) {
     type,
     title: panel.title,
     description: panel.description,
-    datasource: panel.datasource || { type: "prometheus", uid: datasourceUid },
+    datasource: panel.datasource || defaultDatasource,
     gridPos: {
       h: type === "stat" || type === "gauge" ? 6 : 8,
       w: width,
@@ -376,7 +376,7 @@ function buildPanel(panel: PanelSpec, index: number, datasourceUid: string) {
   };
 }
 
-function buildDashboard(title: string, panels: PanelSpec[], datasourceUid: string, rangeMinutes: number) {
+function buildDashboard(title: string, panels: PanelSpec[], defaultDatasource: NonNullable<PanelSpec["datasource"]>, rangeMinutes: number) {
   return {
     id: null,
     uid: `mosaic-${Date.now().toString(36)}`,
@@ -390,7 +390,7 @@ function buildDashboard(title: string, panels: PanelSpec[], datasourceUid: strin
       from: `now-${Math.max(1, Math.trunc(rangeMinutes))}m`,
       to: "now",
     },
-    panels: panels.map((panel, index) => buildPanel(panel, index, datasourceUid)),
+    panels: panels.map((panel, index) => buildPanel(panel, index, defaultDatasource)),
   };
 }
 
@@ -403,6 +403,10 @@ export default definePluginEntry({
     const prometheusUrl = resolveString(api.pluginConfig, "prometheusUrl");
     const uiUrl = resolveString(api.pluginConfig, "uiUrl", DEFAULT_UI_URL);
     const datasourceUid = resolveString(api.pluginConfig, "datasourceUid");
+    const defaultDatasource = {
+      uid: datasourceUid,
+      type: resolveString(api.pluginConfig, "datasourceType", "prometheus") as NonNullable<PanelSpec["datasource"]>["type"],
+    };
     const headers = authHeaders(api.pluginConfig);
     const registerTool = (tool: Parameters<typeof api.registerTool>[0]) => {
       if (isToolEnabled(api.pluginConfig, tool.name)) {
@@ -445,7 +449,7 @@ export default definePluginEntry({
     registerTool({
       name: "grafana_dashboard_validate",
       label: "Validate Grafana Dashboard Queries",
-      description: "Validate dashboard panel PromQL or LogQL against the selected Grafana datasource before creating a dashboard.",
+      description: "Validate dashboard panel PromQL, MetricsQL or LogQL against the selected Grafana datasource before creating a dashboard.",
       parameters: {
         type: "object",
         additionalProperties: false,
@@ -453,7 +457,7 @@ export default definePluginEntry({
           preset: { type: "string", description: `Optional preset: ${Object.keys(PRESETS).join(", ")}.` },
           panels: {
             type: "array",
-            description: "Panel specs: title, query (PromQL or LogQL), datasource {type: prometheus or loki, uid: discovered Grafana UID}, visualization (timeseries, stat, gauge, table, logs), optional unit/legend/description. Omit datasource only to use the configured Prometheus default.",
+            description: "Panel specs: title, query (PromQL, MetricsQL or LogQL), datasource {type: prometheus, victoriametrics-metrics-datasource or loki, uid: discovered Grafana UID}, visualization (timeseries, stat, gauge, table, logs), optional unit/legend/description. Omit datasource only to use the configured metrics default.",
             items: { type: "object" },
           },
           requireData: { type: "boolean", description: "If true, empty query results are validation errors. Default true." },
@@ -469,7 +473,7 @@ export default definePluginEntry({
           grafanaUrl,
           uiUrl,
           headers,
-          datasourceUid,
+          defaultDatasource,
           panels,
           boolParam(rawParams.requireData, true),
           Math.max(1, Math.min(10080, numberParam(rawParams.rangeMinutes, 60))),
@@ -481,7 +485,7 @@ export default definePluginEntry({
     registerTool({
       name: "grafana_dashboard_create",
       label: "Create Grafana Dashboard",
-      description: "Create a Grafana dashboard after validating every Prometheus or Loki panel query, then open it in the UI Grafana tab.",
+      description: "Create a Grafana dashboard after validating every Prometheus, VictoriaMetrics or Loki panel query, then open it in the UI Grafana tab.",
       parameters: {
         type: "object",
         additionalProperties: false,
@@ -490,7 +494,7 @@ export default definePluginEntry({
           preset: { type: "string", description: `Optional preset: ${Object.keys(PRESETS).join(", ")}.` },
           panels: {
             type: "array",
-            description: "Panel specs: title, query (PromQL or LogQL), datasource {type: prometheus or loki, uid: discovered Grafana UID}, visualization (timeseries, stat, gauge, table, logs), optional unit/legend/description. Omit datasource only to use the configured Prometheus default.",
+            description: "Panel specs: title, query (PromQL, MetricsQL or LogQL), datasource {type: prometheus, victoriametrics-metrics-datasource or loki, uid: discovered Grafana UID}, visualization (timeseries, stat, gauge, table, logs), optional unit/legend/description. Omit datasource only to use the configured metrics default.",
             items: { type: "object" },
           },
           rangeMinutes: { type: "number", description: "Dashboard time range in minutes. Default 60." },
@@ -507,7 +511,7 @@ export default definePluginEntry({
 
         const requireData = boolParam(rawParams.requireData, true);
         const rangeMinutes = Math.max(1, Math.min(10080, numberParam(rawParams.rangeMinutes, 60)));
-        const validation = await validatePanels(prometheusUrl, grafanaUrl, uiUrl, headers, datasourceUid, panels, requireData, rangeMinutes);
+        const validation = await validatePanels(prometheusUrl, grafanaUrl, uiUrl, headers, defaultDatasource, panels, requireData, rangeMinutes);
         if (!validation.ok) {
           return jsonToolResult({
             created: false,
@@ -516,7 +520,7 @@ export default definePluginEntry({
           });
         }
 
-        const dashboard = buildDashboard(title, panels, datasourceUid, rangeMinutes);
+        const dashboard = buildDashboard(title, panels, defaultDatasource, rangeMinutes);
         const createPayload = await fetchJson(`${grafanaUrl}/api/dashboards/db`, {
           method: "POST",
           headers,
diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/grafana/openclaw.plugin.json b/helm/mosaic-stack/files/openclaw-seed/extensions/grafana/openclaw.plugin.json
index d5577cbc..202892f2 100644
--- a/helm/mosaic-stack/files/openclaw-seed/extensions/grafana/openclaw.plugin.json
+++ b/helm/mosaic-stack/files/openclaw-seed/extensions/grafana/openclaw.plugin.json
@@ -19,7 +19,7 @@
       },
       "prometheusUrl": {
         "type": "string",
-        "description": "Prometheus service base URL used for validation."
+        "description": "Prometheus-compatible query base URL, including any VictoriaMetrics cluster tenant prefix."
       },
       "uiUrl": {
         "type": "string",
@@ -50,6 +50,14 @@
         "type": "array",
         "items": { "type": "string" },
         "description": "Optional list of Grafana tool names to suppress."
+      },
+      "datasourceType": {
+        "type": "string",
+        "enum": [
+          "prometheus",
+          "victoriametrics-metrics-datasource"
+        ],
+        "description": "Grafana metrics datasource type. Defaults to prometheus."
       }
     },
     "required": ["grafanaUrl", "prometheusUrl", "datasourceUid"]
diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/grafana.ts b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/grafana.ts
index b9c62148..20ab659b 100644
--- a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/grafana.ts
+++ b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/grafana.ts
@@ -6,7 +6,7 @@ declare const Buffer: { from(value: string): { toString(encoding: string): strin
 type PanelSpec = {
   title: string;
   query: string;
-  datasource?: { type: "prometheus" | "loki"; uid: string };
+  datasource?: { type: "prometheus" | "victoriametrics-metrics-datasource" | "loki"; uid: string };
   unit?: string;
   visualization?: "timeseries" | "stat" | "gauge" | "table" | "logs";
   legend?: string;
@@ -118,12 +118,12 @@ async function fetchJson(url: string, options?: RequestInit) {
   return payload as Record<string, unknown>;
 }
 
-function buildGrafanaQueryPayload(panel: PanelSpec, rangeMinutes: number, datasourceUid: string) {
+function buildGrafanaQueryPayload(panel: PanelSpec, rangeMinutes: number, defaultDatasource: NonNullable<PanelSpec["datasource"]>) {
   return {
     queries: [
       {
         refId: "A",
-        datasource: panel.datasource || { type: "prometheus", uid: datasourceUid },
+        datasource: panel.datasource || defaultDatasource,
         expr: panel.query,
         ...(panel.datasource?.type === "loki" ? {queryType: "range", maxLines: 1000} : {}),
         range: true,
@@ -171,7 +171,7 @@ async function validatePanels(
   grafanaUrl: string,
   uiUrl: string,
   headers: Record<string, string>,
-  datasourceUid: string,
+  defaultDatasource: NonNullable<PanelSpec["datasource"]>,
   panels: PanelSpec[],
   requireData: boolean,
   rangeMinutes: number,
@@ -181,8 +181,8 @@ async function validatePanels(
 
   const datasources = await fetchJson(`${grafanaUrl}/api/datasources`, {headers}) as unknown as Array<{uid: string; type: string; name: string}>;
   for (const panel of panels) {
-    const selected = panel.datasource || {type: "prometheus", uid: datasourceUid};
-    if (!["prometheus", "loki"].includes(selected.type) || !datasources.some(source => source.uid === selected.uid && source.type === selected.type)) {
+    const selected = panel.datasource || defaultDatasource;
+    if (!["prometheus", "victoriametrics-metrics-datasource", "loki"].includes(selected.type) || !datasources.some(source => source.uid === selected.uid && source.type === selected.type)) {
       issues.push({panelTitle: panel.title, query: panel.query, severity: "error", message: "Panel datasource UID or type is missing or unavailable. Select the intended datasource from availableDatasources and retry."});
       continue;
     }
@@ -193,10 +193,10 @@ async function validatePanels(
     const url = new URL(`${prometheusUrl}/api/v1/query`);
     url.searchParams.set("query", panel.query);
     try {
-      const payload = selected.type === "prometheus" ? await fetchJson(url.toString()) : {};
+      const payload = selected.type !== "loki" ? await fetchJson(url.toString()) : {};
       const data = payload.data as { result?: unknown[] } | undefined;
       const result = Array.isArray(data?.result) ? data.result : [];
-      if (requireData && selected.type === "prometheus" && result.length === 0) {
+      if (requireData && selected.type !== "loki" && result.length === 0) {
         issues.push({
           panelTitle: panel.title,
           query: panel.query,
@@ -209,7 +209,7 @@ async function validatePanels(
         const grafanaPayload = await fetchJson(`${grafanaUrl}/api/ds/query`, {
           method: "POST",
           headers,
-          body: JSON.stringify(buildGrafanaQueryPayload(panel, rangeMinutes, datasourceUid)),
+          body: JSON.stringify(buildGrafanaQueryPayload(panel, rangeMinutes, defaultDatasource)),
         });
         frameCount = grafanaFrameCount(grafanaPayload);
         if (requireData && frameCount === 0) {
@@ -232,7 +232,7 @@ async function validatePanels(
         const uiProxyPayload = await fetchJson(`${uiUrl}/api/grafana/proxy/api/ds/query`, {
           method: "POST",
           headers: { ...uiAuthHeaders(), Origin: uiUrl },
-          body: JSON.stringify(buildGrafanaQueryPayload(panel, rangeMinutes, datasourceUid)),
+          body: JSON.stringify(buildGrafanaQueryPayload(panel, rangeMinutes, defaultDatasource)),
         });
         const uiFrameCount = grafanaFrameCount(uiProxyPayload);
         if (requireData && uiFrameCount === 0) {
@@ -300,7 +300,7 @@ function appendTimeRange(path: string, rangeMinutes: number) {
   return `${path}${separator}orgId=1&from=now-${Math.trunc(rangeMinutes)}m&to=now&kiosk`;
 }
 
-function buildPanel(panel: PanelSpec, index: number, datasourceUid: string) {
+function buildPanel(panel: PanelSpec, index: number, defaultDatasource: NonNullable<PanelSpec["datasource"]>) {
   const type = panel.visualization || "timeseries";
   const width = type === "stat" || type === "gauge" ? 6 : 12;
   return {
@@ -308,7 +308,7 @@ function buildPanel(panel: PanelSpec, index: number, datasourceUid: string) {
     type,
     title: panel.title,
     description: panel.description,
-    datasource: panel.datasource || { type: "prometheus", uid: datasourceUid },
+    datasource: panel.datasource || defaultDatasource,
     gridPos: {
       h: type === "stat" || type === "gauge" ? 6 : 8,
       w: width,
@@ -337,7 +337,7 @@ function buildPanel(panel: PanelSpec, index: number, datasourceUid: string) {
   };
 }
 
-function buildDashboard(title: string, panels: PanelSpec[], datasourceUid: string, rangeMinutes: number) {
+function buildDashboard(title: string, panels: PanelSpec[], defaultDatasource: NonNullable<PanelSpec["datasource"]>, rangeMinutes: number) {
   return {
     id: null,
     uid: `mosaic-${Date.now().toString(36)}`,
@@ -351,7 +351,7 @@ function buildDashboard(title: string, panels: PanelSpec[], datasourceUid: strin
       from: `now-${Math.max(1, Math.trunc(rangeMinutes))}m`,
       to: "now",
     },
-    panels: panels.map((panel, index) => buildPanel(panel, index, datasourceUid)),
+    panels: panels.map((panel, index) => buildPanel(panel, index, defaultDatasource)),
   };
 }
 
@@ -363,6 +363,10 @@ export function registerGrafanaTools(api: PluginApi) {
     const prometheusUrl = resolveString(api.pluginConfig, "prometheusUrl");
     const uiUrl = resolveString(api.pluginConfig, "uiUrl", DEFAULT_UI_URL);
     const datasourceUid = resolveString(api.pluginConfig, "datasourceUid");
+    const defaultDatasource = {
+      uid: datasourceUid,
+      type: resolveString(api.pluginConfig, "datasourceType", "prometheus") as NonNullable<PanelSpec["datasource"]>["type"],
+    };
     const headers = authHeaders(api.pluginConfig);
     const registerTool = (tool: Parameters<typeof api.registerTool>[0]) => {
       if (isToolEnabled(api.pluginConfig, tool.name)) {
@@ -395,7 +399,7 @@ export function registerGrafanaTools(api: PluginApi) {
     registerTool({
       name: "dashboard_create",
       label: "Create Dashboard",
-      description: "Create a Grafana dashboard after validating every Prometheus or Loki panel query, then open it in the UI Grafana tab.",
+      description: "Create a Grafana dashboard after validating every Prometheus, VictoriaMetrics or Loki panel query, then open it in the UI Grafana tab.",
       parameters: {
         type: "object",
         additionalProperties: false,
@@ -403,7 +407,7 @@ export function registerGrafanaTools(api: PluginApi) {
           title: { type: "string", description: "Dashboard title." },
           panels: {
             type: "array",
-            description: "Panel specs: title, query (PromQL or LogQL), datasource {type: prometheus or loki, uid: discovered Grafana UID}, visualization (timeseries, stat, gauge, table, logs), optional unit/legend/description. Omit datasource only to use the configured Prometheus default.",
+            description: "Panel specs: title, query (PromQL, MetricsQL or LogQL), datasource {type: prometheus, victoriametrics-metrics-datasource or loki, uid: discovered Grafana UID}, visualization (timeseries, stat, gauge, table, logs), optional unit/legend/description. Omit datasource only to use the configured metrics default.",
             items: { type: "object" },
           },
           rangeMinutes: { type: "number", description: "Dashboard time range in minutes. Default 60." },
@@ -415,12 +419,12 @@ export function registerGrafanaTools(api: PluginApi) {
         const title = stringParam(rawParams.title) || "Generated Dashboard";
         const panels = panelsFromParams(rawParams);
         if (panels.length === 0) {
-          throw new Error("Provide panels with explicit PromQL or LogQL queries.");
+          throw new Error("Provide panels with explicit PromQL, MetricsQL or LogQL queries.");
         }
 
         const requireData = boolParam(rawParams.requireData, true);
         const rangeMinutes = Math.max(1, Math.min(10080, numberParam(rawParams.rangeMinutes, 60)));
-        const validation = await validatePanels(prometheusUrl, grafanaUrl, uiUrl, headers, datasourceUid, panels, requireData, rangeMinutes);
+        const validation = await validatePanels(prometheusUrl, grafanaUrl, uiUrl, headers, defaultDatasource, panels, requireData, rangeMinutes);
         if (!validation.ok) {
           return jsonToolResult({
             created: false,
@@ -429,7 +433,7 @@ export function registerGrafanaTools(api: PluginApi) {
           });
         }
 
-        const dashboard = buildDashboard(title, panels, datasourceUid, rangeMinutes);
+        const dashboard = buildDashboard(title, panels, defaultDatasource, rangeMinutes);
         const createPayload = await fetchJson(`${grafanaUrl}/api/dashboards/db`, {
           method: "POST",
           headers,
diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts
index e82768ed..60cbbb06 100644
--- a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts
+++ b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/index.ts
@@ -167,7 +167,7 @@ async function queryRange(baseUrl: string, promql: string, rawParams: Record<str
 export default definePluginEntry({
   id: "observability",
   name: "Observability",
-  description: "Read-only Prometheus and Grafana tools for cluster observability.",
+  description: "Read-only Prometheus-compatible metrics and Grafana tools for cluster observability.",
   register(api) {
     registerGrafanaTools(api);
 
@@ -184,7 +184,7 @@ export default definePluginEntry({
     registerTool({
       name: "observability_metric_names",
       label: "Observability Metric Names",
-      description: "List available Prometheus metric names, optionally filtered by substring or regular expression.",
+      description: "List available Prometheus or VictoriaMetrics metric names, optionally filtered by substring or regular expression.",
       parameters: {
         type: "object",
         additionalProperties: false,
@@ -213,14 +213,14 @@ export default definePluginEntry({
 
     registerTool({
       name: "observability_query",
-      label: "Prometheus Instant Query",
-      description: "Run a read-only instant PromQL query against observability Prometheus.",
+      label: "Metrics Instant Query",
+      description: "Run a read-only instant PromQL or MetricsQL query against the configured metrics service.",
       parameters: {
         type: "object",
         additionalProperties: false,
         required: ["query"],
         properties: {
-          query: { type: "string", description: "PromQL instant query." },
+          query: { type: "string", description: "PromQL or MetricsQL instant query supported by the configured backend." },
           time: { type: "string", description: "Optional RFC3339 or Unix timestamp." },
           maxSeries: { type: "number", description: "Maximum returned series." },
         },
@@ -240,14 +240,14 @@ export default definePluginEntry({
 
     registerTool({
       name: "observability_range_query",
-      label: "Prometheus Range Query",
-      description: "Run a historical PromQL range query.",
+      label: "Metrics Range Query",
+      description: "Run a historical PromQL or MetricsQL range query.",
       parameters: {
         type: "object",
         additionalProperties: false,
         required: ["query"],
         properties: {
-          query: { type: "string", description: "PromQL range query." },
+          query: { type: "string", description: "PromQL or MetricsQL range query supported by the configured backend." },
           rangeMinutes: { type: "number", description: "Lookback window in minutes. Default 60." },
           stepSeconds: { type: "number", description: "Prometheus step in seconds. Default 60." },
           start: { type: "string", description: "Optional explicit start time." },
diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/openclaw.plugin.json b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/openclaw.plugin.json
index dbf6078d..b03b2a08 100644
--- a/helm/mosaic-stack/files/openclaw-seed/extensions/observability/openclaw.plugin.json
+++ b/helm/mosaic-stack/files/openclaw-seed/extensions/observability/openclaw.plugin.json
@@ -18,7 +18,7 @@
     "properties": {
       "prometheusUrl": {
         "type": "string",
-        "description": "Prometheus service base URL used for dashboard validation."
+        "description": "Prometheus-compatible query base URL, including any VictoriaMetrics cluster tenant prefix."
       },
       "grafanaEnabled": {
         "type": "boolean",
@@ -69,6 +69,14 @@
       "lokiTenantId": {
         "type": "string",
         "description": "Optional operator-configured Loki tenant."
+      },
+      "datasourceType": {
+        "type": "string",
+        "enum": [
+          "prometheus",
+          "victoriametrics-metrics-datasource"
+        ],
+        "description": "Grafana metrics datasource type. Defaults to prometheus."
       }
     }
   }
diff --git a/helm/mosaic-stack/files/openclaw-seed/nemoclaw.json b/helm/mosaic-stack/files/openclaw-seed/nemoclaw.json
index 47448820..0e94b506 100644
--- a/helm/mosaic-stack/files/openclaw-seed/nemoclaw.json
+++ b/helm/mosaic-stack/files/openclaw-seed/nemoclaw.json
@@ -323,6 +323,7 @@
         "config": {
           "grafanaUrl": {{ .Values.observability.grafanaUrl | quote }},
           "prometheusUrl": {{ .Values.observability.prometheusUrl | quote }},
+          "datasourceType": {{ .Values.observability.grafanaDatasourceType | quote }},
           "datasourceUid": {{ .Values.observability.grafanaDatasourceUid | quote }}{{- if .Values.observability.grafanaAuth.existingSecret }},
           "basicUser": "${MOSAIC_GRAFANA_BASIC_USER}",
           "basicPassword": "${MOSAIC_GRAFANA_BASIC_PASSWORD}"{{- end }}
@@ -336,6 +337,7 @@
           "grafanaEnabled": {{ .Values.modules.grafana.enabled }},
           "grafanaUrl": {{ .Values.observability.grafanaUrl | quote }},
           "prometheusUrl": {{ .Values.observability.prometheusUrl | quote }},
+          "datasourceType": {{ .Values.observability.grafanaDatasourceType | quote }},
           "datasourceUid": {{ .Values.observability.grafanaDatasourceUid | quote }}{{- if .Values.observability.grafanaAuth.existingSecret }},
           "basicUser": "${MOSAIC_GRAFANA_BASIC_USER}",
           "basicPassword": "${MOSAIC_GRAFANA_BASIC_PASSWORD}"{{- end }}
diff --git a/helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md b/helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md
index 24f159ba..84aafe4b 100644
--- a/helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md
+++ b/helm/mosaic-stack/files/openclaw-seed/skills/observability/SKILL.md
@@ -23,13 +23,19 @@ Prefer these tools over shell `curl` when answering questions about historical c
 
 ## Useful starting points
 
-- `observability_metric_names` lists available Prometheus metrics.
-- `observability_query` runs an instant PromQL query.
+- `observability_metric_names` lists available Prometheus or VictoriaMetrics metrics.
+- `observability_query` runs an instant PromQL or MetricsQL query supported by the configured backend.
 - `observability_range_query` runs a historical range query.
 - `dashboard_list` discovers Grafana datasource names, types, and UIDs and searches dashboards.
-- `dashboard_create` validates explicit panel PromQL or LogQL, creates a Grafana dashboard, and opens it in the UI Grafana tab.
+- `dashboard_create` validates explicit panel PromQL, MetricsQL or LogQL, creates a Grafana dashboard, and opens it in the UI Grafana tab.
 - `dashboard_open` opens an existing Grafana dashboard UID in the UI Grafana tab.
 
+For VictoriaMetrics, the same metrics tools use its Prometheus-compatible API.
+Use MetricsQL extensions only when the configured metrics backend is VictoriaMetrics.
+For dashboards, use the datasource UID and type returned by `dashboard_list`.
+VictoriaMetrics may use either `prometheus` or its native
+`victoriametrics-metrics-datasource` Grafana plugin. Preserve the discovered type.
+
 ## Example PromQL
 
 - `avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)`
diff --git a/helm/mosaic-stack/tests/chart-render.test.mjs b/helm/mosaic-stack/tests/chart-render.test.mjs
index a599df83..f8360b1e 100644
--- a/helm/mosaic-stack/tests/chart-render.test.mjs
+++ b/helm/mosaic-stack/tests/chart-render.test.mjs
@@ -1154,3 +1154,21 @@ test("configures optional Loki access without exposing its credential to UI or s
   const defaults = render("--show-only", "templates/openclaw.yaml");
   assert.doesNotMatch(defaults, /name: MOSAIC_LOKI_AUTHORIZATION/);
 });
+
+
+test("configures a tenant-scoped VictoriaMetrics backend and native Grafana datasource", () => {
+  const endpoint = "http://vmselect.metrics.svc:8481/select/17/prometheus";
+  const args = ["--set-string", `observability.prometheusUrl=${endpoint}`,
+    "--set-string", "observability.grafanaDatasourceUid=vm-metrics",
+    "--set-string", "observability.grafanaDatasourceType=victoriametrics-metrics-datasource"];
+  const seed = render(...args, "--show-only", "templates/openclaw-seed-configmap.yaml");
+  const config = JSON.parse(configMapBlock(seed, "openclaw.json"));
+  for (const name of ["observability", "grafana"]) {
+    assert.equal(config.plugins.entries[name].config.prometheusUrl, endpoint);
+    assert.equal(config.plugins.entries[name].config.datasourceUid, "vm-metrics");
+    assert.equal(config.plugins.entries[name].config.datasourceType, "victoriametrics-metrics-datasource");
+  }
+  for (const template of ["templates/openclaw.yaml", "templates/mosaic-ui.yaml"]) {
+    assert.ok(render(...args, "--show-only", template).includes(endpoint));
+  }
+});
diff --git a/helm/mosaic-stack/tests/mixed-dashboards.test.mjs b/helm/mosaic-stack/tests/mixed-dashboards.test.mjs
index 5e6abc98..0c59fccf 100644
--- a/helm/mosaic-stack/tests/mixed-dashboards.test.mjs
+++ b/helm/mosaic-stack/tests/mixed-dashboards.test.mjs
@@ -11,9 +11,9 @@ const hooks=registerHooks({resolve(s,c,next){return s==='openclaw/plugin-sdk/plu
 const {default:plugin}=await import('../files/openclaw-seed/extensions/grafana/index.ts');
 hooks.deregister();
 for(const [register,name] of [[registerGrafanaTools,'dashboard_create'],[plugin.register,'grafana_dashboard_create']]) {
- test(`${name} routes mixed panels to their datasources and rejects failed validation`,async t=>{
+ for (const metricsType of ['prometheus', 'victoriametrics-metrics-datasource']) test(`${name} (${metricsType}) routes mixed panels to their datasources and rejects failed validation`,async t=>{
   const saved=[];const queries=[];let fail=false;let empty=false;
-  const sources=[{uid:'metrics-test',type:'prometheus',name:'Metrics'},{uid:'logs-test',type:'loki',name:'Logs'}];
+  const sources=[{uid:'metrics-test',type:metricsType,name:'Metrics'},{uid:'logs-test',type:'loki',name:'Logs'}];
   const server=createServer(async(req,res)=>{
    let body='';for await(const part of req)body+=part;
    const data=body?JSON.parse(body):{};
@@ -30,13 +30,13 @@ for(const [register,name] of [[registerGrafanaTools,'dashboard_create'],[plugin.
   }).listen(0,'127.0.0.1');await new Promise(r=>server.once('listening',r));
   t.after(()=>{server.closeAllConnections();server.close()});
   const url=`http://127.0.0.1:${server.address().port}`;
-  const tools={};register({pluginConfig:{grafanaEnabled:true,grafanaUrl:url+'/monitoring',prometheusUrl:url,uiUrl:url,datasourceUid:'metrics-test'},registerTool:x=>tools[x.name]=x});
+  const tools={};register({pluginConfig:{grafanaEnabled:true,grafanaUrl:url+'/monitoring',prometheusUrl:url,uiUrl:url,datasourceUid:'metrics-test',...(metricsType === 'prometheus' ? {} : {datasourceType:metricsType})},registerTool:x=>tools[x.name]=x});
   const panels=[{title:'Health',query:'up',visualization:'stat'},{title:'Events',query:'{service="example"}',visualization:'logs',datasource:{uid:'logs-test',type:'loki'}}];
   const result=await tools[name].execute('test',{title:'Correlation',panels,openInUi:false});
   assert.equal(result.details.created,true);
   assert.ok(result.details.iframeUrl.startsWith('/api/grafana/proxy/d/test-board?'));
   assert.equal(result.details.dashboardUrl,url+'/monitoring/d/test-board');
-  assert.deepEqual(saved[0].panels.map(p=>p.datasource),[{type:'prometheus',uid:'metrics-test'},{type:'loki',uid:'logs-test'}]);
+  assert.deepEqual(saved[0].panels.map(p=>p.datasource),[{type:metricsType,uid:'metrics-test'},{type:'loki',uid:'logs-test'}]);
   assert.equal(saved[0].panels[1].type,'logs');
   assert.ok(queries.some(q=>q.datasource.uid==='logs-test'&&q.expr===panels[1].query));
   fail=true;
@@ -65,5 +65,11 @@ for(const [register,name] of [[registerGrafanaTools,'dashboard_create'],[plugin.
   assert.equal(recovered.created,true);
   assert.equal(saved.length,2);
   assert.equal(saved[1].panels[0].datasource.uid,datasource.uid);
+  const alternate={uid:'alternate-metrics',type:metricsType === 'prometheus' ? 'victoriametrics-metrics-datasource' : 'prometheus',name:'Alternate metrics'};
+  sources.push(alternate);
+  const explicit=(await tools[name].execute('test',{panels:[{...panels[0],datasource:alternate}],openInUi:false})).details;
+  assert.equal(explicit.created,true);
+  assert.equal(saved[2].panels[0].datasource.type,alternate.type);
+  assert.equal(queries.at(-1).datasource.uid,alternate.uid);
  });
 }
diff --git a/helm/mosaic-stack/tests/victoriametrics.test.mjs b/helm/mosaic-stack/tests/victoriametrics.test.mjs
new file mode 100644
index 00000000..f2765f8f
--- /dev/null
+++ b/helm/mosaic-stack/tests/victoriametrics.test.mjs
@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import assert from 'node:assert/strict';
+import {createServer} from 'node:http';
+import {registerHooks} from 'node:module';
+import test from 'node:test';
+
+const hooks = registerHooks({resolve(specifier, context, next) {
+  return specifier === 'openclaw/plugin-sdk/plugin-entry'
+    ? {url:'data:text/javascript,export const definePluginEntry = x => x;', shortCircuit:true}
+    : next(specifier, context);
+}});
+const {default:plugin} = await import('../files/openclaw-seed/extensions/observability/index.ts');
+hooks.deregister();
+
+function tools(base) {
+  const registered = {};
+  plugin.register({pluginConfig:{grafanaEnabled:false, prometheusUrl:base},
+    registerTool:tool => { registered[tool.name] = params => tool.execute('test', params); }});
+  return registered;
+}
+
+for (const prefix of ['', '/select/23/prometheus']) {
+  test(`metrics discovery and queries preserve the configured API prefix (${prefix || 'root'})`, async t => {
+    const requests = [];
+    const expression = 'WITH (load = vector(3)) load * 2';
+    const server = createServer((req,res) => {
+      const url = new URL(req.url, 'http://localhost');
+      requests.push({url,method:req.method});
+      const data = url.pathname.endsWith('/values') ? ['node_load1','node_memory_MemAvailable_bytes']
+        : url.pathname.endsWith('/query_range')
+          ? {resultType:'matrix',result:[{metric:{node:'worker-b'},values:[[1790812800,'6'],[1790812860,'8']]}]}
+          : {resultType:'vector',result:[{metric:{node:'worker-b'},value:[1790812800,'6']}]};
+      res.setHeader('Content-Type','application/json');
+      res.end(JSON.stringify({status:'success',data}));
+    }).listen(0,'127.0.0.1');
+    await new Promise(resolve => server.once('listening',resolve));
+    t.after(() => { server.closeAllConnections(); server.close(); });
+    const api = tools(`http://127.0.0.1:${server.address().port}${prefix}/`);
+    assert.equal((await api.observability_metric_names({})).details.names.length,2);
+    const instant = (await api.observability_query({query:expression})).details;
+    assert.deepEqual(instant.result[0],{metric:{node:'worker-b'},value:{timestamp:1790812800,value:6}});
+    const range = (await api.observability_range_query({query:expression,start:'2026-10-01T00:00:00Z',end:'2026-10-01T00:05:00Z',stepSeconds:60})).details;
+    assert.equal(range.result[0].values[1].value,8);
+    assert.deepEqual(requests.map(({url}) => url.pathname),[
+      `${prefix}/api/v1/label/__name__/values`, `${prefix}/api/v1/query`, `${prefix}/api/v1/query_range`,
+    ]);
+    for (const request of requests) assert.equal(request.method,'GET');
+    for (const {url} of requests.slice(1)) assert.equal(url.searchParams.get('query'),expression);
+    assert.equal(requests[2].url.searchParams.get('step'),'60');
+  });
+}
+
+test('VictoriaMetrics serves discovery, MetricsQL instant queries and historical queries', {
+  skip:!process.env.VICTORIAMETRICS_TEST_URL,
+}, async () => {
+  const api = tools(process.env.VICTORIAMETRICS_TEST_URL);
+  const names = (await api.observability_metric_names({})).details;
+  assert.ok(names.names.length > 0);
+  const query = 'WITH (x = vector(2)) x + 3';
+  const instant = (await api.observability_query({query})).details;
+  assert.equal(instant.resultType,'vector');
+  assert.equal(instant.result[0].value.value,5);
+  const range = (await api.observability_range_query({query,rangeMinutes:2,stepSeconds:30})).details;
+  assert.equal(range.resultType,'matrix');
+  assert.ok(range.result[0].values.length > 1);
+  assert.ok(range.result[0].values.every(sample => sample.value === 5));
+});
diff --git a/helm/mosaic-stack/values.yaml b/helm/mosaic-stack/values.yaml
index 8aeb523e..1b219688 100644
--- a/helm/mosaic-stack/values.yaml
+++ b/helm/mosaic-stack/values.yaml
@@ -17,10 +17,12 @@ namespace:
   labels: {}
 
 observability:
+  # Prometheus-compatible query URL; VictoriaMetrics cluster URLs include /select/<tenant>/prometheus.
   prometheusUrl: http://kube-prometheus-stack-prometheus.prometheus.svc.cluster.local:9090
   grafanaUrl: http://kube-prometheus-stack-grafana.prometheus.svc.cluster.local/grafana
   alertmanagerUrl: http://kube-prometheus-stack-alertmanager.prometheus.svc.cluster.local:9093
   grafanaDatasourceUid: prometheus
+  grafanaDatasourceType: prometheus
   loki:
     url: ""
     tenantId: ""
diff --git a/index.md b/index.md
index 5605c925..f5b2716a 100644
--- a/index.md
+++ b/index.md
@@ -44,6 +44,7 @@ Security model <docs/security_model>
 Tenant and operator deployments <docs/multitenancy>
 Use cases <docs/use_cases>
 Slurm root cause analysis <docs/slurm_rca>
+VictoriaMetrics <docs/victoriametrics>
 Loki query reference <docs/loki>
 Headless integration <docs/skills/ai-factory-operations-agent-headless/SKILL>
 ```

From 1105c32a65de1ce3752d17a5fb61b717d9dea2cb Mon Sep 17 00:00:00 2001
From: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
Date: Fri, 16 Oct 2026 12:54:06 -0700
Subject: [PATCH 2/6] Integrate VictoriaMetrics into observability installation
 steps

Consolidate endpoint selection and Grafana datasource configuration into the existing installation workflow and remove the standalone guide.

Signed-off-by: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
---
 docs/installation.md    | 37 ++++++++++++++++++--
 docs/victoriametrics.md | 77 -----------------------------------------
 index.md                |  1 -
 3 files changed, 35 insertions(+), 80 deletions(-)
 delete mode 100644 docs/victoriametrics.md

diff --git a/docs/installation.md b/docs/installation.md
index e2759c4f..7f460139 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -202,7 +202,19 @@ All module changes below update an existing AI Factory Operations Agent release
 
 ### Observability
 
-To connect AI Factory Operations Agent to an existing Prometheus-compatible metrics service, run:
+Connect to an existing Prometheus or VictoriaMetrics service with
+`observability.prometheusUrl`. Choose the query base URL for your backend:
+
+| Backend | Example base URL |
+| --- | --- |
+| Prometheus | `http://kube-prometheus-stack-prometheus.prometheus.svc.cluster.local:9090` |
+| Single-node VictoriaMetrics | `http://victoriametrics.monitoring.svc.cluster.local:8428` |
+| VictoriaMetrics cluster | `http://vmselect.monitoring.svc.cluster.local:8481/select/0/prometheus` |
+
+For a VictoriaMetrics cluster, replace `0` with the intended tenant ID and use
+`vmselect`. Include any proxy prefix and omit `/api/v1`; the tools append API
+paths themselves. See the [VictoriaMetrics API examples](https://docs.victoriametrics.com/url-examples.html).
+Set `PROMETHEUS_URL` to your chosen endpoint, then run:
 
 ```bash
 export PROMETHEUS_URL='http://kube-prometheus-stack-prometheus.prometheus.svc.cluster.local:9090'
@@ -216,7 +228,13 @@ helm upgrade mosaic "$MOSAIC_CHART" \
   --timeout 12m
 ```
 
-VictoriaMetrics uses the same endpoint setting. See [VictoriaMetrics metrics and dashboards](victoriametrics.md) for single-node and cluster URLs and native Grafana datasource configuration.
+The endpoint must be reachable from the agent pod. Grafana credentials authenticate
+Grafana requests, not direct metrics queries. Keep credentials out of URLs and
+committed values. Mosaic connects to your metrics backend; it does not install it.
+
+Verify access by asking Mosaic to list metric names, query a discovered metric,
+and show its history for the past hour. VictoriaMetrics also supports MetricsQL;
+see its [PromQL differences and extensions](https://docs.victoriametrics.com/victoriametrics/metricsql/).
 
 ### Read-only Loki logs
 
@@ -262,6 +280,7 @@ To connect AI Factory Operations Agent to an existing Grafana service, run:
 ```bash
 export GRAFANA_URL='http://kube-prometheus-stack-grafana.prometheus.svc.cluster.local/grafana'
 export GRAFANA_DATASOURCE_UID='prometheus'
+export GRAFANA_DATASOURCE_TYPE='prometheus'
 export GRAFANA_USERNAME='admin'
 read -rsp 'Grafana password: ' GRAFANA_PASSWORD; echo
 
@@ -277,6 +296,7 @@ helm upgrade mosaic "$MOSAIC_CHART" \
   --set modules.grafana.enabled=true \
   --set-string observability.grafanaUrl="$GRAFANA_URL" \
   --set-string observability.grafanaDatasourceUid="$GRAFANA_DATASOURCE_UID" \
+  --set-string observability.grafanaDatasourceType="$GRAFANA_DATASOURCE_TYPE" \
   --set observability.grafanaAuth.existingSecret=mosaic-grafana-auth \
   --wait \
   --timeout 12m
@@ -286,6 +306,19 @@ unset GRAFANA_PASSWORD
 
 `grafanaUrl` includes Grafana's configured serving path. Use the service root for a root-served Grafana or `/grafana` for the NMC deployment.
 
+Use the datasource's actual UID. For VictoriaMetrics, configure Grafana to use the
+same backend and tenant as `observability.prometheusUrl`. Grafana's Prometheus
+datasource works with the default `GRAFANA_DATASOURCE_TYPE=prometheus`. For the
+[native VictoriaMetrics plugin](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/datasource/),
+have the Grafana administrator install and configure it, then set
+`GRAFANA_DATASOURCE_TYPE=victoriametrics-metrics-datasource` before running the
+commands above.
+
+Ask Mosaic to discover Grafana datasources before creating a dashboard. It
+preserves the selected UID and type in panels and query validation. An explicit
+panel datasource overrides the configured default, and Loki panels can share a
+dashboard with either metrics datasource type.
+
 ### Terminal
 
 To enable the optional browser terminal service, run:
diff --git a/docs/victoriametrics.md b/docs/victoriametrics.md
deleted file mode 100644
index f7b68864..00000000
--- a/docs/victoriametrics.md
+++ /dev/null
@@ -1,77 +0,0 @@
-<!-- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
-<!-- SPDX-License-Identifier: Apache-2.0 -->
-
-# VictoriaMetrics
-
-Connect the observability component to an existing VictoriaMetrics service through
-its Prometheus-compatible API. The metric discovery, instant query and historical
-query tools support PromQL and MetricsQL. Mosaic does not install VictoriaMetrics.
-
-## Metrics endpoint
-
-Set `observability.prometheusUrl` to the query base URL. This setting accepts
-Prometheus-compatible services, including VictoriaMetrics; no backend selector is
-required.
-
-| Deployment | Example base URL |
-| --- | --- |
-| Single-node VictoriaMetrics | `http://victoriametrics.monitoring.svc.cluster.local:8428` |
-| VictoriaMetrics cluster | `http://vmselect.monitoring.svc.cluster.local:8481/select/0/prometheus` |
-
-For a cluster, replace `0` with the intended tenant ID. Use the read endpoint
-(`vmselect`), rather than `vminsert` or `vmagent`. Include the tenant path and any
-proxy prefix, and omit `/api/v1`; the tools append the API paths themselves.
-See the [VictoriaMetrics API examples](https://docs.victoriametrics.com/url-examples.html).
-
-Use these values with your existing installation configuration:
-
-```yaml
-modules:
-  observability:
-    enabled: true
-observability:
-  prometheusUrl: http://vmselect.monitoring.svc.cluster.local:8481/select/0/prometheus
-```
-
-The endpoint must be reachable from the agent pod. Grafana credentials authenticate
-Grafana requests, not direct metrics queries. Do not put credentials in the URL or
-committed values.
-
-## Grafana dashboards
-
-VictoriaMetrics works with Grafana's Prometheus datasource and with the native
-VictoriaMetrics datasource plugin. Configure Grafana to query the same backend and
-tenant as `observability.prometheusUrl`, then use the datasource's actual UID.
-
-For an existing Prometheus datasource, keep the default type:
-
-```yaml
-observability:
-  grafanaDatasourceUid: metrics
-  grafanaDatasourceType: prometheus
-```
-
-For Grafana's native VictoriaMetrics plugin:
-
-```yaml
-observability:
-  grafanaDatasourceUid: victoria-metrics
-  grafanaDatasourceType: victoriametrics-metrics-datasource
-```
-
-The Grafana administrator installs and configures the plugin; see the
-[VictoriaMetrics Grafana integration](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/datasource/).
-Mosaic preserves this datasource type in dashboard panels, direct Grafana query
-validation and frontend proxy validation. An explicit panel datasource overrides
-the configured default. Loki panels can share a dashboard with either metrics
-datasource type.
-
-## Verify access
-
-Ask Mosaic to list the available metric names, query a discovered metric, and
-show its history for the past hour. Ask it to discover Grafana datasources before
-creating a dashboard. Use the discovered UID and type for each panel.
-
-MetricsQL extensions require a VictoriaMetrics backend. PromQL queries remain
-supported, with the differences described in the
-[MetricsQL documentation](https://docs.victoriametrics.com/victoriametrics/metricsql/).
diff --git a/index.md b/index.md
index f5b2716a..5605c925 100644
--- a/index.md
+++ b/index.md
@@ -44,7 +44,6 @@ Security model <docs/security_model>
 Tenant and operator deployments <docs/multitenancy>
 Use cases <docs/use_cases>
 Slurm root cause analysis <docs/slurm_rca>
-VictoriaMetrics <docs/victoriametrics>
 Loki query reference <docs/loki>
 Headless integration <docs/skills/ai-factory-operations-agent-headless/SKILL>
 ```

From bf88bd4601b15f1f64023e72605abf6fcb7cfb0d Mon Sep 17 00:00:00 2001
From: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
Date: Fri, 16 Oct 2026 12:55:12 -0700
Subject: [PATCH 3/6] Keep VictoriaMetrics installation guidance limited to
 configuration

Remove usage advice and implementation details, and replace the unnecessary shared-backend instruction with the native Grafana plugin UID and type settings.

Signed-off-by: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
---
 docs/installation.md | 21 +++++----------------
 1 file changed, 5 insertions(+), 16 deletions(-)

diff --git a/docs/installation.md b/docs/installation.md
index 7f460139..2a6bd841 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -232,10 +232,6 @@ The endpoint must be reachable from the agent pod. Grafana credentials authentic
 Grafana requests, not direct metrics queries. Keep credentials out of URLs and
 committed values. Mosaic connects to your metrics backend; it does not install it.
 
-Verify access by asking Mosaic to list metric names, query a discovered metric,
-and show its history for the past hour. VictoriaMetrics also supports MetricsQL;
-see its [PromQL differences and extensions](https://docs.victoriametrics.com/victoriametrics/metricsql/).
-
 ### Read-only Loki logs
 
 To connect Mosaic to an existing Loki service, run:
@@ -306,18 +302,11 @@ unset GRAFANA_PASSWORD
 
 `grafanaUrl` includes Grafana's configured serving path. Use the service root for a root-served Grafana or `/grafana` for the NMC deployment.
 
-Use the datasource's actual UID. For VictoriaMetrics, configure Grafana to use the
-same backend and tenant as `observability.prometheusUrl`. Grafana's Prometheus
-datasource works with the default `GRAFANA_DATASOURCE_TYPE=prometheus`. For the
-[native VictoriaMetrics plugin](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/datasource/),
-have the Grafana administrator install and configure it, then set
-`GRAFANA_DATASOURCE_TYPE=victoriametrics-metrics-datasource` before running the
-commands above.
-
-Ask Mosaic to discover Grafana datasources before creating a dashboard. It
-preserves the selected UID and type in panels and query validation. An explicit
-panel datasource overrides the configured default, and Loki panels can share a
-dashboard with either metrics datasource type.
+For Grafana's [native VictoriaMetrics plugin](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/datasource/),
+set `GRAFANA_DATASOURCE_UID` to the configured datasource UID and
+`GRAFANA_DATASOURCE_TYPE=victoriametrics-metrics-datasource` in the commands above.
+The plugin must already be installed in Grafana. A Prometheus datasource pointing
+to VictoriaMetrics uses the default type, `prometheus`.
 
 ### Terminal
 

From e38292b9c038484cf999c412f61e33ca8a8ee3ff Mon Sep 17 00:00:00 2001
From: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
Date: Fri, 16 Oct 2026 12:57:48 -0700
Subject: [PATCH 4/6] Trim VictoriaMetrics installation commentary

Remove redundant reachability and credential reminders and keep endpoint and Grafana settings concise.

Signed-off-by: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
---
 docs/installation.md | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/docs/installation.md b/docs/installation.md
index 2a6bd841..0ff85bb3 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -211,10 +211,8 @@ Connect to an existing Prometheus or VictoriaMetrics service with
 | Single-node VictoriaMetrics | `http://victoriametrics.monitoring.svc.cluster.local:8428` |
 | VictoriaMetrics cluster | `http://vmselect.monitoring.svc.cluster.local:8481/select/0/prometheus` |
 
-For a VictoriaMetrics cluster, replace `0` with the intended tenant ID and use
-`vmselect`. Include any proxy prefix and omit `/api/v1`; the tools append API
-paths themselves. See the [VictoriaMetrics API examples](https://docs.victoriametrics.com/url-examples.html).
-Set `PROMETHEUS_URL` to your chosen endpoint, then run:
+Replace `0` with your tenant ID for a VictoriaMetrics cluster. Include any proxy
+prefix and omit `/api/v1`. Set `PROMETHEUS_URL` to your endpoint:
 
 ```bash
 export PROMETHEUS_URL='http://kube-prometheus-stack-prometheus.prometheus.svc.cluster.local:9090'
@@ -228,10 +226,6 @@ helm upgrade mosaic "$MOSAIC_CHART" \
   --timeout 12m
 ```
 
-The endpoint must be reachable from the agent pod. Grafana credentials authenticate
-Grafana requests, not direct metrics queries. Keep credentials out of URLs and
-committed values. Mosaic connects to your metrics backend; it does not install it.
-
 ### Read-only Loki logs
 
 To connect Mosaic to an existing Loki service, run:
@@ -302,11 +296,10 @@ unset GRAFANA_PASSWORD
 
 `grafanaUrl` includes Grafana's configured serving path. Use the service root for a root-served Grafana or `/grafana` for the NMC deployment.
 
-For Grafana's [native VictoriaMetrics plugin](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/datasource/),
-set `GRAFANA_DATASOURCE_UID` to the configured datasource UID and
-`GRAFANA_DATASOURCE_TYPE=victoriametrics-metrics-datasource` in the commands above.
-The plugin must already be installed in Grafana. A Prometheus datasource pointing
-to VictoriaMetrics uses the default type, `prometheus`.
+For an installed [native VictoriaMetrics datasource](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/datasource/),
+set `GRAFANA_DATASOURCE_UID` to its UID and
+`GRAFANA_DATASOURCE_TYPE=victoriametrics-metrics-datasource`.
+For a Prometheus datasource, keep `prometheus`.
 
 ### Terminal
 

From 98ff23b70cf1306c0897c2901256c5aec2fb2ee4 Mon Sep 17 00:00:00 2001
From: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
Date: Fri, 16 Oct 2026 13:10:46 -0700
Subject: [PATCH 5/6] Expose Grafana datasource type in the installation
 builder

Keep Prometheus as the default and carry an explicit datasource type into generated Helm commands. Cover both default and native VictoriaMetrics selections.

Signed-off-by: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
---
 docs/_static/install-builder.mjs    | 5 +++--
 tests/docs/install-builder.test.mjs | 6 ++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/_static/install-builder.mjs b/docs/_static/install-builder.mjs
index ddbb41ce..416ae3e5 100644
--- a/docs/_static/install-builder.mjs
+++ b/docs/_static/install-builder.mjs
@@ -20,7 +20,7 @@ const fields = {
   lokiSecret: ['Loki Authorization Secret (optional)', ''],
   lokiKey: ['Loki Secret key', 'authorization'],
   grafanaUrl: ['Grafana URL, including serving path', '', 'https://grafana.example.com/grafana'],
-  grafanaUid: ['Prometheus datasource UID', 'prometheus'], grafanaSecret: ['Grafana Secret (username, password)', 'mosaic-grafana-auth'],
+  grafanaUid: ['Metrics datasource UID', 'prometheus'], grafanaType: ['Metrics datasource type', 'prometheus'], grafanaSecret: ['Grafana Secret (username, password)', 'mosaic-grafana-auth'],
   bcmHead: ['BCM SSH head host', '', 'user@bcm-head.example.com'], bcmSecret: ['BCM SSH Secret (id_ecdsa)', 'bcm-host-ssh-key'],
   evidenceRoot: ['Slurm evidence directory on the collector node', '', '/slurm'],
   evidenceNode: ['Slurm collector node hostname', '', 'slurm-login-node'],
@@ -57,6 +57,7 @@ export function buildCommand(state) {
   }
   if (enabled.has('grafana')) {
     set('observability.grafanaUrl', value('grafanaUrl'), true); set('observability.grafanaDatasourceUid', value('grafanaUid'), true);
+    set('observability.grafanaDatasourceType', value('grafanaType'), true);
     set('observability.grafanaAuth.existingSecret', value('grafanaSecret'), true);
   }
   if (enabled.has('bcm')) {
@@ -140,7 +141,7 @@ if (root) {
       if (enabled.has('prometheus')) keys.push('prometheusUrl');
       if (enabled.has('alertmanager')) keys.push('alertmanagerUrl');
       if (enabled.has('loki')) keys.push('lokiUrl', 'lokiTenant', 'lokiSecret', 'lokiKey');
-      if (enabled.has('grafana')) keys.push('grafanaUrl', 'grafanaUid', 'grafanaSecret');
+      if (enabled.has('grafana')) keys.push('grafanaUrl', 'grafanaUid', 'grafanaType', 'grafanaSecret');
       if (enabled.has('slurm')) settingsBody.append(select('slurmBackend', 'Slurm integration', [['bcm', 'Through BCM'], ['vanilla', 'Standalone Slurm']]));
       if (enabled.has('bcm')) keys.push('bcmHead', 'bcmSecret');
       if (enabled.has('slurm') && state.slurmBackend === 'vanilla') keys.push('evidenceRoot', 'evidenceNode');
diff --git a/tests/docs/install-builder.test.mjs b/tests/docs/install-builder.test.mjs
index 6ab7ff4d..35730bbe 100644
--- a/tests/docs/install-builder.test.mjs
+++ b/tests/docs/install-builder.test.mjs
@@ -155,3 +155,9 @@ print(json.dumps({'catalog':catalog,'cases':cases}))`;
   }]));
   for (const {state, components} of cases) assert.deepEqual(normalize(selectedComponents(catalog, state, enabledModules(state))), normalize(components));
 });
+
+for (const grafanaType of [undefined, 'victoriametrics-metrics-datasource']) test(`Grafana datasource type preserves the default or configured selection: ${grafanaType}`, () => {
+  const args = argumentsFor({...example, modules:['grafana'], grafanaType, grafanaUid:'metrics'});
+  assert.ok(args.includes(`observability.grafanaDatasourceType=${grafanaType || 'prometheus'}`));
+  assert.ok(args.includes('observability.grafanaDatasourceUid=metrics'));
+});

From 0f6c279264fdde170cf430c83b5219908803aee5 Mon Sep 17 00:00:00 2001
From: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
Date: Fri, 16 Oct 2026 13:18:38 -0700
Subject: [PATCH 6/6] Select metrics datasource type from a dropdown

Offer Prometheus and native VictoriaMetrics choices with Prometheus selected initially; preserve the chosen type when other settings rerender.

Signed-off-by: Rohan Adwankar <39285979+RohanAdwankar@users.noreply.github.com>
---
 docs/_static/install-builder.mjs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/_static/install-builder.mjs b/docs/_static/install-builder.mjs
index 416ae3e5..dd3daf79 100644
--- a/docs/_static/install-builder.mjs
+++ b/docs/_static/install-builder.mjs
@@ -93,7 +93,7 @@ export function buildCommand(state) {
 
 const root = typeof document === 'undefined' ? null : document.querySelector('#install-command-builder');
 if (root) {
-  const state = {inference: 'external', channel: 'devel', slurmBackend: 'bcm', modules: ['kubernetes']};
+  const state = {inference: 'external', channel: 'devel', slurmBackend: 'bcm', grafanaType: 'prometheus', modules: ['kubernetes']};
   const form = document.createElement('form'); form.addEventListener('submit', event => event.preventDefault());
   const options = document.createElement('fieldset'); options.innerHTML = '<legend>Installation options</legend>';
   function select(key, label, choices) {
@@ -147,6 +147,10 @@ if (root) {
       if (enabled.has('slurm') && state.slurmBackend === 'vanilla') keys.push('evidenceRoot', 'evidenceNode');
       if (enabled.has('research')) keys.push('corpus');
       for (const key of keys) {
+        if (key === 'grafanaType') {
+          settingsBody.append(select(key, fields[key][0], [['prometheus', 'Prometheus'], ['victoriametrics-metrics-datasource', 'VictoriaMetrics (native plugin)']]));
+          continue;
+        }
         const [label, initial, placeholder] = fields[key]; const wrapper = document.createElement('label'); wrapper.textContent = label;
         const input = document.createElement('input'); input.name = key; input.value = state[key] ?? initial; input.placeholder = placeholder || '';
         input.addEventListener('input', () => {state[key] = input.value; update(false);}); wrapper.append(input); settingsBody.append(wrapper);