Capability statement versions pulled from metadata

README.md

@@ -37,16 +37,22 @@ git submodule update --init --recursive
 
 Configuration settings are set in a JSON configuration file. The file `conf/localhost.json` provides a sample configuration.
 
+The authoritative definition of the configuration is the JSON Schema at
+[`assets/schema/cql-test-configuration.schema.json`](assets/schema/cql-test-configuration.schema.json).
+Configuration files are validated against it at load time, and it is also served over MCP as the
+`cql-test-configuration-schema` resource. The reference table below explains what each setting
+actually does; the schema is the place to look for the exact types and required fields.
+
 ```json
 {
   "FhirServer": {
     "BaseUrl": "https://fhirServerBaseUrl",
     "CqlOperation": "$cql"
   },
   "Build": {
-    "CqlFileVersion": "1.0.000",
+    "CqlFileVersion": "1.0.0",
     "CqlOutputPath": "./cql",
-    "testsRunDescription": '',
+    "CqlVersion": "1.5",
     "testsRunDescription": "Local host test run",
     "cqlTranslator": "Java CQFramework Translator",
     "cqlTranslatorVersion": "Unknown",
@@ -63,6 +69,103 @@ Configuration settings are set in a JSON configuration file. The file `conf/loca
 }
 ```
 
+#### Configuration Reference
+
+`FhirServer` — where and how tests are evaluated.
+
+| Setting | Required | Default | What it does |
+| --- | --- | --- | --- |
+| `BaseUrl` | yes | `https://cloud.alphora.com/sandbox/r4/cds/fhir` | Base URL of the FHIR server under test. A trailing slash is stripped on load. Combined with `CqlOperation` to form the request URL: `<BaseUrl>/$cql` or `<BaseUrl>/Library/$evaluate`. |
+| `CqlOperation` | yes | `$cql` | Which operation evaluates the tests. One of `$cql` (system-level) or `$evaluate` (`Library/$evaluate`). |
+| `ogBaseUrl` | no | — | Accepted by the schema and set in `conf/smile-cdr-local.json`, but **not read by any code**. Retained for backward compatibility; setting it has no effect. |
+
+`Build` — CQL generation settings, plus provenance recorded in the results report.
+
+| Setting | Required | Default | What it does |
+| --- | --- | --- | --- |
+| `CqlFileVersion` | yes | `1.0.0` | The version literal written into the `library` declaration of each `.cql` file produced by the `build-cql` command — `CqlFileVersion: "1.0.0"` yields `library CqlAggregateTest version '1.0.0'`. Use plain semantic versioning. It has **no effect on running tests**, and is unrelated to the CQL language version (see `CqlVersion`). Only `build-cql` reads it. |
+| `CqlOutputPath` | yes | `./cql` | Directory `build-cql` writes generated `.cql` files to. The `build-cql` CLI output argument takes precedence. |
+| `CqlVersion` | no | `1.5` | The CQL language version the target engine implements. Drives version gating: tests carrying a `version`/`versionTo` outside this are skipped rather than run (for example the CQL 2.0 `Slice` tests are skipped against a 1.5 engine). Used unless `CapabilityVersionExtensions` is configured and the server declares a CQL version — see [Engine and version reporting](#engine-and-version-reporting). |
+| `testsRunDescription` | no | — | Free-text label for the run, copied into the results report. |
+| `cqlTranslator` | no | `Unknown` | Name of the translator under test. Recorded in the results report only. |
+| `cqlTranslatorVersion` | no | `Unknown` | Version of the translator. Recorded in the results report only. |
+| `cqlEngine` | no | `Unknown` | Name of the engine under test. Recorded in the results report only. |
+| `cqlEngineVersion` | no | `Unknown` | Version of the engine. Recorded in the results report only. |
+| `CapabilityVersionExtensions` | no | — | Extension urls under which the target server publishes the five values above, so they can be read from its `CapabilityStatement` instead of taken from here. No such extension is registered, so there is no default and detection is off unless configured — see [Engine and version reporting](#engine-and-version-reporting). |
+
+##### Engine and version reporting
+
+At the start of a run the runner reads the target server's `CapabilityStatement` from
+`{BaseUrl}/metadata`. Where the server declares a value about its own CQL implementation, that is
+preferred; anything it does not declare falls back to the configuration file, so these settings
+remain the effective source for most servers. Which source was used is logged.
+
+`CqlVersion` is the one with behavioural consequence — it drives version gating. The other four
+(`cqlTranslator`, `cqlTranslatorVersion`, `cqlEngine`, `cqlEngineVersion`) are provenance recorded
+in the results report. A wrong configured value is not detected, so keep them accurate for the
+server you are pointing at.
+
+**In practice, expect the fallback.** A `CapabilityStatement` has no element for a CQL language,
+translator or engine version, and **no extension for them is registered** — HL7's guidance is that a
+server documents its expression-language capabilities in the statement's narrative text rather than
+computably. Because there is no url to rely on, the runner guesses at none. Version detection is
+therefore opt-in: point `Build.CapabilityVersionExtensions` at the extension urls your server
+actually publishes.
+
+```jsonc
+"Build": {
+  "CqlVersion": "1.5",
+  "cqlEngineVersion": "4.1.0",
+  // Only needed for a server that publishes these; omit it otherwise.
+  "CapabilityVersionExtensions": {
+    "cqlVersion": "http://example.org/fhir/StructureDefinition/our-cql-version",
+    "cqlEngineVersion": "http://example.org/fhir/StructureDefinition/our-engine-version"
+  }
+}
+```
+
+Each key is optional, and any of `cqlVersion`, `cqlEngine`, `cqlEngineVersion`, `cqlTranslator` and
+`cqlTranslatorVersion` may be given a url. A url is looked for on the `CapabilityStatement` itself
+and on its `rest` entry, taking `valueString` or `valueCode`; a field with no configured url is never
+looked up. With the setting omitted the run logs
+
+```
+No Build.CapabilityVersionExtensions configured; using the configured version values.
+```
+
+which is the expected outcome against current servers. HAPI FHIR 8.10.0, for example, reports only
+
+```json
+{ "fhirVersion": "4.0.1",
+  "software": { "name": "HAPI FHIR Server", "version": "8.10.0" } }
+```
+
+`software.name`/`software.version` are reported separately as **FHIR server** provenance and are
+deliberately never used as the CQL engine's identity: the FHIR server (HAPI 8.10.0) is a different
+product from the CQL engine it hosts (for example the CQF engine 4.1.0). Substituting one for the
+other would record a wrong engine version and, if applied to `CqlVersion`, change which tests are
+gated.
+
+What a `CapabilityStatement` *does* declare reliably is which operations a server supports, so the
+runner also checks that the configured `CqlOperation` is advertised — the system-level `$cql`
+operation, or `$evaluate` on the `Library` resource — and warns before running roughly 1800 tests
+against a server that does not offer it. A server that cannot be reached, or that declares no
+operations at all, is not treated as a failure: the run proceeds on the configured values.
+
+`Tests` — which tests run and where results land.
+
+| Setting | Required | Default | What it does |
+| --- | --- | --- | --- |
+| `ResultsPath` | yes | `./results` | Directory the results JSON is written to. The CLI output argument takes precedence. |
+| `SkipList` | yes | `[]` | Tests to skip, each `{ testsName, groupName, testName, reason }`. Skipped tests appear in the results with status `skip` and the given reason. See below. |
+| `OnlyList` | no | `[]` | If non-empty, only these tests run and all others are skipped, each `{ testsName, groupName, testName }`. See below. |
+
+`Debug` — development aids.
+
+| Setting | Required | Default | What it does |
+| --- | --- | --- | --- |
+| `QuickTest` | yes | `false` | Smoke-test mode. When `true`, loading stops after the **first group of the first test file**, so only a handful of tests run. Use it to check connectivity to a server, not to assess conformance. Equivalent to the `--quick` CLI flag. |
+
 To skip tests, add entries to the `SkipList` with the corresponding `testsName`, `groupName`, `testName`, and `reason`.
 
 ```jsonc
@@ -192,9 +295,34 @@ export CQL_OPERATION=$cql
 export CQL_TESTS_PATH=cql-tests/tests/cql
 ```
 
-`CQL_TESTS_PATH` sets the directory the test loader reads test XML from, defaulting to
-`cql-tests/tests/cql`. Point it at another directory in the `cql-tests` submodule — for
-example `cql-tests/tests/connectathonTests` — to run a different suite without editing a
+Every environment variable below takes precedence over the corresponding value in the
+configuration file. An unset variable falls back to the file, and then to the default listed in
+the [Configuration Reference](#configuration-reference).
+
+| Environment variable | Overrides |
+| --- | --- |
+| `SERVER_BASE_URL` | `FhirServer.BaseUrl` |
+| `CQL_OPERATION` | `FhirServer.CqlOperation` |
+| `CQL_FILE_VERSION` | `Build.CqlFileVersion` |
+| `CQL_OUTPUT_PATH` | `Build.CqlOutputPath` |
+| `CQL_VERSION` | `Build.CqlVersion` |
+| `TESTS_RUN_DESCRIPTION` | `Build.testsRunDescription` |
+| `CQL_TRANSLATOR` | `Build.cqlTranslator` |
+| `CQL_TRANSLATOR_VERSION` | `Build.cqlTranslatorVersion` |
+| `CQL_ENGINE` | `Build.cqlEngine` |
+| `CQL_ENGINE_VERSION` | `Build.cqlEngineVersion` |
+| `RESULTS_PATH` | `Tests.ResultsPath` |
+| `SKIP_LIST` | `Tests.SkipList` — a JSON array string. If it fails to parse, a warning is logged and the value in the configuration file is used. |
+| `ONLY_LIST` | `Tests.OnlyList` — a JSON array string, same parse-failure behaviour as `SKIP_LIST`. |
+| `QUICK_TEST` | `Debug.QuickTest` — only the exact string `true` enables it; any other value is `false`. |
+
+There is no environment variable for `FhirServer.ogBaseUrl`, which is unused (see the
+[Configuration Reference](#configuration-reference)).
+
+`CQL_TESTS_PATH` is separate from the configuration file — it is read directly by the test loader
+and has no configuration-file equivalent. It sets the directory the loader reads test XML from,
+defaulting to `cql-tests/tests/cql`. Point it at another directory in the `cql-tests` submodule —
+for example `cql-tests/tests/connectathonTests` — to run a different suite without editing a
 configuration file.
 
 ### Development Environment

assets/schema/cql-test-configuration.schema.json

@@ -65,6 +65,38 @@
         "cqlEngineVersion": {
           "type": "string",
           "description": "Which version of the CQL engine is used in this test run"
+        },
+        "CapabilityVersionExtensions": {
+          "type": "object",
+          "description": "Extension urls under which the target server publishes its CQL implementation versions, read from its CapabilityStatement in preference to the values above. No extension for these is registered, so there is no default: omit this to use the configured values as-is.",
+          "properties": {
+            "cqlVersion": {
+              "type": "string",
+              "format": "uri",
+              "description": "Extension url carrying the CQL language version"
+            },
+            "cqlEngine": {
+              "type": "string",
+              "format": "uri",
+              "description": "Extension url carrying the CQL engine name"
+            },
+            "cqlEngineVersion": {
+              "type": "string",
+              "format": "uri",
+              "description": "Extension url carrying the CQL engine version"
+            },
+            "cqlTranslator": {
+              "type": "string",
+              "format": "uri",
+              "description": "Extension url carrying the CQL translator name"
+            },
+            "cqlTranslatorVersion": {
+              "type": "string",
+              "format": "uri",
+              "description": "Extension url carrying the CQL translator version"
+            }
+          },
+          "additionalProperties": false
         }
       },
       "required": ["CqlFileVersion", "CqlOutputPath"],

conf/cql-execution-local.json

@@ -4,7 +4,7 @@
 		"CqlOperation": "$cql"
 	},
 	"Build": {
-		"CqlFileVersion": "1.0.000",
+		"CqlFileVersion": "1.0.0",
 		"CqlOutputPath": "./cql",
 		"CqlVersion": "1.5",
 		"testsRunDescription": "JavaScript CQL Execution Framework local test run. See: https://github.com/cqframework/cql-execution",

conf/development.json

@@ -4,7 +4,7 @@
     "CqlOperation": "$cql"
   },
   "Build": {
-    "CqlFileVersion": "1.0.000",
+    "CqlFileVersion": "1.0.0",
     "CqlOutputPath": "./cql",
     "CqlVersion": "1.5",
     "testsRunDescription": "Development test run",

conf/localhost-evaluate.json

@@ -4,7 +4,7 @@
     "CqlOperation": "$evaluate"
   },
   "Build": {
-    "CqlFileVersion": "1.0.000",
+    "CqlFileVersion": "1.0.0",
     "CqlOutputPath": "./cql",
     "CqlVersion": "1.5",
     "testsRunDescription": "Local host test run via Library/$evaluate",

conf/localhost.json

@@ -4,7 +4,7 @@
     "CqlOperation": "$cql"
   },
   "Build": {
-    "CqlFileVersion": "1.0.000",
+    "CqlFileVersion": "1.0.0",
     "CqlOutputPath": "./cql",
     "CqlVersion": "1.5",
     "testsRunDescription": "Local host test run",

conf/smile-cdr-local.json

@@ -5,14 +5,14 @@
     "CqlOperation": "$cql"
   },
   "Build": {
-    "CqlFileVersion": "1.0.000",
+    "CqlFileVersion": "1.0.0",
     "CqlOutputPath": "./cql",
     "CqlVersion": "1.5",
     "testsRunDescription": "Smile-cdr-local test run",
     "cqlTranslator": "Java CQFramework Translator",
     "cqlTranslatorVersion": "Unknown",
     "cqlEngine": "Java CQFramework Engine",
-    "cqlEngineVersion": "Unknown",
+    "cqlEngineVersion": "Unknown"
   },
   "Debug": {
     "QuickTest": false

src/conf/config-loader.ts

@@ -1,6 +1,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import type { Config, SkipItem, OnlyItem } from '../models/config-types.js';
+import type { CapabilityVersionExtensions } from '../cql-engine/capability-statement.js';
 import { ConfigValidator } from './config-validator.js';
 import type { ValidationError } from './config-validator.js';
 
@@ -18,6 +19,7 @@ export class ConfigLoader implements Config {
 		cqlTranslatorVersion: string;
 		cqlEngine: string;
 		cqlEngineVersion: string;
+		CapabilityVersionExtensions?: CapabilityVersionExtensions;
 	};
 	Tests: {
 		ResultsPath: string;
@@ -50,7 +52,7 @@ export class ConfigLoader implements Config {
 
 		this.Build = {
 			CqlFileVersion:
-				process.env.CQL_FILE_VERSION || configData.Build?.CqlFileVersion || '1.0.000',
+				process.env.CQL_FILE_VERSION || configData.Build?.CqlFileVersion || '1.0.0',
 			CqlOutputPath:
 				process.env.CQL_OUTPUT_PATH || configData.Build?.CqlOutputPath || './cql',
 			CqlVersion:
@@ -63,8 +65,10 @@ export class ConfigLoader implements Config {
 				process.env.CQL_TRANSLATOR_VERSION || configData.Build?.cqlTranslatorVersion || 'Unknown',
 			cqlEngine:
 				process.env.CQL_ENGINE || configData.Build?.cqlEngine || 'Unknown',
-			cqlEngineVersion: 
-				process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'Unknown'
+			cqlEngineVersion:
+				process.env.CQL_ENGINE_VERSION || configData.Build?.cqlEngineVersion || 'Unknown',
+			// No default: with no configured urls nothing is read from the CapabilityStatement.
+			CapabilityVersionExtensions: configData.Build?.CapabilityVersionExtensions
 		};
 
 		this.Tests = {