From 288bfb6f1aa261fd88970f634e4bb705ca222f14 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sat, 24 Oct 2026 13:17:32 +0100 Subject: [PATCH 1/2] Add optional test case description --- CHANGELOG.md | 4 + examples/with-description.json | 33 ++++ schema/ctrf.schema.json | 4 + spec/ctrf.md | 58 ++++++ tests/ctrf.test.json | 5 + .../normative/additional-properties.test.json | 44 +++++ tests/normative/type-validation.test.json | 185 ++++++++++++++++++ tests/normative/valid-documents.test.json | 79 ++++++++ 8 files changed, 412 insertions(+) create mode 100644 examples/with-description.json diff --git a/CHANGELOG.md b/CHANGELOG.md index ed379ad..addc78c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Optional Test `description` in specification 0.3.0 for human-readable case context, separate from its name, suite and attempt diagnostics. + ### Changed - Clarified output-line arrays and valid omission or empty arrays when no stdout or stderr was captured, on final results and previous attempts. diff --git a/examples/with-description.json b/examples/with-description.json new file mode 100644 index 0000000..3e72ec9 --- /dev/null +++ b/examples/with-description.json @@ -0,0 +1,33 @@ +{ + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": "Test reachability with ICMP ping", + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } +} diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index ad0068a..6cd3df0 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -163,6 +163,10 @@ "type": "string", "minLength": 1 }, + "description": { + "description": "Optional human-readable description of the test case, separate from its name, suite hierarchy and attempt diagnostics", + "type": "string" + }, "status": { "description": "Final outcome of the test case", "enum": [ "passed", "failed", "skipped", "pending", "other" ] diff --git a/spec/ctrf.md b/spec/ctrf.md index 52133f9..dcb0f8e 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -967,6 +967,17 @@ The name or title of the test case. `name` MUST be present. It MUST be a non-empty string. +### 9.4.1. `description` + +**Description:** +A human-readable description of the test case, such as its purpose or the behavior it checks. + +`name` remains the case's primary display name, and `suite` describes its grouping hierarchy. `description` provides additional context; attempt-specific failure diagnostics belong in `message` and `trace`. + +**Requirements:** +`description` is OPTIONAL. +If present, it MUST be a string. + ### 9.5. `status` **Description:** @@ -2285,6 +2296,10 @@ to this specification. "type": "string", "minLength": 1 }, + "description": { + "description": "Optional human-readable description of the test case, separate from its name, suite hierarchy and attempt diagnostics", + "type": "string" + }, "status": { "description": "Final outcome of the test case", "enum": [ "passed", "failed", "skipped", "pending", "other" ] @@ -3878,3 +3893,46 @@ This example matches `examples/portable-attachments.json`. } } ``` + + +### D.10. Test Case Description + +`name` is the case's primary display name, `suite` is its grouping hierarchy, and `description` provides additional human-readable context. No description is required, and it does not replace failure diagnostics. + +This example matches `examples/with-description.json`. + +```json title="Test case description" +{ + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": "Test reachability with ICMP ping", + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } +} +``` diff --git a/tests/ctrf.test.json b/tests/ctrf.test.json index d41b390..e8faa12 100644 --- a/tests/ctrf.test.json +++ b/tests/ctrf.test.json @@ -213,6 +213,11 @@ "description": "Durable attachment references with producer-defined local paths and digests", "valid": true, "dataPath": "../examples/portable-attachments.json" + }, + { + "description": "Test case with an optional human-readable description", + "valid": true, + "dataPath": "../examples/with-description.json" } ] } diff --git a/tests/normative/additional-properties.test.json b/tests/normative/additional-properties.test.json index a483850..c7b92e8 100644 --- a/tests/normative/additional-properties.test.json +++ b/tests/normative/additional-properties.test.json @@ -211,6 +211,50 @@ }] } } + }, + { + "description": "Description is a Test field, not an attempt history field", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py", + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "description": "Not an attempt field" + } + ] + } + ] + } + } } ] } diff --git a/tests/normative/type-validation.test.json b/tests/normative/type-validation.test.json index 0419fad..6f37ec2 100644 --- a/tests/normative/type-validation.test.json +++ b/tests/normative/type-validation.test.json @@ -1234,6 +1234,191 @@ ] } } + }, + { + "description": "Test description rejects null", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": null, + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } + }, + { + "description": "Test description rejects number", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": 42, + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } + }, + { + "description": "Test description rejects boolean", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": false, + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } + }, + { + "description": "Test description rejects array", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": [], + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } + }, + { + "description": "Test description rejects object", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": {}, + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index e7bc6f8..5a5cc90 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -100,6 +100,85 @@ ] } } + }, + { + "description": "Test case with an optional human-readable description", + "valid": true, + "dataPath": "../../examples/with-description.json" + }, + { + "description": "Test description accepts empty string", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": "", + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } + }, + { + "description": "Test description accepts multiline string", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "example-runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test_icmp_ping", + "status": "passed", + "duration": 100, + "description": "Checks reachability.\nUses ICMP echo.", + "suite": [ + "tests", + "test_ping", + "TestPingTestCase" + ], + "filePath": "tests/test_ping.py" + } + ] + } + } } ] } From 72723529c6f83c91ecf6ecb4c9b2289f58471782 Mon Sep 17 00:00:00 2001 From: Matthew Thomas <25988124+Ma11hewThomas@users.noreply.github.com> Date: Sun, 25 Oct 2026 18:05:02 +0000 Subject: [PATCH 2/2] Simplify test description documentation --- examples/with-description.json | 33 ----------------- schema/ctrf.schema.json | 2 +- spec/ctrf.md | 45 +---------------------- tests/ctrf.test.json | 5 --- tests/normative/valid-documents.test.json | 5 --- 5 files changed, 2 insertions(+), 88 deletions(-) delete mode 100644 examples/with-description.json diff --git a/examples/with-description.json b/examples/with-description.json deleted file mode 100644 index 3e72ec9..0000000 --- a/examples/with-description.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "reportFormat": "CTRF", - "specVersion": "0.3.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "test_icmp_ping", - "status": "passed", - "duration": 100, - "description": "Test reachability with ICMP ping", - "suite": [ - "tests", - "test_ping", - "TestPingTestCase" - ], - "filePath": "tests/test_ping.py" - } - ] - } -} diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index e86f984..06aa9ae 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -164,7 +164,7 @@ "minLength": 1 }, "description": { - "description": "Optional human-readable description of the test case, separate from its name, suite hierarchy and attempt diagnostics", + "description": "Human-readable description of the test case", "type": "string" }, "status": { diff --git a/spec/ctrf.md b/spec/ctrf.md index 58eb815..f037f3b 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -2295,7 +2295,7 @@ to this specification. "minLength": 1 }, "description": { - "description": "Optional human-readable description of the test case, separate from its name, suite hierarchy and attempt diagnostics", + "description": "Human-readable description of the test case", "type": "string" }, "status": { @@ -3798,46 +3798,3 @@ Each block below matches its named standalone example. } } ``` - - -### D.9. Test Case Description - -`name` is the case's primary display name, `suite` is its grouping hierarchy, and `description` provides additional human-readable context. No description is required, and it does not replace failure diagnostics. - -This example matches `examples/with-description.json`. - -```json title="Test case description" -{ - "reportFormat": "CTRF", - "specVersion": "0.3.0", - "results": { - "tool": { - "name": "example-runner" - }, - "summary": { - "tests": 1, - "passed": 1, - "failed": 0, - "pending": 0, - "skipped": 0, - "other": 0, - "start": 1609459200000, - "stop": 1609459201000 - }, - "tests": [ - { - "name": "test_icmp_ping", - "status": "passed", - "duration": 100, - "description": "Test reachability with ICMP ping", - "suite": [ - "tests", - "test_ping", - "TestPingTestCase" - ], - "filePath": "tests/test_ping.py" - } - ] - } -} -``` diff --git a/tests/ctrf.test.json b/tests/ctrf.test.json index 339b074..41af69f 100644 --- a/tests/ctrf.test.json +++ b/tests/ctrf.test.json @@ -208,11 +208,6 @@ "description": "Merged coordinated retry report", "valid": true, "dataPath": "../examples/coordinated-retry-merged.json" - }, - { - "description": "Test case with an optional human-readable description", - "valid": true, - "dataPath": "../examples/with-description.json" } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index d21dc8f..b3cfd44 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -96,11 +96,6 @@ } } }, - { - "description": "Test case with an optional human-readable description", - "valid": true, - "dataPath": "../../examples/with-description.json" - }, { "description": "Test description accepts empty string", "valid": true,