From 18b98daf7f70ab586b6036d33fad8b4a59cb6a24 Mon Sep 17 00:00:00 2001 From: Alex Shamrai Date: Mon, 26 Oct 2026 17:32:15 +0200 Subject: [PATCH] Add optional threadId to attempt history entries Attempt history entries can record the thread or worker that executed them, using the same contract as the Test threadId. The Test threadId now explicitly describes the final attempt, and producers that run test cases in more than one process or on more than one host are asked to qualify threadId values. No omission rule and no serial-execution guarantee are added. --- CHANGELOG.md | 4 + examples/comprehensive.json | 1 + schema/ctrf.schema.json | 6 +- spec/ctrf.md | 25 ++- tests/normative/type-validation.test.json | 190 ++++++++++++++++++++++ tests/normative/valid-documents.test.json | 46 ++++++ 6 files changed, 267 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5007ddc..80086ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Optional `threadId` on attempt history entries in specification 0.3.0, using the same contract as Test `threadId`, so previous attempts can record the thread or worker that executed them (#74). + - Optional Test `description` in specification 0.3.0 for human-readable case context, separate from its name, suite and attempt diagnostics. ### Changed +- Clarified that Test `threadId` identifies the thread or worker that executed the final attempt, and recommended qualifying `threadId` values when test cases run across processes or hosts (#74). + - Clarified output-line arrays and valid omission or empty arrays when no stdout or stderr was captured, on final results and previous attempts. - Documented legacy `id` removal in 1.0.0 and version-based migration while retaining current acceptance and optional `testId`. diff --git a/examples/comprehensive.json b/examples/comprehensive.json index bc574d8..f9eb656 100644 --- a/examples/comprehensive.json +++ b/examples/comprehensive.json @@ -104,6 +104,7 @@ "duration": 5100, "start": 1609459203500, "stop": 1609459208600, + "threadId": "worker-1", "message": "Payment gateway timeout", "trace": "Error: Payment gateway timeout\n at processPayment (payment.js:23:10)", "attachments": [ diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 06aa9ae..7681857 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -341,6 +341,10 @@ "description": "Attempt end time (milliseconds since Unix epoch)", "type": "integer" }, + "threadId": { + "description": "Thread or worker that executed this attempt", + "type": "string" + }, "attachments": { "description": "Artifacts from this attempt", "type": "array", @@ -401,7 +405,7 @@ } }, "threadId": { - "description": "Thread or worker that executed the test case", + "description": "Thread or worker that executed the final attempt represented by the test object", "type": "string" }, "browser": { diff --git a/spec/ctrf.md b/spec/ctrf.md index 910a5b2..17b1695 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -1182,11 +1182,12 @@ If no output was captured, producers MAY omit `stderr` or emit an empty array. ### 9.25. `threadId` **Description:** -Identifies the thread or worker that executed the test case. +Identifies the thread or worker that executed the final attempt represented by this test object. Earlier attempts can record their own `threadId` in `retryAttempts` (Section 11). **Requirements:** `threadId` is OPTIONAL. -If present, it MUST be a string. +If present, it MUST be a string. +When test cases run in more than one process or on more than one host, producers SHOULD qualify the value with the process or host to avoid collisions, for example, `"host-a/worker-1"`. ### 9.26. `browser` @@ -1586,6 +1587,16 @@ The timestamp indicating when the attempt ended, in milliseconds since the Unix If present, it MUST be an integer. If both `start` and `stop` are present, `stop` SHOULD be greater than or equal to `start`. +### 11.12.1. `threadId` + +**Description:** +Identifies the thread or worker that executed this attempt. + +**Requirements:** +`threadId` is OPTIONAL. +If present, it MUST be a string. +When test cases run in more than one process or on more than one host, producers SHOULD qualify the value with the process or host to avoid collisions, for example, `"host-a/worker-1"`. + ### 11.13. `attachments` **Description:** @@ -2472,6 +2483,10 @@ to this specification. "description": "Attempt end time (milliseconds since Unix epoch)", "type": "integer" }, + "threadId": { + "description": "Thread or worker that executed this attempt", + "type": "string" + }, "attachments": { "description": "Artifacts from this attempt", "type": "array", @@ -2532,7 +2547,7 @@ to this specification. } }, "threadId": { - "description": "Thread or worker that executed the test case", + "description": "Thread or worker that executed the final attempt represented by the test object", "type": "string" }, "browser": { @@ -3382,10 +3397,12 @@ It includes: "duration": 1500, "message": "Timeout waiting for response", "stdout": ["Submitting login form"], - "stderr": ["Request timed out"] + "stderr": ["Request timed out"], + "threadId": "worker-1" } ], "stdout": ["Test completed successfully"], + "threadId": "worker-2", "insights": { "passRate": { "current": 0.9, diff --git a/tests/normative/type-validation.test.json b/tests/normative/type-validation.test.json index 8ab0c1d..627fb82 100644 --- a/tests/normative/type-validation.test.json +++ b/tests/normative/type-validation.test.json @@ -1419,6 +1419,196 @@ ] } } + }, + { + "description": "RetryAttempt with threadId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": 12345 + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as null instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": null + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as boolean instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": true + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as array instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": [] + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as object instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": {} + } + ] + } + ] + } + } } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index 1fd4a72..b7c3d2e 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -169,6 +169,52 @@ ] } } + }, + { + "description": "Previous attempt threadId differs from the final attempt threadId", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "flaky": 1, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "checkout accepts a saved card", + "status": "passed", + "duration": 500, + "start": 1609459200500, + "stop": 1609459201000, + "threadId": "fork-2/Test worker", + "flaky": true, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "duration": 400, + "start": 1609459200000, + "stop": 1609459200400, + "threadId": "fork-1/Test worker" + } + ] + } + ] + } + } } ] }