diff --git a/CHANGELOG.md b/CHANGELOG.md index 5007ddc..80086ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Optional `threadId` on attempt history entries in specification 0.3.0, using the same contract as Test `threadId`, so previous attempts can record the thread or worker that executed them (#74). + - Optional Test `description` in specification 0.3.0 for human-readable case context, separate from its name, suite and attempt diagnostics. ### Changed +- Clarified that Test `threadId` identifies the thread or worker that executed the final attempt, and recommended qualifying `threadId` values when test cases run across processes or hosts (#74). + - Clarified output-line arrays and valid omission or empty arrays when no stdout or stderr was captured, on final results and previous attempts. - Documented legacy `id` removal in 1.0.0 and version-based migration while retaining current acceptance and optional `testId`. diff --git a/examples/comprehensive.json b/examples/comprehensive.json index bc574d8..f9eb656 100644 --- a/examples/comprehensive.json +++ b/examples/comprehensive.json @@ -104,6 +104,7 @@ "duration": 5100, "start": 1609459203500, "stop": 1609459208600, + "threadId": "worker-1", "message": "Payment gateway timeout", "trace": "Error: Payment gateway timeout\n at processPayment (payment.js:23:10)", "attachments": [ diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 06aa9ae..7681857 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -341,6 +341,10 @@ "description": "Attempt end time (milliseconds since Unix epoch)", "type": "integer" }, + "threadId": { + "description": "Thread or worker that executed this attempt", + "type": "string" + }, "attachments": { "description": "Artifacts from this attempt", "type": "array", @@ -401,7 +405,7 @@ } }, "threadId": { - "description": "Thread or worker that executed the test case", + "description": "Thread or worker that executed the final attempt represented by the test object", "type": "string" }, "browser": { diff --git a/spec/ctrf.md b/spec/ctrf.md index 910a5b2..17b1695 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -1182,11 +1182,12 @@ If no output was captured, producers MAY omit `stderr` or emit an empty array. ### 9.25. `threadId` **Description:** -Identifies the thread or worker that executed the test case. +Identifies the thread or worker that executed the final attempt represented by this test object. Earlier attempts can record their own `threadId` in `retryAttempts` (Section 11). **Requirements:** `threadId` is OPTIONAL. -If present, it MUST be a string. +If present, it MUST be a string. +When test cases run in more than one process or on more than one host, producers SHOULD qualify the value with the process or host to avoid collisions, for example, `"host-a/worker-1"`. ### 9.26. `browser` @@ -1586,6 +1587,16 @@ The timestamp indicating when the attempt ended, in milliseconds since the Unix If present, it MUST be an integer. If both `start` and `stop` are present, `stop` SHOULD be greater than or equal to `start`. +### 11.12.1. `threadId` + +**Description:** +Identifies the thread or worker that executed this attempt. + +**Requirements:** +`threadId` is OPTIONAL. +If present, it MUST be a string. +When test cases run in more than one process or on more than one host, producers SHOULD qualify the value with the process or host to avoid collisions, for example, `"host-a/worker-1"`. + ### 11.13. `attachments` **Description:** @@ -2472,6 +2483,10 @@ to this specification. "description": "Attempt end time (milliseconds since Unix epoch)", "type": "integer" }, + "threadId": { + "description": "Thread or worker that executed this attempt", + "type": "string" + }, "attachments": { "description": "Artifacts from this attempt", "type": "array", @@ -2532,7 +2547,7 @@ to this specification. } }, "threadId": { - "description": "Thread or worker that executed the test case", + "description": "Thread or worker that executed the final attempt represented by the test object", "type": "string" }, "browser": { @@ -3382,10 +3397,12 @@ It includes: "duration": 1500, "message": "Timeout waiting for response", "stdout": ["Submitting login form"], - "stderr": ["Request timed out"] + "stderr": ["Request timed out"], + "threadId": "worker-1" } ], "stdout": ["Test completed successfully"], + "threadId": "worker-2", "insights": { "passRate": { "current": 0.9, diff --git a/tests/normative/type-validation.test.json b/tests/normative/type-validation.test.json index 8ab0c1d..627fb82 100644 --- a/tests/normative/type-validation.test.json +++ b/tests/normative/type-validation.test.json @@ -1419,6 +1419,196 @@ ] } } + }, + { + "description": "RetryAttempt with threadId as number instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": 12345 + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as null instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": null + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as boolean instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": true + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as array instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": [] + } + ] + } + ] + } + } + }, + { + "description": "RetryAttempt with threadId as object instead of string", + "valid": false, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "test", + "status": "passed", + "duration": 100, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "threadId": {} + } + ] + } + ] + } + } } ] } diff --git a/tests/normative/valid-documents.test.json b/tests/normative/valid-documents.test.json index 1fd4a72..b7c3d2e 100644 --- a/tests/normative/valid-documents.test.json +++ b/tests/normative/valid-documents.test.json @@ -169,6 +169,52 @@ ] } } + }, + { + "description": "Previous attempt threadId differs from the final attempt threadId", + "valid": true, + "data": { + "reportFormat": "CTRF", + "specVersion": "0.3.0", + "results": { + "tool": { + "name": "runner" + }, + "summary": { + "tests": 1, + "passed": 1, + "failed": 0, + "pending": 0, + "skipped": 0, + "other": 0, + "flaky": 1, + "start": 1609459200000, + "stop": 1609459201000 + }, + "tests": [ + { + "name": "checkout accepts a saved card", + "status": "passed", + "duration": 500, + "start": 1609459200500, + "stop": 1609459201000, + "threadId": "fork-2/Test worker", + "flaky": true, + "retries": 1, + "retryAttempts": [ + { + "attempt": 1, + "status": "failed", + "duration": 400, + "start": 1609459200000, + "stop": 1609459200400, + "threadId": "fork-1/Test worker" + } + ] + } + ] + } + } } ] }