From 28024fb9c35e5b02e729088d3cd210a2aeb4f28d Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:13:11 +0100 Subject: [PATCH 1/4] security: the ODCS and plugin-extension checks let a trailing newline through a pattern The FLUID fix (schema_validator, #799) left six JSON Schema checks on jsonschema's own pattern keyword, Python's re.search, whose $ also matches before a final newline: - forge/core/artifact_validators.py (fluid validate-artifacts: ODCS, Bitol ODPS) - providers/odcs/validation.py::collect_errors (fluid odcs validate) - providers/odps_standard/validation.py::validate - providers/opds/validator.py (pinned Draft202012Validator) - forge_datamodel/emit/osi_sidecar.py (the copilot's Ossie check) - extension_schemas.py::assemble_proposed_extensions (pinned Draft7Validator) So ODCS refused schema[].id "t 1" but took "t1\n", which an id: | block produces, and the copilot kept an LLM-proposed customScaffold block with use: "ci:basic\n", ref: "main\n" or id: "ci\n". All six now go through schema_validator, and say so with the same line-break hint. schema_validator takes a default dialect for a schema that declares none, passed to jsonschema's validator_for(default=), the way OpenStack Rally uses it. An extension schema is read in the dialect it declares, draft-07 when it declares none (the documented plugin contract), so a 2020-12 schema's unevaluatedProperties: false is enforced instead of ignored. OPDS keeps 2020-12 as its default. patternProperties keys keep re.search. jsonschema matches them in patternProperties, additionalProperties and unevaluatedProperties, and check-jsonschema's override of patternProperties alone lets a key that one keyword matches and another does not through additionalProperties: false unchecked. Only OPDS has key patterns (^[a-z]{2}$, three places). The tripwires now cover every shipped schema, pin that set, and also fail on a $recursiveRef or $dynamicRef, which re-pick the stock class as a nested $schema does. Follows #799. --- CHANGELOG.md | 11 + fluid_build/extension_schemas.py | 10 +- fluid_build/forge/core/artifact_validators.py | 12 +- .../forge_datamodel/emit/osi_sidecar.py | 8 +- fluid_build/providers/odcs/validation.py | 12 +- .../providers/odps_standard/validation.py | 7 +- fluid_build/providers/opds/validator.py | 8 +- fluid_build/schema_manager.py | 22 +- tests/test_schema_pattern_end_anchor.py | 48 +++-- ...test_standard_schema_pattern_end_anchor.py | 197 ++++++++++++++++++ 10 files changed, 298 insertions(+), 37 deletions(-) create mode 100644 tests/test_standard_schema_pattern_end_anchor.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f27a69f2e..ffd9244e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -602,6 +602,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ``- **column_added** `exposes[0].contract.schema.email` — `column 'email' added to expose 'orders'` ``. The `fluid forge --refine` preview prints the same text without markup. +- **The ODCS and plugin-extension checks let a value with a trailing newline through a `pattern`, + too.** The fix above covered the FLUID schemas only. `fluid odcs validate` and + `fluid validate-artifacts` accepted an ODCS `schema[].id: "t1\n"` while refusing `"t 1"`, and + the forge copilot kept an LLM-proposed extension block whose value ended in a line break, such + as `use: "ci:basic\n"` for `customScaffold`. Every JSON Schema check now reads `$` as the end of + the value, with the same hint, including the ODPS, OPDS and Ossie checks, whose vendored schemas + have no such pattern yet. An extension schema is read in the dialect its `$schema` declares, and + as draft-07 when it declares none, so a 2020-12 `unevaluatedProperties: false` is no longer + ignored. `patternProperties` keys are still matched as before, so OPDS's language-code key + `^[a-z]{2}$` accepts `"en\n"`. + ## [0.26.0] — 2026-11-01 ### Added diff --git a/fluid_build/extension_schemas.py b/fluid_build/extension_schemas.py index db8304993..82ab6aa1e 100644 --- a/fluid_build/extension_schemas.py +++ b/fluid_build/extension_schemas.py @@ -22,7 +22,8 @@ customScaffold = "data_product_forge_custom_scaffold.validation:get_extension_schema" The provider has the signature ``get_extension_schema(fluid_version=None) -> dict`` -and returns a draft-07 JSON Schema describing the data *under* the extension key. +and returns a JSON Schema describing the data *under* the extension key: draft-07, +unless the schema declares another dialect in its ``$schema``. These helpers let the ``fluid forge`` copilot (a) ground contract generation on the installed extension schemas and (b) validate the generated blocks before @@ -200,6 +201,7 @@ def assemble_proposed_extensions( from jsonschema import Draft7Validator except ImportError: # pragma: no cover — jsonschema is a hard dependency Draft7Validator = None # type: ignore[assignment] + from fluid_build.schema_manager import schema_validator out: Dict[str, Any] = {} for key, block in proposed.items(): @@ -207,7 +209,11 @@ def assemble_proposed_extensions( if schema is None: continue if Draft7Validator is not None: - if next(iter(Draft7Validator(schema).iter_errors(block)), None) is not None: + # The dialect the plugin's schema declares, draft-07 when it declares + # none; Draft 7 ignores a newer keyword rather than enforcing it. A + # pattern's "$" is the end of the value, as ECMA-262 reads it. + validator = schema_validator(schema, default=Draft7Validator) + if next(iter(validator.iter_errors(block)), None) is not None: continue # invalid — let the repair loop surface it; don't emit junk out[key] = block return out diff --git a/fluid_build/forge/core/artifact_validators.py b/fluid_build/forge/core/artifact_validators.py index 5577ffd9c..c568e9888 100644 --- a/fluid_build/forge/core/artifact_validators.py +++ b/fluid_build/forge/core/artifact_validators.py @@ -306,7 +306,10 @@ def _validate_against_schema( ) ] - if not _jsonschema_available(): + from fluid_build.schema_manager import JSONSCHEMA_AVAILABLE, schema_validator + + # The flag schema_validator needs: a package that is found can still fail to import. + if not JSONSCHEMA_AVAILABLE: return [ ValidationIssue( file=artifact_path, @@ -320,8 +323,6 @@ def _validate_against_schema( ) ] - import jsonschema # type: ignore - issues: List[ValidationIssue] = [] # Validate with the dialect the schema DECLARES, not a hardcoded one. # @@ -335,7 +336,10 @@ def _validate_against_schema( # and guards nine objects with `unevaluatedProperties: false`. A server entry # carrying an unexpected key validated with ZERO errors, reachable from # `fluid validate-artifacts`. See tests/test_jsonschema_dialect.py. - validator = jsonschema.validators.validator_for(schema)(schema) + # + # schema_validator also reads a pattern's `$` as the end of the value, as + # ECMA-262 does: ODCS refused `schema[].id: "t 1"` but took `"t1\n"`. + validator = schema_validator(schema) errors = sorted(validator.iter_errors(doc), key=lambda e: list(e.absolute_path)) for err in errors: diff --git a/fluid_build/forge_datamodel/emit/osi_sidecar.py b/fluid_build/forge_datamodel/emit/osi_sidecar.py index d9d5f2f10..206749753 100644 --- a/fluid_build/forge_datamodel/emit/osi_sidecar.py +++ b/fluid_build/forge_datamodel/emit/osi_sidecar.py @@ -93,14 +93,14 @@ def validate_osi_document(document: Dict[str, Any]) -> List[str]: Returns a sorted list of human-readable issue strings (empty when conformant). ``jsonschema`` is imported lazily — this module sits on the forge pipeline path, not the CLI cold path, but there is no - reason to tax importers that never validate. + reason to tax importers that never validate. A ``pattern``'s ``$`` is + the end of the value, as ECMA-262 reads it. """ - import jsonschema + from fluid_build.schema_manager import schema_validator schema = _load_ossie_schema() - validator_cls = jsonschema.validators.validator_for(schema) errors = sorted( - validator_cls(schema).iter_errors(document), + schema_validator(schema).iter_errors(document), key=lambda e: ([str(p) for p in e.absolute_path], e.message), ) return [f"{'/'.join(str(p) for p in e.absolute_path) or ''}: {e.message}" for e in errors] diff --git a/fluid_build/providers/odcs/validation.py b/fluid_build/providers/odcs/validation.py index 7cb2260db..9711f5822 100644 --- a/fluid_build/providers/odcs/validation.py +++ b/fluid_build/providers/odcs/validation.py @@ -74,15 +74,17 @@ def collect_errors(odcs: Mapping[str, Any], schema: Mapping[str, Any]) -> List[D Empty list = valid. No-op (empty list) when :mod:`jsonschema` is not installed. Used by ``validate()`` (which raises on non-empty) and by ``fluid odcs validate --report`` (which serializes to JSON for CI). + + A ``pattern``'s ``$`` is the end of the value, as ECMA-262 reads it, so + ``schema[].id: "t1\\n"`` fails ``^[A-Za-z0-9_-]+$`` like ``"t 1"`` does. """ - try: - import jsonschema - except ImportError: + from fluid_build.schema_manager import JSONSCHEMA_AVAILABLE, schema_validator + + if not JSONSCHEMA_AVAILABLE: LOG.warning("jsonschema not installed, skipping ODCS validation") return [] - validator_cls = jsonschema.validators.validator_for(schema) - validator = validator_cls(schema) + validator = schema_validator(schema) out: List[Dict[str, Any]] = [] for err in validator.iter_errors(odcs): out.append( diff --git a/fluid_build/providers/odps_standard/validation.py b/fluid_build/providers/odps_standard/validation.py index b6428c2ea..4d3bd9bb3 100644 --- a/fluid_build/providers/odps_standard/validation.py +++ b/fluid_build/providers/odps_standard/validation.py @@ -81,7 +81,12 @@ def validate(odps: Mapping[str, Any], schema: Mapping[str, Any]) -> None: except ImportError: LOG.warning("jsonschema not installed, skipping ODPS validation") return + from fluid_build.schema_manager import schema_validator + try: - jsonschema.validate(instance=odps, schema=schema) + # cls: the declared dialect, with a pattern's "$" read as the end of the + # value, as ECMA-262 does. jsonschema.validate still checks the schema first. + cls = type(schema_validator(schema)) + jsonschema.validate(instance=odps, schema=schema, cls=cls) except jsonschema.ValidationError as exc: raise ProviderError(f"ODPS validation failed: {exc.message}") from exc diff --git a/fluid_build/providers/opds/validator.py b/fluid_build/providers/opds/validator.py index 1f131cf07..c75b00803 100644 --- a/fluid_build/providers/opds/validator.py +++ b/fluid_build/providers/opds/validator.py @@ -119,8 +119,14 @@ def validate_against_opds_schema( valid, errors = _basic_validation(opds_data, version) return valid, errors, "basic" + from fluid_build.schema_manager import schema_validator + try: - validator = jsonschema.Draft202012Validator(schema) + # The dialect the schema declares (v4.1: 2020-12, also the default), + # with a pattern's "$" read as the end of the value, as ECMA-262 does. + # The patternProperties language-code keys (^[a-z]{2}$) keep + # jsonschema's own re.search; see schema_manager.schema_validator. + validator = schema_validator(schema, default=jsonschema.Draft202012Validator) errors = list(validator.iter_errors(opds_data)) # Filter a known upstream OPDS v4.1 schema false-positive: the diff --git a/fluid_build/schema_manager.py b/fluid_build/schema_manager.py index 1b992d7ee..d78c06a03 100644 --- a/fluid_build/schema_manager.py +++ b/fluid_build/schema_manager.py @@ -28,7 +28,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Callable, Dict, FrozenSet, Iterator, List, Optional, Union +from typing import Any, Callable, Dict, FrozenSet, Iterator, List, Mapping, Optional, Union import yaml @@ -184,20 +184,28 @@ def _end_anchored_validator(validator_class: Any) -> Any: return jsonschema.validators.extend(validator_class, {"pattern": _pattern_keyword}) -def schema_validator(schema: Dict[str, Any]) -> Any: +def schema_validator(schema: Mapping[str, Any], default: Any = None) -> Any: """A jsonschema validator for ``schema``, in the dialect it declares, whose ``pattern`` ends at the end of the value (:func:`_end_anchored`). Validate - against a FLUID schema through this, never ``validator_for`` directly. + against a FLUID schema, or a vendored or plugin one, through this, never + ``validator_for`` directly. ``default`` is the dialect of a schema that + declares none, jsonschema's newest when not given. The keyword is replaced the way check-jsonschema replaces it, with ``jsonschema.validators.extend``. Two places keep jsonschema's own ``re.search``: ``patternProperties`` keys, as its ``additionalProperties`` and :func:`_locate_error` read them, and a subschema that declares its own - ``$schema`` (or a ``$ref`` to the root, which does), for which jsonschema - picks that dialect's own class. No FLUID schema has either; a test fails if - one does. + ``$schema`` (or a ``$ref``, ``$recursiveRef`` or ``$dynamicRef`` to the + root, which does), for which jsonschema picks that dialect's own class. No + FLUID schema has either. Of the vendored ones, only OPDS keys + ``patternProperties``, with ``^[a-z]{2}$`` language codes. A test fails if + either changes (``_KEY_PATTERNS`` in tests/test_schema_pattern_end_anchor.py). """ - return _end_anchored_validator(jsonschema.validators.validator_for(schema))(schema) + if default is None: + validator_class = jsonschema.validators.validator_for(schema) + else: + validator_class = jsonschema.validators.validator_for(schema, default=default) + return _end_anchored_validator(validator_class)(schema) def _schema_version_sort_key(version: str) -> tuple[int, int, int]: diff --git a/tests/test_schema_pattern_end_anchor.py b/tests/test_schema_pattern_end_anchor.py index 6b12793da..b51d057bc 100644 --- a/tests/test_schema_pattern_end_anchor.py +++ b/tests/test_schema_pattern_end_anchor.py @@ -239,12 +239,33 @@ def _nodes(node: Any) -> Iterator[Dict[str, Any]]: yield from _nodes(value) +#: Every schema fluid validates against: the FLUID and Ossie ones, and the vendored +#: standards (tests/test_standard_schema_pattern_end_anchor.py tests their checks). +_SHIPPED = sorted(_SCHEMA_DIR.glob("*.json")) + [ + _SCHEMA_DIR.parent / "providers" / "odcs" / "odcs-schema-v3.1.0.json", + _SCHEMA_DIR.parent / "providers" / "odps_standard" / "schemas" / "odps-product-v1.0.0.json", + _SCHEMA_DIR.parent / "providers" / "odps_standard" / "schemas" / "odps-product-v1.1.0.json", + _SCHEMA_DIR.parent / "providers" / "opds" / "opds-schema-v4.1.0.json", +] + +#: ``patternProperties`` keys, by schema. They keep jsonschema's own ``re.search``: +#: jsonschema matches them in ``patternProperties``, ``additionalProperties`` and +#: ``unevaluatedProperties``, and overriding ``patternProperties`` alone, as +#: check-jsonschema does, lets a key that one matches and another does not through +#: ``additionalProperties: false`` unchecked. So OPDS's ISO 639-1 language codes, +#: ``^[a-z]{2}$``, also match ``"en\n"``. Under ``product.details`` and ``declarative``, +#: which take any key, the value is then checked as a language entry. +#: ``$defs/Languages`` takes no other key, so a hand-written OPDS file can carry +#: ``{"en\n": ...}`` there; fluid's emitter writes ``en``. +_KEY_PATTERNS = {"opds-schema-v4.1.0.json": [("^[a-z]{2}$",)] * 3} + + def test_every_bundled_schema_pattern_is_rewritten_anchor_for_anchor() -> None: """The scan reads escapes and sets as Python does, checked against Python's own - parser for every ``pattern`` the bundled schemas hold.""" + parser for every ``pattern`` the shipped schemas hold.""" patterns = { node["pattern"] - for path in _SCHEMA_DIR.glob("fluid-*schema-*.json") + for path in _SHIPPED for node in _nodes(json.loads(path.read_text(encoding="utf-8"))) if isinstance(node.get("pattern"), str) } @@ -252,16 +273,17 @@ def test_every_bundled_schema_pattern_is_rewritten_anchor_for_anchor() -> None: assert [p for p in patterns if not _only_end_anchors_changed(p)] == [] -@pytest.mark.parametrize( - "path", sorted(_SCHEMA_DIR.glob("fluid-schema-*.json")), ids=lambda path: path.name -) +@pytest.mark.parametrize("path", _SHIPPED, ids=lambda path: path.name) def test_every_pattern_of_the_schema_is_checked_with_the_rewrite(path: Path) -> None: """jsonschema picks the validator class again for each subschema it enters, by - that subschema's ``$schema``. One that declares it, or a ``$ref`` to the root, - which does, would be checked with jsonschema's own ``pattern`` again. - ``patternProperties`` keys are not rewritten (``schema_validator`` says why), so - a schema that starts to use the keyword has to come back here.""" - below_root = list(_nodes(json.loads(path.read_text(encoding="utf-8"))))[1:] - assert [n for n in below_root if isinstance(n.get("$schema"), str)] == [] - assert [n for n in below_root if n.get("$ref") == "#"] == [] - assert [n for n in below_root if "patternProperties" in n] == [] + that subschema's ``$schema``. One that declares it, or a ``$ref``, + ``$recursiveRef`` or ``$dynamicRef`` to the root, which does, would be checked + with jsonschema's own ``pattern`` again. ``patternProperties`` keys are not + rewritten, so a schema that starts to use the keyword has to come back to + :data:`_KEY_PATTERNS`.""" + nodes = list(_nodes(json.loads(path.read_text(encoding="utf-8")))) + assert [n for n in nodes[1:] if isinstance(n.get("$schema"), str)] == [] + assert [n for n in nodes if n.get("$ref") == "#"] == [] + assert [n for n in nodes if "$recursiveRef" in n or "$dynamicRef" in n] == [] + keys = [tuple(n["patternProperties"]) for n in nodes if "patternProperties" in n] + assert keys == _KEY_PATTERNS.get(path.name, []) diff --git a/tests/test_standard_schema_pattern_end_anchor.py b/tests/test_standard_schema_pattern_end_anchor.py new file mode 100644 index 000000000..9cbdab12a --- /dev/null +++ b/tests/test_standard_schema_pattern_end_anchor.py @@ -0,0 +1,197 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The other schema checks read a ``pattern``'s ``$`` as ECMA-262 does, too. + +``tests/test_schema_pattern_end_anchor.py`` covers the FLUID schemas. fluid also +validates against the vendored ODCS, Bitol ODPS and OPDS schemas, the Ossie +schema, and the schema an installed plugin gives its ``extensions.`` block. +Each of those checks ran jsonschema's own ``pattern``, Python's ``re.search``, +whose ``$`` also matches before a final ``\\n``. So ODCS refused +``schema[].id: "t 1"`` but took ``"t1\\n"``, which an ``id: |`` block produces. + +Of the vendored schemas only ODCS has an end-anchored ``pattern`` today, so the +other checks are tested with a schema of the test's own. The tripwires in +``tests/test_schema_pattern_end_anchor.py`` cover the vendored schemas too: what +they would have to start using for a check to leave the rewrite. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Dict + +import jsonschema +import pytest +import yaml + +import fluid_build.extension_schemas as extension_schemas +from fluid_build.forge.core.artifact_validators import validate_odcs +from fluid_build.forge_datamodel.emit import osi_sidecar +from fluid_build.providers.base import ProviderError +from fluid_build.providers.odcs.validation import collect_errors, load_schema +from fluid_build.providers.odps_standard import validation as odps_validation +from fluid_build.providers.opds import validator as opds_validator +from tests.test_jsonschema_dialect import _ODCS_BASE + +pytestmark = [pytest.mark.unit] + +_LINE_BREAK_HINT = " (it ends in a line break, which a YAML | or > block keeps and |- or >- drops)" +#: A schema of the test's own, for the checks whose vendored schema has no pattern. +_NAME_SCHEMA: Dict[str, Any] = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"name": {"type": "string", "pattern": "^[a-z]+$"}}, +} + + +def _odcs(table_id: str) -> Dict[str, Any]: + return {**_ODCS_BASE, "schema": [{"id": table_id, "name": "orders", "logicalType": "object"}]} + + +def _odcs_yaml(block: str) -> str: + """An ODCS document whose ``schema[0].id`` is a YAML block scalar: ``|`` or ``|-``.""" + body = yaml.safe_dump(_ODCS_BASE) + return f"{body}schema:\n- id: {block}\n t1\n name: orders\n logicalType: object\n" + + +# ── ODCS: fluid odcs validate, fluid validate-artifacts ───────────────────── + + +def test_odcs_refuses_a_table_id_that_ends_in_a_newline() -> None: + """``collect_errors`` is ``fluid odcs validate``'s jsonschema pass.""" + schema = load_schema() + assert schema is not None + assert collect_errors(_odcs("t1"), schema) == [] + + errors = [e for e in collect_errors(_odcs("t1\n"), schema) if e["validator"] == "pattern"] + + assert [e["path"] for e in errors] == ["schema.0.id"], errors + assert errors[0]["message"].endswith(_LINE_BREAK_HINT), errors + + +def test_fluid_odcs_validate_refuses_a_block_scalar_table_id(tmp_path: Path, capsys: Any) -> None: + from fluid_build.cli import main + + path = tmp_path / "orders.odcs.yaml" + path.write_text(_odcs_yaml("|"), encoding="utf-8") + assert yaml.safe_load(path.read_text(encoding="utf-8"))["schema"][0]["id"] == "t1\n" + + rc = main(["odcs", "validate", str(path), "--no-vowl"]) + + text = capsys.readouterr().out + assert rc == 1, text + assert "schema.0.id: 't1\\n' does not match" in text, text + + +def test_validate_artifacts_refuses_a_block_scalar_table_id() -> None: + """``validate_odcs`` is what ``fluid validate-artifacts`` runs on ``odcs/*.odcs.yaml``.""" + assert validate_odcs("odcs/orders.odcs.yaml", _odcs_yaml("|-").encode()) == [] + + issues = validate_odcs("odcs/orders.odcs.yaml", _odcs_yaml("|").encode()) + + messages = [i.message for i in issues] + assert any(m.startswith("at schema/0/id: 't1\\n' does not match ") for m in messages), messages + + +# ── Bitol ODPS, OPDS, Ossie: no pattern in the vendored schema yet ────────── + + +def test_odps_validate_reads_a_pattern_as_ecma_does() -> None: + odps_validation.validate({"name": "orders"}, _NAME_SCHEMA) + + with pytest.raises(ProviderError, match=re.escape("'orders\\n' does not match '^[a-z]+$'")): + odps_validation.validate({"name": "orders\n"}, _NAME_SCHEMA) + + +def test_odps_validate_still_checks_the_schema_first() -> None: + """It still runs ``jsonschema.validate``, now with the rewritten class.""" + with pytest.raises(jsonschema.SchemaError): + odps_validation.validate({}, {"type": 5}) + + +def test_opds_validation_reads_a_pattern_as_ecma_does(monkeypatch: Any) -> None: + monkeypatch.setitem(opds_validator._SCHEMA_CACHE, "0.0-test", _NAME_SCHEMA) + validate = opds_validator.validate_against_opds_schema + assert validate({"name": "orders"}, "0.0-test") == (True, None, "full_schema") + + valid, errors, kind = validate({"name": "orders\n"}, "0.0-test") + + assert (valid, kind) == (False, "full_schema") + assert errors == [f"name: 'orders\\n' does not match '^[a-z]+$'{_LINE_BREAK_HINT}"] + + +def test_ossie_document_validation_reads_a_pattern_as_ecma_does(monkeypatch: Any) -> None: + monkeypatch.setattr(osi_sidecar, "_load_ossie_schema", lambda: _NAME_SCHEMA) + assert osi_sidecar.validate_osi_document({"name": "orders"}) == [] + + issues = osi_sidecar.validate_osi_document({"name": "orders\n"}) + + assert issues == [f"name: 'orders\\n' does not match '^[a-z]+$'{_LINE_BREAK_HINT}"] + + +# ── plugin extension blocks ───────────────────────────────────────────────── + +#: The shape of data-product-forge-custom-scaffold's schema: draft-07, "$"-anchored. +_SCAFFOLD_SCHEMA: Dict[str, Any] = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "patterns": { + "type": "array", + "items": { + "type": "object", + "properties": { + "use": { + "type": "string", + "pattern": "^[a-zA-Z][a-zA-Z0-9_-]*:[a-zA-Z][a-zA-Z0-9_-]*$", + } + }, + }, + } + }, +} + + +def _assemble(monkeypatch: Any, schema: Dict[str, Any], block: Any) -> Dict[str, Any]: + monkeypatch.setattr(extension_schemas, "iter_extension_schemas", lambda *a, **k: {"x": schema}) + return extension_schemas.assemble_proposed_extensions({"x": block}) + + +def test_an_extension_block_whose_value_ends_in_a_newline_is_dropped(monkeypatch: Any) -> None: + kept = {"patterns": [{"use": "ci:basic"}]} + assert _assemble(monkeypatch, _SCAFFOLD_SCHEMA, kept) == {"x": kept} + assert _assemble(monkeypatch, _SCAFFOLD_SCHEMA, {"patterns": [{"use": "ci:basic\n"}]}) == {} + + +def test_an_extension_schema_is_read_in_the_dialect_it_declares(monkeypatch: Any) -> None: + """Draft 7 ignores ``unevaluatedProperties``, so the extra key used to be kept.""" + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"libraries": {"type": "array"}}, + "unevaluatedProperties": False, + } + assert _assemble(monkeypatch, schema, {"libraries": []}) == {"x": {"libraries": []}} + assert _assemble(monkeypatch, schema, {"libraries": [], "librarys": []}) == {} + + +def test_an_extension_schema_that_declares_no_dialect_is_draft_07(monkeypatch: Any) -> None: + """The plugin contract says draft-07. 2020-12, jsonschema's default, has no + ``dependencies`` keyword and would keep this block.""" + schema = {"type": "object", "dependencies": {"output": ["patterns"]}} + kept = {"output": "x", "patterns": []} + assert _assemble(monkeypatch, schema, kept) == {"x": kept} + assert _assemble(monkeypatch, schema, {"output": "x"}) == {} From abaa405f29b11f736a03910d7b86fe521e1e715f Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:13:11 +0100 Subject: [PATCH 2/4] security: review follow-ups for the vendored-schema end-anchor change - fluid odcs validate cuts each schema message to 120 characters, so for an id like customer_orders_v1 the hint ended "... block keeps and |", which reads as advice to use |. _brief cuts the message ahead of the hint instead; the hint is schema_manager.LINE_BREAK_HINT now, so the command can find it. - schema_validator raises ImportError when jsonschema could not be imported. validate_osi_document, the one unguarded caller, raised NameError: jsonschema instead. validate_opds reads the same JSONSCHEMA_AVAILABLE flag as the other checks; the find_spec probe that found an unimportable jsonschema is gone. - A test walks fluid_build/ and fails on any validator_for(), Draft*Validator() or jsonschema.validate() without cls= outside schema_manager, so a check added later cannot read $ as Python does again. It finds the six sites on the base and seven on the old main. - Tests for the flag branches of collect_errors and the artifact validators, and for OPDS reading an undeclared schema as 2020-12. - CHANGELOG: the extension check shows no message, so it carries no hint. - odps_standard.validate validates against the schema for the document's apiVersion, not only v1.0.0. --- CHANGELOG.md | 11 +-- fluid_build/cli/odcs.py | 11 ++- fluid_build/forge/core/artifact_validators.py | 9 +-- .../providers/odps_standard/validation.py | 3 +- fluid_build/schema_manager.py | 12 ++- ...test_standard_schema_pattern_end_anchor.py | 75 ++++++++++++++++++- 6 files changed, 102 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffd9244e0..cc8066434 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -607,11 +607,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `fluid validate-artifacts` accepted an ODCS `schema[].id: "t1\n"` while refusing `"t 1"`, and the forge copilot kept an LLM-proposed extension block whose value ended in a line break, such as `use: "ci:basic\n"` for `customScaffold`. Every JSON Schema check now reads `$` as the end of - the value, with the same hint, including the ODPS, OPDS and Ossie checks, whose vendored schemas - have no such pattern yet. An extension schema is read in the dialect its `$schema` declares, and - as draft-07 when it declares none, so a 2020-12 `unevaluatedProperties: false` is no longer - ignored. `patternProperties` keys are still matched as before, so OPDS's language-code key - `^[a-z]{2}$` accepts `"en\n"`. + the value, including the ODPS, OPDS and Ossie checks, whose vendored schemas have no such + pattern yet. Their errors carry the same hint; an extension block that fails is dropped, as + before. An extension schema is read in the dialect its `$schema` declares, and as draft-07 when + it declares none, so a 2020-12 `unevaluatedProperties: false` is no longer ignored. + `patternProperties` keys are still matched as before, so OPDS's language-code key `^[a-z]{2}$` + accepts `"en\n"`. ## [0.26.0] — 2026-11-01 diff --git a/fluid_build/cli/odcs.py b/fluid_build/cli/odcs.py index b277d913e..736785014 100644 --- a/fluid_build/cli/odcs.py +++ b/fluid_build/cli/odcs.py @@ -504,6 +504,15 @@ def _validate_imported_contract(contract: dict) -> list: return [] if result.is_valid else list(result.errors) +def _brief(message: str, limit: int = 120) -> str: + """``message`` cut to ``limit`` characters, keeping a line-break hint whole: cut, + ``... block keeps and |`` reads as advice to use ``|``.""" + from fluid_build.schema_manager import LINE_BREAK_HINT + + head = message.removesuffix(LINE_BREAK_HINT) + return head[:limit] + message[len(head) :] + + def _run_odcs_validate(args): """Run ODCS validate command — multi-pass with structured report. @@ -559,7 +568,7 @@ def _run_odcs_validate(args): report["ok"] = False cprint(f"✗ jsonschema: {len(errors)} error(s)") for e in errors[:10]: - cprint(f" {e['path'] or ''}: {e['message'][:120]}") + cprint(f" {e['path'] or ''}: {_brief(e['message'])}") if len(errors) > 10: cprint(f" ... +{len(errors) - 10} more") else: diff --git a/fluid_build/forge/core/artifact_validators.py b/fluid_build/forge/core/artifact_validators.py index c568e9888..1d0c99f97 100644 --- a/fluid_build/forge/core/artifact_validators.py +++ b/fluid_build/forge/core/artifact_validators.py @@ -36,7 +36,6 @@ from __future__ import annotations import hashlib -import importlib.util import json import logging import shutil @@ -113,10 +112,6 @@ def _load_schema(path: Path) -> Optional[Dict[str, Any]]: # --------------------------------------------------------------------------- -def _jsonschema_available() -> bool: - return importlib.util.find_spec("jsonschema") is not None - - def _conftest_available() -> bool: """``conftest`` binary on PATH — independent of python package install.""" return shutil.which("conftest") is not None @@ -401,7 +396,9 @@ def validate_opds(path: str, content: bytes) -> List[ValidationIssue]: ) ] - if not _jsonschema_available(): + from fluid_build.schema_manager import JSONSCHEMA_AVAILABLE + + if not JSONSCHEMA_AVAILABLE: return [ ValidationIssue( file=path, diff --git a/fluid_build/providers/odps_standard/validation.py b/fluid_build/providers/odps_standard/validation.py index 4d3bd9bb3..d3a0b183c 100644 --- a/fluid_build/providers/odps_standard/validation.py +++ b/fluid_build/providers/odps_standard/validation.py @@ -75,7 +75,8 @@ def schema_for_document(odps: Mapping[str, Any]) -> Optional[Dict[str, Any]]: def validate(odps: Mapping[str, Any], schema: Mapping[str, Any]) -> None: - """Validate an ODPS product against the vendored v1.0.0 JSON Schema.""" + """Validate an ODPS product against ``schema``, the vendored one for its + ``apiVersion`` (:func:`schema_for_document`).""" try: import jsonschema except ImportError: diff --git a/fluid_build/schema_manager.py b/fluid_build/schema_manager.py index d78c06a03..e4f0b0c7b 100644 --- a/fluid_build/schema_manager.py +++ b/fluid_build/schema_manager.py @@ -43,6 +43,7 @@ # anywhere in this module — and RefResolver is deprecated, so the day a # jsonschema release drops it this ImportError would have silently disabled # schema validation across the CLI. Never name more in a probe than you use. +_JSONSCHEMA_IMPORT_ERROR: Optional[ImportError] = None try: import jsonschema @@ -66,6 +67,9 @@ #: Validators whose jsonschema message names only keys, never a value. _KEY_ONLY_VALIDATORS = frozenset({"required", "additionalProperties"}) +#: Ends a ``pattern`` error when the value's final line break is all that fails it. +LINE_BREAK_HINT = " (it ends in a line break, which a YAML | or > block keeps and |- or >- drops)" + def format_schema_error(error: Any) -> str: """``error.message``, without the failing value under :data:`_UNQUOTED_VALUE_PATHS`. @@ -175,7 +179,7 @@ def _pattern_keyword(validator: Any, pattern: str, instance: Any, schema: Any) - return message = f"{instance!r} does not match {pattern!r}" if re.search(pattern, instance): # only the final line break fails - message += " (it ends in a line break, which a YAML | or > block keeps and |- or >- drops)" + message += LINE_BREAK_HINT yield jsonschema.ValidationError(message) @@ -189,7 +193,9 @@ def schema_validator(schema: Mapping[str, Any], default: Any = None) -> Any: ``pattern`` ends at the end of the value (:func:`_end_anchored`). Validate against a FLUID schema, or a vendored or plugin one, through this, never ``validator_for`` directly. ``default`` is the dialect of a schema that - declares none, jsonschema's newest when not given. + declares none, jsonschema's newest when not given. Raises ``ImportError`` + when jsonschema could not be imported; check :data:`JSONSCHEMA_AVAILABLE` + first to skip instead. The keyword is replaced the way check-jsonschema replaces it, with ``jsonschema.validators.extend``. Two places keep jsonschema's own @@ -201,6 +207,8 @@ def schema_validator(schema: Mapping[str, Any], default: Any = None) -> Any: ``patternProperties``, with ``^[a-z]{2}$`` language codes. A test fails if either changes (``_KEY_PATTERNS`` in tests/test_schema_pattern_end_anchor.py). """ + if not JSONSCHEMA_AVAILABLE: + raise ImportError("jsonschema is not available") from _JSONSCHEMA_IMPORT_ERROR if default is None: validator_class = jsonschema.validators.validator_for(schema) else: diff --git a/tests/test_standard_schema_pattern_end_anchor.py b/tests/test_standard_schema_pattern_end_anchor.py index 9cbdab12a..196880458 100644 --- a/tests/test_standard_schema_pattern_end_anchor.py +++ b/tests/test_standard_schema_pattern_end_anchor.py @@ -29,16 +29,18 @@ from __future__ import annotations +import ast import re from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, Iterator import jsonschema import pytest import yaml +import fluid_build import fluid_build.extension_schemas as extension_schemas -from fluid_build.forge.core.artifact_validators import validate_odcs +from fluid_build.forge.core.artifact_validators import validate_odcs, validate_opds from fluid_build.forge_datamodel.emit import osi_sidecar from fluid_build.providers.base import ProviderError from fluid_build.providers.odcs.validation import collect_errors, load_schema @@ -61,10 +63,10 @@ def _odcs(table_id: str) -> Dict[str, Any]: return {**_ODCS_BASE, "schema": [{"id": table_id, "name": "orders", "logicalType": "object"}]} -def _odcs_yaml(block: str) -> str: +def _odcs_yaml(block: str, table_id: str = "t1") -> str: """An ODCS document whose ``schema[0].id`` is a YAML block scalar: ``|`` or ``|-``.""" body = yaml.safe_dump(_ODCS_BASE) - return f"{body}schema:\n- id: {block}\n t1\n name: orders\n logicalType: object\n" + return f"{body}schema:\n- id: {block}\n {table_id}\n name: orders\n logicalType: object\n" # ── ODCS: fluid odcs validate, fluid validate-artifacts ───────────────────── @@ -96,6 +98,21 @@ def test_fluid_odcs_validate_refuses_a_block_scalar_table_id(tmp_path: Path, cap assert "schema.0.id: 't1\\n' does not match" in text, text +def test_fluid_odcs_validate_keeps_the_hint_whole(tmp_path: Path, capsys: Any) -> None: + """The command cuts each message to 120 characters. Cut inside the hint, a long id's + message ended ``... block keeps and |``, which reads as advice to use ``|``.""" + from fluid_build.cli import main + + path = tmp_path / "orders.odcs.yaml" + path.write_text(_odcs_yaml("|", "customer_orders_v1"), encoding="utf-8") + + rc = main(["odcs", "validate", str(path), "--no-vowl"]) + + text = " ".join(capsys.readouterr().out.split()) # undo the console's line wrapping + assert rc == 1, text + assert f"'customer_orders_v1\\n' does not match '^[A-Za-z0-9_-]+$'{_LINE_BREAK_HINT}" in text + + def test_validate_artifacts_refuses_a_block_scalar_table_id() -> None: """``validate_odcs`` is what ``fluid validate-artifacts`` runs on ``odcs/*.odcs.yaml``.""" assert validate_odcs("odcs/orders.odcs.yaml", _odcs_yaml("|-").encode()) == [] @@ -133,6 +150,17 @@ def test_opds_validation_reads_a_pattern_as_ecma_does(monkeypatch: Any) -> None: assert errors == [f"name: 'orders\\n' does not match '^[a-z]+$'{_LINE_BREAK_HINT}"] +def test_an_opds_schema_that_declares_no_dialect_is_2020_12(monkeypatch: Any) -> None: + """What the check pinned before. Draft 7 has no ``dependentRequired`` and would + pass this document.""" + schema = {"type": "object", "dependentRequired": {"a": ["b"]}} + monkeypatch.setitem(opds_validator._SCHEMA_CACHE, "0.0-test", schema) + + valid, errors, _ = opds_validator.validate_against_opds_schema({"a": 1}, "0.0-test") + + assert (valid, errors) == (False, ["root: 'b' is a dependency of 'a'"]) + + def test_ossie_document_validation_reads_a_pattern_as_ecma_does(monkeypatch: Any) -> None: monkeypatch.setattr(osi_sidecar, "_load_ossie_schema", lambda: _NAME_SCHEMA) assert osi_sidecar.validate_osi_document({"name": "orders"}) == [] @@ -142,6 +170,18 @@ def test_ossie_document_validation_reads_a_pattern_as_ecma_does(monkeypatch: Any assert issues == [f"name: 'orders\\n' does not match '^[a-z]+$'{_LINE_BREAK_HINT}"] +def test_without_jsonschema_each_check_says_it_is_missing(monkeypatch: Any) -> None: + """The checks read the flag ``schema_validator`` needs. A jsonschema that is found + but fails to import is reported as missing, not as ``NameError: jsonschema``.""" + monkeypatch.setattr("fluid_build.schema_manager.JSONSCHEMA_AVAILABLE", False) + + assert [i.code for i in validate_odcs("odcs/o.odcs.yaml", b"id: x")] == ["ODCS-LIB-MISSING"] + assert [i.code for i in validate_opds("opds/o.opds.json", b"{}")] == ["OPDS-LIB-MISSING"] + assert collect_errors(_odcs("t1\n"), {}) == [] + with pytest.raises(ImportError, match="jsonschema is not available"): + osi_sidecar.validate_osi_document({}) + + # ── plugin extension blocks ───────────────────────────────────────────────── #: The shape of data-product-forge-custom-scaffold's schema: draft-07, "$"-anchored. @@ -195,3 +235,30 @@ def test_an_extension_schema_that_declares_no_dialect_is_draft_07(monkeypatch: A kept = {"output": "x", "patterns": []} assert _assemble(monkeypatch, schema, kept) == {"x": kept} assert _assemble(monkeypatch, schema, {"output": "x"}) == {} + + +# ── one way to build a validator ──────────────────────────────────────────── + + +def _validator_builds(path: Path) -> Iterator[str]: + """Calls in ``path`` that build a jsonschema validator themselves.""" + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if not isinstance(node, ast.Call): + continue + func = node.func + name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "") + if name == "validator_for" or re.fullmatch(r"Draft\d+Validator", name): + yield f"{path.name}:{node.lineno}: {name}()" + elif name == "validate" and ast.unparse(func) == "jsonschema.validate": + if all(k.arg != "cls" for k in node.keywords): + yield f"{path.name}:{node.lineno}: jsonschema.validate()" + + +def test_every_validator_is_built_by_schema_validator() -> None: + """A check that builds its own validator reads ``$`` as Python does again; #799 and + this change each found such checks one at a time. Give ``schema_validator`` the + dialect class as ``default=`` instead of building it.""" + package = Path(fluid_build.__file__).parent + paths = [p for p in sorted(package.rglob("*.py")) if p != package / "schema_manager.py"] + assert len(paths) > 500, len(paths) # the glob found the package + assert [build for path in paths for build in _validator_builds(path)] == [] From c1a2b3771b21d604191ba0da899b703dda20f97c Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:19:51 +0100 Subject: [PATCH 3/4] fix: a bracket in a value or an unusable plugin schema no longer stops the run Defect-scan follow-ups for the vendored-schema end-anchor change: - fluid odcs validate printed each schema error as Rich markup, so a failing value with "[/]" in it ended the command with "Unexpected error: closing tag '[/]' ... has nothing to close" (exit 2, also on main). The line is printed with markup=False, like the vowl line in the same function. - assemble_proposed_extensions raised when a plugin schema's $schema was not a string: validator_for cannot look it up, where Draft7Validator ignored it. A schema that cannot be applied now drops its block with a redacted warning, as a block that fails its schema is dropped, and the run goes on. This also covers {"type": 5}, which raised on main. - The guard test also catches `from jsonschema import validate`, `import jsonschema as js` with js.validate(...), jsonschema.validators.validate/extend/create, and a cls= that does not come from schema_validator. ODPS passes cls=type(schema_validator(...)) inline so the guard can see it. It parses only files that mention jsonschema. --- CHANGELOG.md | 5 ++ fluid_build/cli/odcs.py | 3 +- fluid_build/extension_schemas.py | 12 +++- .../providers/odps_standard/validation.py | 3 +- ...test_standard_schema_pattern_end_anchor.py | 61 +++++++++++++++++-- 5 files changed, 74 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc8066434..903495d20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -257,6 +257,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 was applied and suggests a rename. A mode without a build phase, and a contract without builds, are not checked, as before. +- **`fluid odcs validate` stopped with `Unexpected error: closing tag '[/]' ... has nothing to + close`** when a value that fails a schema check contained `[/]` or another Rich markup tag. The + error line quotes the value, and it was printed as markup. It is printed as plain text now, so + the command reports the error and exits 1. + - **`fluid diff --baseline` reported nothing for column changes in a 0.7.x contract.** It read columns from `exposes[].schema`, but a 0.7.x contract keeps them in `exposes[].contract.schema` (as does every `fluid init` template), so a removed column, a type change or a new NOT NULL diff --git a/fluid_build/cli/odcs.py b/fluid_build/cli/odcs.py index 736785014..bebc294e3 100644 --- a/fluid_build/cli/odcs.py +++ b/fluid_build/cli/odcs.py @@ -568,7 +568,8 @@ def _run_odcs_validate(args): report["ok"] = False cprint(f"✗ jsonschema: {len(errors)} error(s)") for e in errors[:10]: - cprint(f" {e['path'] or ''}: {_brief(e['message'])}") + # markup=False: the message quotes the value, and "[/]" in it is a Rich tag. + cprint(f" {e['path'] or ''}: {_brief(e['message'])}", markup=False) if len(errors) > 10: cprint(f" ... +{len(errors) - 10} more") else: diff --git a/fluid_build/extension_schemas.py b/fluid_build/extension_schemas.py index 82ab6aa1e..98a3555ce 100644 --- a/fluid_build/extension_schemas.py +++ b/fluid_build/extension_schemas.py @@ -212,8 +212,16 @@ def assemble_proposed_extensions( # The dialect the plugin's schema declares, draft-07 when it declares # none; Draft 7 ignores a newer keyword rather than enforcing it. A # pattern's "$" is the end of the value, as ECMA-262 reads it. - validator = schema_validator(schema, default=Draft7Validator) - if next(iter(validator.iter_errors(block)), None) is not None: + try: + validator = schema_validator(schema, default=Draft7Validator) + invalid = next(iter(validator.iter_errors(block)), None) is not None + except Exception as e: # a schema that cannot be applied, e.g. a non-string $schema + from fluid_build.observability.secret_redactor import redact_secret_text + + log = logger or logging.getLogger(__name__) + log.warning("%s", redact_secret_text(f"extension schema {key!r} unusable: {e}")) + continue # unchecked — don't emit it + if invalid: continue # invalid — let the repair loop surface it; don't emit junk out[key] = block return out diff --git a/fluid_build/providers/odps_standard/validation.py b/fluid_build/providers/odps_standard/validation.py index d3a0b183c..7f33d6bca 100644 --- a/fluid_build/providers/odps_standard/validation.py +++ b/fluid_build/providers/odps_standard/validation.py @@ -87,7 +87,6 @@ def validate(odps: Mapping[str, Any], schema: Mapping[str, Any]) -> None: try: # cls: the declared dialect, with a pattern's "$" read as the end of the # value, as ECMA-262 does. jsonschema.validate still checks the schema first. - cls = type(schema_validator(schema)) - jsonschema.validate(instance=odps, schema=schema, cls=cls) + jsonschema.validate(instance=odps, schema=schema, cls=type(schema_validator(schema))) except jsonschema.ValidationError as exc: raise ProviderError(f"ODPS validation failed: {exc.message}") from exc diff --git a/tests/test_standard_schema_pattern_end_anchor.py b/tests/test_standard_schema_pattern_end_anchor.py index 196880458..23eb38ec3 100644 --- a/tests/test_standard_schema_pattern_end_anchor.py +++ b/tests/test_standard_schema_pattern_end_anchor.py @@ -113,6 +113,23 @@ def test_fluid_odcs_validate_keeps_the_hint_whole(tmp_path: Path, capsys: Any) - assert f"'customer_orders_v1\\n' does not match '^[A-Za-z0-9_-]+$'{_LINE_BREAK_HINT}" in text +def test_fluid_odcs_validate_prints_a_value_with_brackets_literally( + tmp_path: Path, capsys: Any +) -> None: + """The message quotes the value, and Rich read ``[/]`` in it as a closing tag: the + command ended in ``Unexpected error: closing tag '[/]' ... has nothing to close``.""" + from fluid_build.cli import main + + path = tmp_path / "orders.odcs.yaml" + path.write_text(yaml.safe_dump(_odcs("t[/]1")), encoding="utf-8") + + rc = main(["odcs", "validate", str(path), "--no-vowl"]) + + text = " ".join(capsys.readouterr().out.split()) + assert rc == 1, text + assert "schema.0.id: 't[/]1' does not match" in text, text + + def test_validate_artifacts_refuses_a_block_scalar_table_id() -> None: """``validate_odcs`` is what ``fluid validate-artifacts`` runs on ``odcs/*.odcs.yaml``.""" assert validate_odcs("odcs/orders.odcs.yaml", _odcs_yaml("|-").encode()) == [] @@ -228,6 +245,18 @@ def test_an_extension_schema_is_read_in_the_dialect_it_declares(monkeypatch: Any assert _assemble(monkeypatch, schema, {"libraries": [], "librarys": []}) == {} +def test_an_extension_schema_that_cannot_be_applied_drops_only_its_block( + monkeypatch: Any, caplog: Any +) -> None: + """A malformed plugin schema neither stops the run nor lets its block through.""" + schemas = {"bad": {"$schema": 7, "type": "string"}, "x": _SCAFFOLD_SCHEMA} + monkeypatch.setattr(extension_schemas, "iter_extension_schemas", lambda *a, **k: schemas) + kept = {"patterns": [{"use": "ci:basic"}]} + + assert extension_schemas.assemble_proposed_extensions({"bad": "a", "x": kept}) == {"x": kept} + assert "extension schema 'bad' unusable" in caplog.text + + def test_an_extension_schema_that_declares_no_dialect_is_draft_07(monkeypatch: Any) -> None: """The plugin contract says draft-07. 2020-12, jsonschema's default, has no ``dependencies`` keyword and would keep this block.""" @@ -241,17 +270,39 @@ def test_an_extension_schema_that_declares_no_dialect_is_draft_07(monkeypatch: A def _validator_builds(path: Path) -> Iterator[str]: - """Calls in ``path`` that build a jsonschema validator themselves.""" - for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + """Spellings in ``path`` that build a jsonschema validator without ``schema_validator``. + It goes by names, so a class first bound to another name (``V = Draft7Validator``) + gets past it.""" + text = path.read_text(encoding="utf-8") + if not any(word in text for word in ("jsonschema", "Validator", "validator_for")): + return + tree = ast.parse(text) + modules = {"jsonschema"} # names that hold jsonschema or jsonschema.validators + for node in ast.walk(tree): + if isinstance(node, ast.Import): + modules |= {a.asname for a in node.names if a.name == "jsonschema" and a.asname} + elif isinstance(node, ast.ImportFrom) and (node.module or "").startswith("jsonschema"): + modules |= {a.asname or a.name for a in node.names if a.name == "validators"} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and (node.module or "").startswith("jsonschema"): + for a in node.names: + builder = a.name == "validator_for" or re.fullmatch(r"Draft\d+Validator", a.name) + if a.name == "validate" or (a.asname and builder): + yield f"{path.name}:{node.lineno}: from {node.module} import {a.name}" if not isinstance(node, ast.Call): continue func = node.func name = func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "") if name == "validator_for" or re.fullmatch(r"Draft\d+Validator", name): yield f"{path.name}:{node.lineno}: {name}()" - elif name == "validate" and ast.unparse(func) == "jsonschema.validate": - if all(k.arg != "cls" for k in node.keywords): - yield f"{path.name}:{node.lineno}: jsonschema.validate()" + elif name in ("validate", "extend", "create") and isinstance(func, ast.Attribute): + root = func.value + while isinstance(root, ast.Attribute): + root = root.value + cls = next((k.value for k in node.keywords if k.arg == "cls"), None) + routed = cls is not None and "schema_validator" in ast.unparse(cls) + if isinstance(root, ast.Name) and root.id in modules and not routed: + yield f"{path.name}:{node.lineno}: {ast.unparse(func)}()" def test_every_validator_is_built_by_schema_validator() -> None: From 88ba59bf6b8ff482fda871399c10f7c116ddc5a9 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:22:43 +0100 Subject: [PATCH 4/4] docs(changelog): keep the two pattern end-anchor entries together The rebase onto main put two new fluid diff --baseline entries between the FLUID-schema entry (#799) and this branch's, which begins "The fix above covered the FLUID schemas only". --- CHANGELOG.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 903495d20..49f160784 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -582,6 +582,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and |- or >- drops)`. A `sink.catalogAuth` field gets no hint, since its value is never shown. The schemas themselves are unchanged. +- **The ODCS and plugin-extension checks let a value with a trailing newline through a `pattern`, + too.** The fix above covered the FLUID schemas only. `fluid odcs validate` and + `fluid validate-artifacts` accepted an ODCS `schema[].id: "t1\n"` while refusing `"t 1"`, and + the forge copilot kept an LLM-proposed extension block whose value ended in a line break, such + as `use: "ci:basic\n"` for `customScaffold`. Every JSON Schema check now reads `$` as the end of + the value, including the ODPS, OPDS and Ossie checks, whose vendored schemas have no such + pattern yet. Their errors carry the same hint; an extension block that fails is dropped, as + before. An extension schema is read in the dialect its `$schema` declares, and as draft-07 when + it declares none, so a 2020-12 `unevaluatedProperties: false` is no longer ignored. + `patternProperties` keys are still matched as before, so OPDS's language-code key `^[a-z]{2}$` + accepts `"en\n"`. + - **`fluid diff --baseline` compared contracts that no schema accepts.** Its rules read only the shapes a schema allows, so a value of the wrong type hid a change: a column retyped from `decimal(18,2)` to the number `18` passed `--fail-on-breaking` with exit 0. Both contracts are @@ -607,18 +619,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ``- **column_added** `exposes[0].contract.schema.email` — `column 'email' added to expose 'orders'` ``. The `fluid forge --refine` preview prints the same text without markup. -- **The ODCS and plugin-extension checks let a value with a trailing newline through a `pattern`, - too.** The fix above covered the FLUID schemas only. `fluid odcs validate` and - `fluid validate-artifacts` accepted an ODCS `schema[].id: "t1\n"` while refusing `"t 1"`, and - the forge copilot kept an LLM-proposed extension block whose value ended in a line break, such - as `use: "ci:basic\n"` for `customScaffold`. Every JSON Schema check now reads `$` as the end of - the value, including the ODPS, OPDS and Ossie checks, whose vendored schemas have no such - pattern yet. Their errors carry the same hint; an extension block that fails is dropped, as - before. An extension schema is read in the dialect its `$schema` declares, and as draft-07 when - it declares none, so a 2020-12 `unevaluatedProperties: false` is no longer ignored. - `patternProperties` keys are still matched as before, so OPDS's language-code key `^[a-z]{2}$` - accepts `"en\n"`. - ## [0.26.0] — 2026-11-01 ### Added