security: the ODCS and plugin-extension checks let a trailing newline through a pattern

CHANGELOG.md

@@ -266,6 +266,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   was applied and suggests a rename. A mode without a build phase, and a contract without builds,
   are not checked, as before.
 
+- **`fluid odcs validate` stopped with `Unexpected error: closing tag '[/]' ... has nothing to
+  close`** when a value that fails a schema check contained `[/]` or another Rich markup tag. The
+  error line quotes the value, and it was printed as markup. It is printed as plain text now, so
+  the command reports the error and exits 1.
+
 - **A missing optional dependency names the extra to install, and silent fallbacks warn.**
   litellm, the MCP SDK, openlineage, boto3 and duckdb are imported through one gate: when one is
   missing, the command prints `<feature> needs <package>, which is not installed.` and
@@ -595,6 +600,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   and |- or >- drops)`. A `sink.catalogAuth` field gets no hint, since its value is never shown.
   The schemas themselves are unchanged.
 
+- **The ODCS and plugin-extension checks let a value with a trailing newline through a `pattern`,
+  too.** The fix above covered the FLUID schemas only. `fluid odcs validate` and
+  `fluid validate-artifacts` accepted an ODCS `schema[].id: "t1\n"` while refusing `"t 1"`, and
+  the forge copilot kept an LLM-proposed extension block whose value ended in a line break, such
+  as `use: "ci:basic\n"` for `customScaffold`. Every JSON Schema check now reads `$` as the end of
+  the value, including the ODPS, OPDS and Ossie checks, whose vendored schemas have no such
+  pattern yet. Their errors carry the same hint; an extension block that fails is dropped, as
+  before. An extension schema is read in the dialect its `$schema` declares, and as draft-07 when
+  it declares none, so a 2020-12 `unevaluatedProperties: false` is no longer ignored.
+  `patternProperties` keys are still matched as before, so OPDS's language-code key `^[a-z]{2}$`
+  accepts `"en\n"`.
+
 - **`fluid diff --baseline` compared contracts that no schema accepts.** Its rules read only the
   shapes a schema allows, so a value of the wrong type hid a change: a column retyped from
   `decimal(18,2)` to the number `18` passed `--fail-on-breaking` with exit 0. Both contracts are

fluid_build/cli/odcs.py

@@ -504,6 +504,15 @@ def _validate_imported_contract(contract: dict) -> list:
     return [] if result.is_valid else list(result.errors)
 
 
+def _brief(message: str, limit: int = 120) -> str:
+    """``message`` cut to ``limit`` characters, keeping a line-break hint whole: cut,
+    ``... block keeps and |`` reads as advice to use ``|``."""
+    from fluid_build.schema_manager import LINE_BREAK_HINT
+
+    head = message.removesuffix(LINE_BREAK_HINT)
+    return head[:limit] + message[len(head) :]
+
+
 def _run_odcs_validate(args):
     """Run ODCS validate command — multi-pass with structured report.
 
@@ -559,7 +568,8 @@ def _run_odcs_validate(args):
             report["ok"] = False
             cprint(f"✗ jsonschema: {len(errors)} error(s)")
             for e in errors[:10]:
-                cprint(f"   {e['path'] or '<root>'}: {e['message'][:120]}")
+                # markup=False: the message quotes the value, and "[/]" in it is a Rich tag.
+                cprint(f"   {e['path'] or '<root>'}: {_brief(e['message'])}", markup=False)
             if len(errors) > 10:
                 cprint(f"   ... +{len(errors) - 10} more")
         else:

fluid_build/extension_schemas.py

@@ -22,7 +22,8 @@
     customScaffold = "data_product_forge_custom_scaffold.validation:get_extension_schema"
 
 The provider has the signature ``get_extension_schema(fluid_version=None) -> dict``
-and returns a draft-07 JSON Schema describing the data *under* the extension key.
+and returns a JSON Schema describing the data *under* the extension key: draft-07,
+unless the schema declares another dialect in its ``$schema``.
 
 These helpers let the ``fluid forge`` copilot (a) ground contract generation on
 the installed extension schemas and (b) validate the generated blocks before
@@ -200,14 +201,27 @@ def assemble_proposed_extensions(
         from jsonschema import Draft7Validator
     except ImportError:  # pragma: no cover — jsonschema is a hard dependency
         Draft7Validator = None  # type: ignore[assignment]
+    from fluid_build.schema_manager import schema_validator
 
     out: Dict[str, Any] = {}
     for key, block in proposed.items():
         schema = schemas.get(key)
         if schema is None:
             continue
         if Draft7Validator is not None:
-            if next(iter(Draft7Validator(schema).iter_errors(block)), None) is not None:
+            # The dialect the plugin's schema declares, draft-07 when it declares
+            # none; Draft 7 ignores a newer keyword rather than enforcing it. A
+            # pattern's "$" is the end of the value, as ECMA-262 reads it.
+            try:
+                validator = schema_validator(schema, default=Draft7Validator)
+                invalid = next(iter(validator.iter_errors(block)), None) is not None
+            except Exception as e:  # a schema that cannot be applied, e.g. a non-string $schema
+                from fluid_build.observability.secret_redactor import redact_secret_text
+
+                log = logger or logging.getLogger(__name__)
+                log.warning("%s", redact_secret_text(f"extension schema {key!r} unusable: {e}"))
+                continue  # unchecked — don't emit it
+            if invalid:
                 continue  # invalid — let the repair loop surface it; don't emit junk
         out[key] = block
     return out

fluid_build/forge/core/artifact_validators.py

@@ -36,7 +36,6 @@
 from __future__ import annotations
 
 import hashlib
-import importlib.util
 import json
 import logging
 import shutil
@@ -113,10 +112,6 @@ def _load_schema(path: Path) -> Optional[Dict[str, Any]]:
 # ---------------------------------------------------------------------------
 
 
-def _jsonschema_available() -> bool:
-    return importlib.util.find_spec("jsonschema") is not None
-
-
 def _conftest_available() -> bool:
     """``conftest`` binary on PATH — independent of python package install."""
     return shutil.which("conftest") is not None
@@ -306,7 +301,10 @@ def _validate_against_schema(
             )
         ]
 
-    if not _jsonschema_available():
+    from fluid_build.schema_manager import JSONSCHEMA_AVAILABLE, schema_validator
+
+    # The flag schema_validator needs: a package that is found can still fail to import.
+    if not JSONSCHEMA_AVAILABLE:
         return [
             ValidationIssue(
                 file=artifact_path,
@@ -320,8 +318,6 @@ def _validate_against_schema(
             )
         ]
 
-    import jsonschema  # type: ignore
-
     issues: List[ValidationIssue] = []
     # Validate with the dialect the schema DECLARES, not a hardcoded one.
     #
@@ -335,7 +331,10 @@ def _validate_against_schema(
     # and guards nine objects with `unevaluatedProperties: false`. A server entry
     # carrying an unexpected key validated with ZERO errors, reachable from
     # `fluid validate-artifacts`. See tests/test_jsonschema_dialect.py.
-    validator = jsonschema.validators.validator_for(schema)(schema)
+    #
+    # schema_validator also reads a pattern's `$` as the end of the value, as
+    # ECMA-262 does: ODCS refused `schema[].id: "t 1"` but took `"t1\n"`.
+    validator = schema_validator(schema)
 
     errors = sorted(validator.iter_errors(doc), key=lambda e: list(e.absolute_path))
     for err in errors:
@@ -397,7 +396,9 @@ def validate_opds(path: str, content: bytes) -> List[ValidationIssue]:
             )
         ]
 
-    if not _jsonschema_available():
+    from fluid_build.schema_manager import JSONSCHEMA_AVAILABLE
+
+    if not JSONSCHEMA_AVAILABLE:
         return [
             ValidationIssue(
                 file=path,

fluid_build/forge_datamodel/emit/osi_sidecar.py

@@ -93,14 +93,14 @@ def validate_osi_document(document: Dict[str, Any]) -> List[str]:
     Returns a sorted list of human-readable issue strings (empty when
     conformant). ``jsonschema`` is imported lazily — this module sits on
     the forge pipeline path, not the CLI cold path, but there is no
-    reason to tax importers that never validate.
+    reason to tax importers that never validate. A ``pattern``'s ``$`` is
+    the end of the value, as ECMA-262 reads it.
     """
-    import jsonschema
+    from fluid_build.schema_manager import schema_validator
 
     schema = _load_ossie_schema()
-    validator_cls = jsonschema.validators.validator_for(schema)
     errors = sorted(
-        validator_cls(schema).iter_errors(document),
+        schema_validator(schema).iter_errors(document),
         key=lambda e: ([str(p) for p in e.absolute_path], e.message),
     )
     return [f"{'/'.join(str(p) for p in e.absolute_path) or '<root>'}: {e.message}" for e in errors]

fluid_build/providers/odcs/validation.py

@@ -74,15 +74,17 @@ def collect_errors(odcs: Mapping[str, Any], schema: Mapping[str, Any]) -> List[D
     Empty list = valid. No-op (empty list) when :mod:`jsonschema` is not
     installed. Used by ``validate()`` (which raises on non-empty) and by
     ``fluid odcs validate --report`` (which serializes to JSON for CI).
+
+    A ``pattern``'s ``$`` is the end of the value, as ECMA-262 reads it, so
+    ``schema[].id: "t1\\n"`` fails ``^[A-Za-z0-9_-]+$`` like ``"t 1"`` does.
     """
-    try:
-        import jsonschema
-    except ImportError:
+    from fluid_build.schema_manager import JSONSCHEMA_AVAILABLE, schema_validator
+
+    if not JSONSCHEMA_AVAILABLE:
         LOG.warning("jsonschema not installed, skipping ODCS validation")
         return []
 
-    validator_cls = jsonschema.validators.validator_for(schema)
-    validator = validator_cls(schema)
+    validator = schema_validator(schema)
     out: List[Dict[str, Any]] = []
     for err in validator.iter_errors(odcs):
         out.append(

fluid_build/providers/odps_standard/validation.py

@@ -75,13 +75,18 @@ def schema_for_document(odps: Mapping[str, Any]) -> Optional[Dict[str, Any]]:
 
 
 def validate(odps: Mapping[str, Any], schema: Mapping[str, Any]) -> None:
-    """Validate an ODPS product against the vendored v1.0.0 JSON Schema."""
+    """Validate an ODPS product against ``schema``, the vendored one for its
+    ``apiVersion`` (:func:`schema_for_document`)."""
     try:
         import jsonschema
     except ImportError:
         LOG.warning("jsonschema not installed, skipping ODPS validation")
         return
+    from fluid_build.schema_manager import schema_validator
+
     try:
-        jsonschema.validate(instance=odps, schema=schema)
+        # cls: the declared dialect, with a pattern's "$" read as the end of the
+        # value, as ECMA-262 does. jsonschema.validate still checks the schema first.
+        jsonschema.validate(instance=odps, schema=schema, cls=type(schema_validator(schema)))
     except jsonschema.ValidationError as exc:
         raise ProviderError(f"ODPS validation failed: {exc.message}") from exc

fluid_build/providers/opds/validator.py

@@ -119,8 +119,14 @@ def validate_against_opds_schema(
         valid, errors = _basic_validation(opds_data, version)
         return valid, errors, "basic"
 
+    from fluid_build.schema_manager import schema_validator
+
     try:
-        validator = jsonschema.Draft202012Validator(schema)
+        # The dialect the schema declares (v4.1: 2020-12, also the default),
+        # with a pattern's "$" read as the end of the value, as ECMA-262 does.
+        # The patternProperties language-code keys (^[a-z]{2}$) keep
+        # jsonschema's own re.search; see schema_manager.schema_validator.
+        validator = schema_validator(schema, default=jsonschema.Draft202012Validator)
         errors = list(validator.iter_errors(opds_data))
 
         # Filter a known upstream OPDS v4.1 schema false-positive: the

fluid_build/schema_manager.py

@@ -28,7 +28,7 @@
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Any, Callable, Dict, FrozenSet, Iterator, List, Optional, Union
+from typing import Any, Callable, Dict, FrozenSet, Iterator, List, Mapping, Optional, Union
 
 import yaml
 
@@ -43,6 +43,7 @@
 # anywhere in this module — and RefResolver is deprecated, so the day a
 # jsonschema release drops it this ImportError would have silently disabled
 # schema validation across the CLI. Never name more in a probe than you use.
+_JSONSCHEMA_IMPORT_ERROR: Optional[ImportError] = None
 try:
     import jsonschema
 
@@ -66,6 +67,9 @@
 #: Validators whose jsonschema message names only keys, never a value.
 _KEY_ONLY_VALIDATORS = frozenset({"required", "additionalProperties"})
 
+#: Ends a ``pattern`` error when the value's final line break is all that fails it.
+LINE_BREAK_HINT = " (it ends in a line break, which a YAML | or > block keeps and |- or >- drops)"
+
 
 def format_schema_error(error: Any) -> str:
     """``error.message``, without the failing value under :data:`_UNQUOTED_VALUE_PATHS`.
@@ -175,7 +179,7 @@ def _pattern_keyword(validator: Any, pattern: str, instance: Any, schema: Any) -
         return
     message = f"{instance!r} does not match {pattern!r}"
     if re.search(pattern, instance):  # only the final line break fails
-        message += " (it ends in a line break, which a YAML | or > block keeps and |- or >- drops)"
+        message += LINE_BREAK_HINT
     yield jsonschema.ValidationError(message)
 
 
@@ -184,20 +188,32 @@ def _end_anchored_validator(validator_class: Any) -> Any:
     return jsonschema.validators.extend(validator_class, {"pattern": _pattern_keyword})
 
 
-def schema_validator(schema: Dict[str, Any]) -> Any:
+def schema_validator(schema: Mapping[str, Any], default: Any = None) -> Any:
     """A jsonschema validator for ``schema``, in the dialect it declares, whose
     ``pattern`` ends at the end of the value (:func:`_end_anchored`). Validate
-    against a FLUID schema through this, never ``validator_for`` directly.
+    against a FLUID schema, or a vendored or plugin one, through this, never
+    ``validator_for`` directly. ``default`` is the dialect of a schema that
+    declares none, jsonschema's newest when not given. Raises ``ImportError``
+    when jsonschema could not be imported; check :data:`JSONSCHEMA_AVAILABLE`
+    first to skip instead.
 
     The keyword is replaced the way check-jsonschema replaces it, with
     ``jsonschema.validators.extend``. Two places keep jsonschema's own
     ``re.search``: ``patternProperties`` keys, as its ``additionalProperties``
     and :func:`_locate_error` read them, and a subschema that declares its own
-    ``$schema`` (or a ``$ref`` to the root, which does), for which jsonschema
-    picks that dialect's own class. No FLUID schema has either; a test fails if
-    one does.
+    ``$schema`` (or a ``$ref``, ``$recursiveRef`` or ``$dynamicRef`` to the
+    root, which does), for which jsonschema picks that dialect's own class. No
+    FLUID schema has either. Of the vendored ones, only OPDS keys
+    ``patternProperties``, with ``^[a-z]{2}$`` language codes. A test fails if
+    either changes (``_KEY_PATTERNS`` in tests/test_schema_pattern_end_anchor.py).
     """
-    return _end_anchored_validator(jsonschema.validators.validator_for(schema))(schema)
+    if not JSONSCHEMA_AVAILABLE:
+        raise ImportError("jsonschema is not available") from _JSONSCHEMA_IMPORT_ERROR
+    if default is None:
+        validator_class = jsonschema.validators.validator_for(schema)
+    else:
+        validator_class = jsonschema.validators.validator_for(schema, default=default)
+    return _end_anchored_validator(validator_class)(schema)
 
 
 def _schema_version_sort_key(version: str) -> tuple[int, int, int]:

tests/test_schema_pattern_end_anchor.py

@@ -239,29 +239,51 @@ def _nodes(node: Any) -> Iterator[Dict[str, Any]]:
             yield from _nodes(value)
 
 
+#: Every schema fluid validates against: the FLUID and Ossie ones, and the vendored
+#: standards (tests/test_standard_schema_pattern_end_anchor.py tests their checks).
+_SHIPPED = sorted(_SCHEMA_DIR.glob("*.json")) + [
+    _SCHEMA_DIR.parent / "providers" / "odcs" / "odcs-schema-v3.1.0.json",
+    _SCHEMA_DIR.parent / "providers" / "odps_standard" / "schemas" / "odps-product-v1.0.0.json",
+    _SCHEMA_DIR.parent / "providers" / "odps_standard" / "schemas" / "odps-product-v1.1.0.json",
+    _SCHEMA_DIR.parent / "providers" / "opds" / "opds-schema-v4.1.0.json",
+]
+
+#: ``patternProperties`` keys, by schema. They keep jsonschema's own ``re.search``:
+#: jsonschema matches them in ``patternProperties``, ``additionalProperties`` and
+#: ``unevaluatedProperties``, and overriding ``patternProperties`` alone, as
+#: check-jsonschema does, lets a key that one matches and another does not through
+#: ``additionalProperties: false`` unchecked. So OPDS's ISO 639-1 language codes,
+#: ``^[a-z]{2}$``, also match ``"en\n"``. Under ``product.details`` and ``declarative``,
+#: which take any key, the value is then checked as a language entry.
+#: ``$defs/Languages`` takes no other key, so a hand-written OPDS file can carry
+#: ``{"en\n": ...}`` there; fluid's emitter writes ``en``.
+_KEY_PATTERNS = {"opds-schema-v4.1.0.json": [("^[a-z]{2}$",)] * 3}
+
+
 def test_every_bundled_schema_pattern_is_rewritten_anchor_for_anchor() -> None:
     """The scan reads escapes and sets as Python does, checked against Python's own
-    parser for every ``pattern`` the bundled schemas hold."""
+    parser for every ``pattern`` the shipped schemas hold."""
     patterns = {
         node["pattern"]
-        for path in _SCHEMA_DIR.glob("fluid-*schema-*.json")
+        for path in _SHIPPED
         for node in _nodes(json.loads(path.read_text(encoding="utf-8")))
         if isinstance(node.get("pattern"), str)
     }
     assert len(patterns) >= 20, patterns  # the glob found the schemas
     assert [p for p in patterns if not _only_end_anchors_changed(p)] == []
 
 
-@pytest.mark.parametrize(
-    "path", sorted(_SCHEMA_DIR.glob("fluid-schema-*.json")), ids=lambda path: path.name
-)
+@pytest.mark.parametrize("path", _SHIPPED, ids=lambda path: path.name)
 def test_every_pattern_of_the_schema_is_checked_with_the_rewrite(path: Path) -> None:
     """jsonschema picks the validator class again for each subschema it enters, by
-    that subschema's ``$schema``. One that declares it, or a ``$ref`` to the root,
-    which does, would be checked with jsonschema's own ``pattern`` again.
-    ``patternProperties`` keys are not rewritten (``schema_validator`` says why), so
-    a schema that starts to use the keyword has to come back here."""
-    below_root = list(_nodes(json.loads(path.read_text(encoding="utf-8"))))[1:]
-    assert [n for n in below_root if isinstance(n.get("$schema"), str)] == []
-    assert [n for n in below_root if n.get("$ref") == "#"] == []
-    assert [n for n in below_root if "patternProperties" in n] == []
+    that subschema's ``$schema``. One that declares it, or a ``$ref``,
+    ``$recursiveRef`` or ``$dynamicRef`` to the root, which does, would be checked
+    with jsonschema's own ``pattern`` again. ``patternProperties`` keys are not
+    rewritten, so a schema that starts to use the keyword has to come back to
+    :data:`_KEY_PATTERNS`."""
+    nodes = list(_nodes(json.loads(path.read_text(encoding="utf-8"))))
+    assert [n for n in nodes[1:] if isinstance(n.get("$schema"), str)] == []
+    assert [n for n in nodes if n.get("$ref") == "#"] == []
+    assert [n for n in nodes if "$recursiveRef" in n or "$dynamicRef" in n] == []
+    keys = [tuple(n["patternProperties"]) for n in nodes if "patternProperties" in n]
+    assert keys == _KEY_PATTERNS.get(path.name, [])